def shell_cmd(cloud_name, machine_id, cmd):
from pssh.clients import ParallelSSHClient
##util.message("# " + str(cmd), "info")
aa, bb, cc, dd, describe, ff, gg = read(cloud_name, machine_id)
if describe == None:
util.message("Node " + str(machine_id) + " not found", "error")
return
key_name = describe['key_name']
host = describe['public_ip']
hosts = host.split()
username, pkey = key.read(key_name)
if username == None:
util.message("key file not found", "error")
return
##util.message("host=" + str(hosts) + ", user="******", pkey=" + str(pkey), "info")
client = ParallelSSHClient(hosts, user=username, pkey=pkey)
output = client.run_command(cmd, use_pty=True, read_timeout=3600)
for host_out in output:
try:
for line in host_out.stdout:
print(line)
except:
time.sleep(3)
continue
return
def setup_server(server, key):
client = ParallelSSHClient([server],
user='******',
pkey=key,
allow_agent=False)
output = client.run_command('rm -rf ~/server_config')
utils.print_pssh_output(output)
output = client.run_command('mkdir ~/server_config')
utils.print_pssh_output(output)
for f in os.listdir(SERVER_CONFIG_DIR):
dest_path = get_server_config_path(f.split('/')[-1])
local_path = os.path.join(SERVER_CONFIG_DIR, f)
fileUpload.upload_file_no_pssh([server], key, local_path, dest_path)
print(f'Wrote file {local_path} to {server}')
output = client.run_command('bash ~/server_config/setup_new_server.sh')
utils.print_pssh_output(output)
with open(utils.get_project_path('all_servers.txt')) as f:
hostnames = {host.strip() for host in f.readlines()}
hostnames.add(server)
hostnames = sorted(hostnames)
with open(utils.get_project_path('all_servers.txt'), 'w') as f:
f.write('\n'.join(hostnames) + '\n')
print(
f"Added {server} to the fleet. Don't forget to commit autogenerated changes to all_servers.txt."
)
async def ssh(ctx, host, user, password, command):
client = ParallelSSHClient(host, user=user, password=password)
output = client.run_command(command)
for host_output in output:
output = "\n".join(host_output.stdout)
await ctx.send(output)
exit_code = host_output.exit_code
def connect_ssh(self, retries=6, user="******"):
kwargs = {"num_retries": retries, "timeout": 10000}
if hasattr(self, "host_config"):
kwargs["host_config"] = self.host_config
else:
kwargs["user"] = user
self.client = ParallelSSHClient(self.hosts, **kwargs)
def __init__(self, external_init_dict=None):
self.logger = logging.getLogger(constants.logging_name)
init_dict = {}
clsname = self.__class__.__name__
if external_init_dict is not None:
self.logger.debug(
"{}: initializing from external dict".format(clsname))
init_dict = external_init_dict
else:
self.logger.debug(
"{}: initializing with default values".format(clsname))
self.hostname = constants.hpc_hostname
self.user = constants.user
self.home_dir = os.path.join(constants.cc_working_dir, self.user)
self.src_data_path = init_dict.get("src_data_path", "./data")
self.template_path = constants.template_path
self.logger.debug("Host being used is {}, under username {}".format(
self.hostname, self.user))
self.keypath = init_dict.get("ssh_key_filename",
constants.ssh_key_filename)
self.client = ParallelSSHClient([self.hostname],
pkey=self.keypath,
user=self.user,
keepalive_seconds=300)
self.remote_abs_working_folder = None
self.remote_working_folder = None
self.active_dataset_name = None
self.live_job_id = None
def command(self, command, timeout=60, sudo=False):
"""Execute a command on the instances. This will be done using an ssh command and potentially with sudo"""
logger.debug(f'Executing {command} with sudo {sudo}.')
client = ParallelSSHClient([i.ip for i in self.hosts], pkey=self.keysfile)
output = client.run_command(command, read_timeout=timeout, sudo=sudo)
client.join()
return output
def test_agent_forwarding(self):
client = ParallelSSHClient(['localhost'], forward_ssh_agent=True,
port=self.port)
output = client.run_command(self.cmd)
stdout = [list(output['localhost'].stdout) for k in output]
expected_stdout = [[self.resp]]
self.assertListEqual(stdout, expected_stdout)
def _exec_scp(self):
client = ParallelSSHClient(self.hosts, port=self.port)
output = client.copy_file(self.source, self.destination, True)
joinall(output, raise_error=True)
nice_output = dict()
for host in output:
nice_output[host] = {'stdout': [], 'stderr': []}
return nice_output
def tryssh(url):
try:
client = ParallelSSHClient([url], user="******")
client.run_command('ls')
except pssh.exceptions.AuthenticationException:
return True
except pssh.exceptions.ConnectionErrorException:
return False
def ssh(hosts, command):
client = ParallelSSHClient(hosts, user='******', password='******')
output = client.run_command(command)
for host, host_output in output.items():
for line in host_output.stdout:
print line
f = open("Computer_Parseble_OUTPUT", "w")
f.write(str(output))
f.close()
def fix_hostnames():
hosts = get_ips()
client = ParallelSSHClient(hosts, user=USER)
host_args = [{
"cmd":
"sudo hostnamectl set-hostname synthetic-bot-up-%s" % (i, )
} for i in range(len(hosts))]
output = client.run_command("%(cmd)s", host_args=host_args)
client.join()
def go():
command = 'uname -a'
nodes = ['localhost']
pw = getpass.getpass()
client = ParallelSSHClient(nodes, password=pw, timeout=3)
client.run_command(command)
output = client.get_last_output()
for node in output:
for line in output[node]['stdout']:
print('{0} {1}'.format(node, line))
def defineConnection(user, password, hosts):
print(f'Defining connection with hosts ({hosts})...', flush=True)
clients = ParallelSSHClient(hosts, user=user, password=password)
output = clients.connect_auth()
joinall(output, raise_error=True)
print('OK!', flush=True)
return clients
def ssh(host_list, cmd, **kargs):
try:
client = ParallelSSHClient(host_list,
user='******',
pkey=f'{sq(args.ssh_key)}')
output = client.run_command(cmd, **kargs)
for host in output:
logger.info(host)
for line in output[host]['stdout']:
logger.info(line)
except:
logger.info('cannot connect to all the hosts')
return
def login_with_pool(self, ipaddress, port, user_passwd_pair_list, pool_size=10):
for user_passwd_pair in user_passwd_pair_list:
try:
client = ParallelSSHClient(hosts=[ipaddress], port=port, user=user_passwd_pair[0],
password=user_passwd_pair[1], num_retries=0, timeout=self.timeout,
pool_size=pool_size)
output = client.run_command('whoami', timeout=self.timeout)
log_success("SSH", ipaddress, port, user_passwd_pair)
except Exception as E:
logger.debug('AuthenticationException: ssh')
continue
finally:
pass
def parallel_ssh(host, user, password, command):
#enable_host_logger()
client = ParallelSSHClient([host], user=user, password=password)
output = client.run_command(command)
client.join()
stdout = ""
for host_output in output:
if host_output.exit_code != 0:
raise Exception("host returned exit code " +
str(host_output.exit_code))
stdout_li = list(host_output.stdout)
for line in stdout_li:
stdout += line + "\n"
return stdout
def login(self, ipaddress, port, user_passwd_pair_list):
parallel_logger = logging.getLogger("parallel")
parallel_logger.setLevel('CRITICAL')
for user_passwd_pair in user_passwd_pair_list:
try:
client = ParallelSSHClient(hosts=[ipaddress], port=port, user=user_passwd_pair[0],
password=user_passwd_pair[1], num_retries=0, timeout=self.timeout)
output = client.run_command('whoami', timeout=self.timeout)
log_success("SSH", ipaddress, port, user_passwd_pair)
except Exception as E:
logger.debug('AuthenticationException: ssh')
continue
finally:
pass
def _exec_ssh(self, cmd):
client = ParallelSSHClient(self.hosts,
user=self.username,
pkey=self.pkey,
password=self.password,
port=self.port)
output = client.run_command(cmd, sudo=self.sudo)
nice_output = dict()
for host_output in output:
host = host_output.host
nice_output[host] = {'stdout': [], 'stderr': []}
nice_output[host]['stdout'] = list(host_output.stdout)
nice_output[host]['stderr'] = list(host_output.stderr)
return nice_output
def _create_cluster_ssh_conns(self):
hostips = [n["publicIpAddress"] for n in self.cluster_nodes]
hostconfigs = [HostConfig(port=n["port"]) for n in self.cluster_nodes]
self._all_ssh = ParallelSSHClient(hostips,
host_config=hostconfigs,
user=self.admin_username)
self._master_ssh = ParallelSSHClient(hostips[:1],
host_config=hostconfigs[:1],
user=self.admin_username)
self._master_scp = SSHClient(hostips[0],
port=hostconfigs[0].port,
user=self.admin_username)
def sar(command, directory, hosts, filenames):
# "keyname": "/Users/mb/Downloads/bilal-us-east.pem.txt"
config = json.load(open('config.json'))
client = ParallelSSHClient(hosts,
user=config["username"],
password="",
pkey=config["keyname"],
timeout=600,
pool_size=64)
if command == "start":
start_parallel(client)
elif command == "terminate":
stop_parallel(client)
elif command == "collectcsv":
count = 0
for host in hosts:
sar_helper(host, directory, filenames[count], command, config)
count += 1
# getfiles_parallel(client, framework, len(hosts), confFile, hosts)
else:
export_parallel(client)
def condor_retire(self):
"""Retire the instances from condor. This means that jobs keep running on them, but they will
not accept new jobs."""
status = self.condor_status()
commands = {}
for host in self.hosts:
for s in status:
if s['host']==host.name:
commands[host.name] = s['fullhost']
commandlist = [f"condor_off -startd -peaceful {c}" for c in commands.values()]
logger.debug(commandlist)
server = ['server']*len(commandlist)
client = ParallelSSHClient(server, pkey=self.keysfile)
output = client.run_command('%s', host_args=commandlist, sudo=True)
def make_client(auth, hosts):
if auth['method'] == 'password':
return ParallelSSHClient(hosts.values(),
user=auth['user'],
password=auth['details']['password'])
raise RuntimeError('unsupported auth method: %s' % auth['method'])
def useSCP(args):
enable_host_logger()
hosts = utils.collect_hostnames(args.servers)
client = ParallelSSHClient(hosts,
user='******',
pkey=args.key,
allow_agent=False)
upload_file(client, args.file, args.destination)
def get_client():
return ParallelSSHClient(
list(host_selected.keys()),
host_config=host_selected,
num_retries=num_retries,
retry_delay=retry_delay,
pool_size=10 if len(host_selected) < 10 else len(host_selected),
)
def login_with_pool_key(self, ipaddress, port, user_passwd_pair_list, key_file_path_list, pool_size=10):
for key_file_path in key_file_path_list:
for user_passwd_pair in user_passwd_pair_list:
# for user in users:
try:
client = ParallelSSHClient(hosts=[ipaddress], port=port, user=user_passwd_pair[0],
pkey=key_file_path,
num_retries=0,
timeout=self.timeout,
pool_size=pool_size)
output = client.run_command('whoami', timeout=self.timeout)
log_success("SSH", ipaddress, port, [user_passwd_pair[0], "key: {}".format(key_file_path)])
except Exception as E:
logger.debug('AuthenticationException: ssh')
continue
finally:
pass
def generate_client(self, hosts):
host_config = {}
for host in hosts:
host_config[host] = {}
host_config[host]['user'] = sv.HOSTS_CONFIG[host]['user']
host_config[host]['password'] = sv.HOSTS_CONFIG[host]['password']
if sv.HOSTS_CONFIG[host].get('port'):
host_config[host]['port'] = sv.HOSTS_CONFIG[host]['port']
return ParallelSSHClient(hosts, host_config=host_config)
def send(cmd, user=USER, pause=0):
droplets = get_servers()
hosts = [d.ip_address for d in droplets]
if pause == 0:
client = ParallelSSHClient(hosts, user=user)
output = client.run_command(cmd)
# for host_output in output:
# for line in host_output.stdout:
# print(line)
# exit_code = host_output.exit_code
else:
for host in hosts:
client = SSHClient(host, user=user)
output = client.run_command(cmd)
for line in output.stdout:
print(line)
exit_code = output.exit_code
time.sleep(pause)
def deploy_remote(host, user, password, port, path_to_private_key):
path_to_monyze_agent = sys.argv[0]
if DEBUG:
path_to_monyze_agent = 'dist/monyze-agent'
filename_bin = str(basename(path_to_monyze_agent))
filename_sh = '/etc/init.d/monyze-agent'
dist_filename_bin = '/usr/local/bin/' + filename_bin
hosts = list()
hosts.append(host)
try:
# client = ParallelSSHClient(hosts, user=user, password=password, port=int(port), pkey=path_to_private_key)
client = ParallelSSHClient(hosts,
user=user,
port=int(port),
pkey=path_to_private_key)
print('Copying monyze-agent to remote hosts...')
sys.stdout.flush()
# deploy ELF monyze-agent
os.system("scp -i " + path_to_private_key + " -P " + port + " " +
path_to_monyze_agent + " " + user + "@" + host + ":" +
str(basename(path_to_monyze_agent)))
remote_sudo_cmd_run(
client, password,
'mv' + ' ' + filename_bin + ' ' + dist_filename_bin)
remote_sudo_cmd_run(client, password,
'chmod 755' + ' ' + dist_filename_bin)
# deploy shell monyze-agent
greenlets = client.copy_file(filename_sh, str(basename(filename_sh)))
joinall(greenlets, raise_error=True)
remote_sudo_cmd_run(
client, password,
'mv ' + str(basename(filename_sh)) + ' ' + filename_sh)
remote_sudo_cmd_run(client, password, 'chmod 755' + ' ' + filename_sh)
remote_sudo_cmd_run(client, password,
'update-rc.d monyze-agent defaults')
remote_sudo_cmd_run(client, password, 'service monyze-agent restart')
except Exception as error:
print('Can not deploy on host: %s' % host)
print('Caught this error: ' + repr(error))
def get_pssh_client(host,
private_key_path,
single_copy=False,
port=8022,
user='******'):
if single_copy and isinstance(host, str):
logger.debug('Trying single copy')
return SSHClient(host, port=port, pkey=private_key_path, user=user)
return ParallelSSHClient(host, port=port, pkey=private_key_path, user=user)
def send(cmd, pause=0, user=USER):
""" Send a command to all bots """
hosts = get_ips()
if pause == 0:
client = ParallelSSHClient(hosts, user=user)
output = client.run_command(cmd)
# for host_output in output:
# for line in host_output.stdout:
# print(line)
# exit_code = host_output.exit_code
else:
for host in hosts:
client = SSHClient(host, user=user)
output = client.run_command(cmd)
# for line in output.stdout:
# print(line)
# exit_code = output.exit_code
time.sleep(pause)
