def run_command(command, hosts, user, verbose=False, proxy_host=None,
timeout=10, **kwargs):
"""Run ssh command using Parallel SSH."""
result = {"0": [], "1": []}
if proxy_host:
client = ParallelSSHClient(hosts, user='******',
proxy_host=proxy_host,
proxy_user=user,
timeout=timeout)
else:
client = ParallelSSHClient(hosts, user=user, timeout=timeout)
output = client.run_command(command, stop_on_errors=False,
**kwargs)
client.join(output)
for host in hosts:
if host not in output:
# Pssh AuthenticationException duplicate output dict key
# {'saclay.iot-lab.info': {'exception': ...},
# {'saclay.iot-lab.info_qzhtyxlt': {'exception': ...}}
site = next(iter(sorted(output)))
raise OpenA8SshAuthenticationException(site)
result['0' if output[host]['exit_code'] == 0
else '1'].append(host)
if verbose:
for host in hosts:
# Pssh >= 1.0.0: stdout is None instead of generator object
# when you have ConnectionErrorException
stdout = output[host].get('stdout')
if stdout:
for _ in stdout:
pass
return result
def connect_using_pssh():
"""
This method uses parallel-ssh package to connect to the devices
"""
device_list = [item['address'] for item in json_out['device_list']]
user = set([item['username'] for item in json_out['device_list']]).pop()
password = set([item['password'] for item in json_out['device_list']]).pop()
client = ParallelSSHClient(device_list, user=user, password=password)
output = client.run_command('whoami')
for host, host_output in output.items():
for line in host_output:
print(line)
def _exec_ssh(self, cmd):
client = ParallelSSHClient(self.hosts)
output = client.run_command(cmd, sudo=self.sudo)
nice_output = dict()
for host in output:
nice_output[host]={
'stdout':[],
'stderr':[]
}
for line in output[host]['stdout']:
nice_output[host]['stdout'].append(line)
for line in output[host]['stderr']:
nice_output[host]['stderr'].append(line)
return nice_output
def run(context, command):
"""Run command across multiple servers."""
hosts = [
instance_name
for instance_name in context.obj.list_by_tag('Name')
]
client = ParallelSSHClient(hosts)
output = client.run_command(command)
# click.echo(json.dumps(output, indent=4))
results = [
'\n'.join([line for line in output[host]['stdout']])
for host in output
]
click.echo(results)
def remote_command():
"""
This function will print the output for remotely executed
command output for given hostid
"""
host_list = sys.argv[1].split(',')
#we will be adding it automatically to the list of known hosts
for each_host_id in host_list:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.load_host_keys(
os.path.expanduser(os.path.join("~", ".ssh", "known_hosts")))
#connecting and executing the command remotely
client = ParallelSSHClient(host_list, user="******")
cmd = input('Please enter the command to be executed on the remote host ')
output = client.run_command(cmd, stop_on_errors=False)
#printing the output of command
for host_name, host_output in output.items():
for line in host_output.stdout:
print "Host [%s] - %s" % (host_name, line)
host_config1[PI_addrs[i]] = {
'user': '******',
'password': '******',
'port': 22
}
PATH_TO_PROJECT = 'devel/ACDis/ACAgentgRPC/'
SCENENARIO = 'config_files/default/'
hosts1 = host_config1.keys()
hosts2 = host_config2.keys()
client1 = ParallelSSHClient(hosts1, host_config=host_config1)
client2 = ParallelSSHClient(hosts2, host_config=host_config2)
cmd = 'python ' + PATH_TO_PROJECT + 'source/admm_agent.py -f ' + PATH_TO_PROJECT + SCENENARIO + 'config_agent{0}.json'
host_args1 = [{'cmd': cmd.format(i)} for i in bus[0:9]]
host_args2 = [{'cmd': cmd.format(i)} for i in bus[10:13]]
output1 = client1.run_command('%(cmd)s', host_args=host_args1)
output2 = client2.run_command('%(cmd)s', host_args=host_args2)
# output = client1.run_command('whoam ')
for host, host_output in output1.items():
for line in host_output.stdout:
print("Host [%s] - %s" % (host, line))
for host, host_output in output2.items():
for line in host_output.stdout:
print("Host [%s] - %s" % (host, line))
import sys
import paramiko
import os
from pssh.pssh_client import ParallelSSHClient
if len(sys.argv) < 2:
print('No hostIDs mentioned')
else:
hostList = sys.argv[1].split(',')
#we will be adding it automatically to the list of known hosts
for hostId in hostList:
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.load_host_keys(os.path.expanduser(os.path.join("~", ".ssh", "known_hosts")))
#connecting and executing the command remotely
client = ParallelSSHClient(hostList, user="******")
cmd = input('Please Enter the Command to be executed on the remote host: ')
output = client.run_command(cmd, stop_on_errors=False)
for host in output:
for line in output[host]['stdout']:
print("Host %s - output: %s" % (host, line))
for line in output[host]['stderr']:
print("Host %s - output: %s" % (host,line))
from __future__ import print_function
from pprint import pprint
from pssh.pssh_client import ParallelSSHClient
from pssh.exceptions import AuthenticationException, \
UnknownHostException, ConnectionErrorException
import pssh.utils
pssh.utils.enable_host_logger()
hosts = ['vm-dc-js00001-dnguyen.svale.netledger.com']
client = ParallelSSHClient(hosts, proxy_host='nx')
try:
print("before run_command")
output = client.run_command('ls -ltrh /home/mkettlewell',
stop_on_errors=False)
print("after run_command")
client.join(output)
print(output)
for host in output:
for line in output[host]['stdout']:
print("Host %s - output: %s" % (host, line))
except (AuthenticationException, UnknownHostException,
ConnectionErrorException):
print("exception...")
pass
def start_client(ips):
client_host = ParallelSSHClient(ips, user='******', pkey=pkey)
client_outputs = client_host.run_command(
'bash $HOME/automate/tools/scripts/run_system.sh')
ssh_client_output(client_outputs)
from pssh.pssh_client import ParallelSSHClient
from pssh.utils import load_private_key
from util import ssh_client_output
import os
from constants import IP_PATH, NODE_CONFIG
HOME = os.environ['HOME']
# find all available hosts
hosts = []
with open(IP_PATH, "r") as f:
for line in f:
hosts.append(line.rstrip())
# ssh commands
pkey = load_private_key(HOME + '/.ssh/id_rsa_pis')
client = ParallelSSHClient(hosts, user='******', pkey=pkey)
output = client.run_command(
'bash $HOME/automate/tools/scripts/update_pi_environment.sh')
ssh_client_output(output)
from constants import IP_PATH, NODE_CONFIG
HOME = os.environ['HOME']
# find all available hosts
hosts = []
with open(IP_PATH, "r") as f:
for line in f:
hosts.append(line.rstrip())
# ssh commands
pkey = load_private_key(HOME + '/.ssh/id_rsa_pis')
client = ParallelSSHClient(hosts, user='******', pkey=pkey)
# terminate the running program by soft terminate signal
output = client.run_command('kill -15 "$(pgrep python)"')
ssh_client_output(output)
config = ConfigParser.ConfigParser()
config.read(NODE_CONFIG)
model, system = config.get('Node Config', 'model',
0), config.get('Node Config', 'system', 0)
dir_name = '/' + model + '-' + system + '-' + strftime("%Y-%m-%d %H:%M:%S",
localtime()) + '/'
subprocess.Popen(['mkdir', HOME + '/stats' + dir_name])
for n in range(1, len(list(config.items('Node IP'))) + 1):
node_id = 'n' + str(n)
ip = config.get('Node IP', node_id, 0)
subprocess.Popen([
from pssh.pssh_client import ParallelSSHClient
from pssh.utils import load_private_key
import os
from util import ssh_client_output
from constants import IP_PATH, NODE_CONFIG
HOME = os.environ['HOME']
# loop through all hosts
hosts = []
with open(IP_PATH, "r") as f:
for line in f:
hosts.append(line.rstrip())
# load ssh private key
pkey = load_private_key(HOME + '/.ssh/id_rsa_pis')
client = ParallelSSHClient(hosts, user='******', pkey=pkey)
# execute commands to Pis
output = client.run_command(
'chmod -R 777 $HOME/automate | rm -rf $HOME/automate')
ssh_client_output(output)
output = client.run_command(
'git clone https://github.com/parallel-ml/tools.git $HOME/automate/tools')
ssh_client_output(output)
output = client.run_command('bash $HOME/automate/tools/scripts/setup.sh')
ssh_client_output(output)
from __future__ import print_function
from pssh.pssh_client import ParallelSSHClient
client = ParallelSSHClient(['localhost'])
output = client.run_command('whoami')
for line in output['localhost'].stdout:
print(line)
import pprint
import pssh.utils
from initializationDataProcessing import inputValidation, configurationFileProcess
from pssh.pssh_client import ParallelSSHClient
from pssh.exceptions import AuthenticationException, UnknownHostException, ConnectionErrorException
if __name__ == '__main__':
# check user input
inputValidation.InputValidation().input_validation()
# instantiate configuration file process obj
configurationFileProcess_obj = configurationFileProcess.ConfigurationFileProcess()
#pprint.pprint(configurationFileProcess_obj.process_conf_file(sys.argv[1]))
client = ParallelSSHClient(['127.0.0.1'], user='******', password='******', port=19013)
try:
output = client.run_command('ls -ltrh /home/tinyos', sudo=True, stop_on_errors=False)
test = client.get_output()
pprint.pprint(output)
for host in output:
for line in output[host]['stdout']:
print line
pprint.pprint(test)
del client
exit(0)
except(AuthenticationException, UnknownHostException, ConnectionErrorException) as e:
print e
exit(1)
Optimization Area:
1. Handle multiple Server with different login credentials.
2. Handle output if host is not reachable.
3. Connection handling if network fluctuate.
'''
import sys
import pssh.utils
from pssh.pssh_client import ParallelSSHClient
from pssh.exceptions import AuthenticationException, \
UnknownHostException, ConnectionErrorException
cmd = None
USERNAME = "******"
PASSWORD = "******"
pssh.utils.enable_host_logger()
client = ParallelSSHClient(sys.argv[1:], user=USERNAME, password=PASSWORD)
while cmd != "exit":
cmd = raw_input("$>")
try:
output = client.run_command(cmd, sudo=True)
client.join(output, consume_output=True)
except (AuthenticationException, UnknownHostException,
ConnectionErrorException):
pass
del client
from __future__ import print_function
from pssh.pssh_client import ParallelSSHClient
host_config = {'192.168.1.101' : {'user': '******', 'password': '******',
'port': 22},
'192.168.1.102' : {'user': '******', 'password': '******',
'port': 22},
}
hosts = host_config.keys()
client = ParallelSSHClient(hosts, host_config=host_config)
output = client.run_command('ls')
for host, host_output in output.items():
for line in host_output.stdout:
print("Host [%s] - %s" % (host, line))
'ec2-54-157-13-35.compute-1.amazonaws.com',
'ec2-54-209-21-173.compute-1.amazonaws.com',
'ec2-54-209-144-193.compute-1.amazonaws.com',
'ec2-54-227-191-220.compute-1.amazonaws.com',
'ec2-54-237-238-149.compute-1.amazonaws.com',
]
key_file_path = '/Users/nbarendt/.ssh/eecs397-spring17.pem'
priv_key = paramiko.RSAKey.from_private_key_file(key_file_path)
master_client = ParallelSSHClient([master_public], pkey=priv_key)
slave_client = ParallelSSHClient(slaves, pkey=priv_key)
# kill any locust processes
print("killing any 'locust' processes in cluster...")
master_output = master_client.run_command('killall -q locust', sudo=True)
slave_output = slave_client.run_command('killall -q locust', sudo=True)
master_client.join(master_output)
slave_client.join(slave_output)
print(" done")
# SFTP latest locust file to all locust instances
print("copying latest mqtt-locustfile.py to all cluster instances")
master_greenlets = master_client.copy_file('mqtt-locustfile.py',
'mqtt-locust/eecs397-locustfile.py')
slave_greenlets = slave_client.copy_file('mqtt-locustfile.py',
'mqtt-locust/eecs397-locustfile.py')
joinall(master_greenlets, raise_error=True)
def init_check():
client = ParallelSSHClient(by_tens)
output = client.run_command("df -h /")
for host, host_output in output.items():
for line in host_output.stdout:
print(line)
class BaseTransporter(object):
'''The more important class, this is like the core of the fabric clone:
it allows to open a tunnel from the (usually) local computer to a certain
numbers of remote endpoints.
This class should implement the functions needed to act upon the remote system,
as filesystem-like commands (cd, mkdir, rm)
>>> transporter = Transporter(hosts=hosts)
>>> with transporter.cd('/var/www/'):
>>> transporter.mkdir(web_root_dir)
'''
def __init__(self, hosts=None):
if not hosts:
raise Exception('hosts parameter must be defined')
self.hosts = hosts
self.client = ParallelSSHClient(hosts)
self._initial_connection()
def _parse_output(self, output):
'''This parse for our usage the output of the execution'''
def _log_channel_output(self, output, prefix=None):
for host in output:
for stdout in output[host]['stdout']:
logger.info('%s%s' % (prefix, stdout))
for stderr in output[host]['stderr']:
logger.warn('%s%s' % (prefix, stderr))
def cd(self, path):
# handle absolute or relative paths
# make clear where the action is done
# TODO: context manager
pass
def mkdir(self, path):
pass
def rm(self, path):
pass
def cp(self, src, dst, **kwargs):
# src can be local
# dst must be remote
logger.debug('cp %s -> %s' % (src, dst))
events = self.client.copy_file(src, dst)
for event in events:
event.get()
def _initial_connection(self):
'''The ratio of this method is that we need to connect immediately to the remote endpoints
in order to be sure that hosts are accessible.
AuthenticationException, UnknownHostException and ConnectionErrorException are possible only
at the first attempt (TODO: check, I'm not sure for ConnectionErrorException, if the server
lost connection?).
'''
try:
self._log_channel_output(self.client.run_command('uname -a'), prefix=' --- start connection: ')
except (AuthenticationException, UnknownHostException, ConnectionErrorException) as e:
logger.exception(e)
raise e
def _exec_command(self, cmd):
return self.client.run_command(cmd)
def run(self, cmd):
return self._exec_command(cmd)
'192.168.1.108': {
'user': '******',
'password': '******',
'port': 22
},
'192.168.1.109': {
'user': '******',
'password': '******',
'port': 22
}
# '192.168.1.110' : {'user': '******', 'password': '******',
# 'port': 22}
}
hosts = host_config.keys()
#
PATH_TO_PROJECT = 'devel/ACDis/ACAgentgRPC/'
CONFIG = 'default'
#
client = ParallelSSHClient(hosts, host_config=host_config)
output = client.run_command('mkdir ' + PATH_TO_PROJECT + 'config_files')
# output = client.run_command('mkdir ' + PATH_TO_PROJECT + 'config_files/' + CONFIG)
# for host, host_output in output.items():
# for line in host_output.stdout:
# print("Host [%s] - %s" % (host, line))
for i in hosts:
cmd = r'pscp -r -pw raspberry E:\lam\Distributed_optimization\projects\admm_cigre\config_files\ ' + CONFIG + ' ' + \
'pi@{0}:/home/pi/devel/ACDis/ACAgentgRPC/config_files/'
subprocess.call(cmd.format(i))
def start_server(ips):
server_hosts = ParallelSSHClient(ips, user='******', pkey=pkey)
servers_outputs = server_hosts.run_command(
'bash $HOME/automate/tools/scripts/run_system.sh server')
ssh_client_output(servers_outputs)
from pssh.pssh_client import ParallelSSHClient
from pssh.utils import load_private_key
from util import ssh_client_output
import os
from constants import IP_PATH, NODE_CONFIG
HOME = os.environ['HOME']
# find all available hosts
hosts = []
with open(IP_PATH, "r") as f:
for line in f:
hosts.append(line.rstrip())
# ssh commands
pkey = load_private_key(HOME + '/.ssh/id_rsa_pis')
client = ParallelSSHClient(hosts, user='******', pkey=pkey)
output = client.run_command('bash $HOME/automate/tools/scripts/update.sh')
ssh_client_output(output)