def configure(update, context):
new_server = context.user_data['new_server']
ip = new_server['ip']
rootpass = new_server['pass']
#check if there's already API running on the server
try:
r = requests.get('http://{}'.format(ip)).json()
if "Hi" in r['message']:
update.message.reply_text(
"Seems like setup is already done on this server. Now you should pick a server.",
reply_markup=choose_server_markup)
context.user_data['servers'].append(new_server)
return CHOOSE_SERVER
except RequestException:
pass
#check if auth is correct
try:
client = SSHClient(ip, user='******', password=rootpass)
client.run_command('whoami', sudo=True)
update.message.reply_text("Auth credentials ok.")
except AuthenticationException:
update.message.reply_text(
"Auth credentials fail. Start-over with /start")
return CONFIGURE
update.message.reply_text(
"Starting fresh server setup, it will take a few minutes...")
command = "wget https://raw.githubusercontent.com/dathbezumniy/kmd-sync-api/master/sync_api_setup.sh " \
"&& chmod u+x sync_api_setup.sh && ./sync_api_setup.sh"
output = client.run_command(command, sudo=True)
#wait until all dependencies downloaded/installed then check if API is up
time.sleep(200)
try:
r = requests.get('http://{}'.format(ip)).json()
if "Hi" in r['message']:
update.message.reply_text(
"Seems like setup is done and API is up. Now you should pick a server.",
reply_markup=choose_server_markup)
context.user_data['servers'].append(new_server)
return CHOOSE_SERVER
except RequestException:
update.message.reply_text(
"Something went wrong. API didn't start, you can try to start over the configuration with /start"
)
return CONFIGURE
update.message.reply_text(
"Something went wrong. API didn't start, you can try to start over the configuration with /start"
)
return CONFIGURE
class SSHConnection:
def __init__(self, host, port, username, password=None, pkey=None):
from pssh.clients import SSHClient
self.client = SSHClient(host=host, user=username, password=pass,
port=port, pkey=pkey)
self.threads = []
def exec(self, *args, **kwargs):
print("Executing command: ", args)
channel, host, stdout, stderr stdin = self.client.run_command(*args, **kwargs)
from threading import Thread
import sys
stdout_thread = Thread(target=self._copy_output, args=(stdout, sys.stdout))
stderr_thread = Thread(target=self._copy_output, args=(stderr, sys.stderr))
stdout_thread.start()
stderr_thread.start()
self.threads.append(stdout_thread)
self.threads.append(stderr_thread)
def make_executable(self, path):
ch, host, stdout, stderr, stdin = self.client.run_command(f"chmod +x {path}")
self.client.close(ch)
def exit(self):
for t in self.threads:
t.join()
self.client.disconnect()
def _copy_output(self, output, to):
for line in output:
print(line, file=to, flush=True, end="")
def put(self, *args):
self.client.scp_send(*args)
def get(self, *args):
self.client.scp_recv(*args)
def _execute(self, command, *args, **kwargs):
if not command.startswith("_"):
targets = (self, self.client)
for t in targets:
if hasattr(t, command):
method = getattr(t, command)
if callable(method):
return method(*args, **kwargs)
raise InvalidCommand(f"Invalid command - {command}")
def _create_cluster_ssh_conns(self):
hostips = [n["publicIpAddress"] for n in self.cluster_nodes]
hostconfigs = [HostConfig(port=n["port"]) for n in self.cluster_nodes]
self._all_ssh = ParallelSSHClient(hostips,
host_config=hostconfigs,
user=self.admin_username)
self._master_ssh = ParallelSSHClient(hostips[:1],
host_config=hostconfigs[:1],
user=self.admin_username)
self._master_scp = SSHClient(hostips[0],
port=hostconfigs[0].port,
user=self.admin_username)
def scp(self, src, dst):
"""Copy file using Parallel SCP native client."""
result = {"0": [], "1": []}
sites = self.groups.keys()
for site in sites:
try:
client = SSHClient(site,
user=self.config_ssh['user'],
pkey=SSH_KEY,
timeout=10)
client.copy_file(src, dst)
result["0"].append(site)
except (SFTPError, UnknownHostError, AuthenticationError,
pssh.exceptions.ConnectionError):
result["1"].append(site)
return _cleanup_result(result)
def get_pssh_client(host,
private_key_path,
single_copy=False,
port=8022,
user='******'):
if single_copy and isinstance(host, str):
logger.debug('Trying single copy')
return SSHClient(host, port=port, pkey=private_key_path, user=user)
return ParallelSSHClient(host, port=port, pkey=private_key_path, user=user)
def __init__(self, ip, user, password):
self.HOST = ip
self.USER = user
self.PASSWD = password
self.last_idle = 0.0
self.last_total = 0.0
print(utils.get_time() + "\tConnecting to " + ip)
try:
self.client = SSHClient(self.HOST,
self.USER,
self.PASSWD,
timeout=1,
num_retries=2)
print(utils.get_time() + "\tSession started")
self.conn = True
except Exception:
print(utils.get_time() + "\tSession not stablisehd. Closing")
exit(-1)
def __init__(self, host, username, password, port=22):
""" Create the SSH instance and returns the object
:param host: Device IP or Hostname
:param username: username
:param password: password
:param port: ssh port
:return: instance of ssh connection
"""
self.host = host
self.username = username
self.password = password
self.port = port
self.device = SSHClient(self.host, user=self.username, password=self.password, port=self.port)
def client(self):
if self._client is None:
pkey = self.sshkey.path if (self.sshkey
and self.sshkey.path) else None
passwd = self.passwd
if pkey:
passwd = self.sshkey.passphrase
self._client = PSSHClient(self.addr_variable,
user=self.login,
password=passwd,
port=self.port,
pkey=pkey,
num_retries=self.timeout / 6,
allow_agent=self.allow_agent,
timeout=5,
retry_delay=1)
return self._client
def send(cmd, pause=0, user=USER):
""" Send a command to all bots """
hosts = get_ips()
if pause == 0:
client = ParallelSSHClient(hosts, user=user)
output = client.run_command(cmd)
# for host_output in output:
# for line in host_output.stdout:
# print(line)
# exit_code = host_output.exit_code
else:
for host in hosts:
client = SSHClient(host, user=user)
output = client.run_command(cmd)
# for line in output.stdout:
# print(line)
# exit_code = output.exit_code
time.sleep(pause)
def send(cmd, user=USER, pause=0):
droplets = get_servers()
hosts = [d.ip_address for d in droplets]
if pause == 0:
client = ParallelSSHClient(hosts, user=user)
output = client.run_command(cmd)
# for host_output in output:
# for line in host_output.stdout:
# print(line)
# exit_code = host_output.exit_code
else:
for host in hosts:
client = SSHClient(host, user=user)
output = client.run_command(cmd)
for line in output.stdout:
print(line)
exit_code = output.exit_code
time.sleep(pause)
def _client(self):
if self._client_ is None:
passwd = self.passwd
if self.sshkey_name:
pkey = self.sshkey_obj.path if (
self.sshkey_obj and self.sshkey_obj.path) else None
if pkey:
passwd = self.sshkey_obj.passphrase_
if self.allow_agent:
passwd = None
pkey = None
passwd = None
from pssh.clients import SSHClient
self._log_debug("ssh connection: %s@%s:%s (passwd:%s,key:%s)" %
(self.login, self.addr, self.port, passwd, pkey))
try:
self._client_ = SSHClient(
host=self.addr,
user=self.login,
password=passwd,
port=self.port,
# proxy_pkey=pkey,
num_retries=10,
allow_agent=self.allow_agent,
timeout=self.timeout,
retry_delay=1,
)
except Exception as e:
if str(e).find("Error connecting to host") != -1:
msg = e.args[0] % e.args[1:]
raise j.exceptions.Base("PSSH:%s" % msg)
raise
return self._client_
def __init__(self, host, port, username, password=None, pkey=None):
from pssh.clients import SSHClient
self.client = SSHClient(host=host, user=username, password=pass,
port=port, pkey=pkey)
self.threads = []
class SSHClient(SSHClientBase):
def __init__(self, instance, data={}, parent=None, interactive=False):
SSHClientBase.__init__(self,
instance=instance,
data=data,
parent=parent,
interactive=interactive)
self._logger = j.logger.get(
"ssh client: %s:%s(%s)" %
(self.addr_variable, self.port, self.login))
self._client = None
self._prefab = None
@property
def client(self):
if self._client is None:
pkey = self.sshkey.path if (self.sshkey
and self.sshkey.path) else None
passwd = self.passwd
if pkey:
passwd = self.sshkey.passphrase
self._client = PSSHClient(self.addr_variable,
user=self.login,
password=passwd,
port=self.port,
pkey=pkey,
num_retries=self.timeout / 6,
allow_agent=self.allow_agent,
timeout=5,
retry_delay=1)
return self._client
def execute(self, cmd, showout=True, die=True, timeout=None):
channel, _, stdout, stderr, _ = self.client.run_command(
cmd, timeout=timeout)
def _consume_stream(stream, printer, buf=None):
buffer = buf or io.StringIO()
for line in stream:
buffer.write(line + '\n')
if showout:
printer(line)
return buffer
out = _consume_stream(stdout, self.logger.debug)
err = _consume_stream(stderr, self.logger.error)
self._client.wait_finished(channel)
_consume_stream(stdout, self.logger.debug, out)
_consume_stream(stderr, self.logger.error, err)
rc = channel.get_exit_status()
output = out.getvalue()
out.close()
error = err.getvalue()
err.close()
channel.close()
if rc and die:
raise j.exceptions.RuntimeError(
"Cannot execute (ssh):\n%s\noutput:\n%serrors:\n%s" %
(cmd, output, error))
return rc, output, error
def connect(self):
self.client
def reset(self):
with self._lock:
if self._client is not None:
self._client = None
@property
def sftp(self):
return self.client._make_sftp()
def close(self):
# TODO: make sure we don't need to clean anything
pass
def copy_file(self, local_file, remote_file, recurse=False, sftp=None):
return self.client.copy_file(local_file,
remote_file,
recurse=recurse,
sftp=sftp)
def rsync_up(self, source, dest, recursive=True):
if dest[0] != "/":
raise j.exceptions.RuntimeError("dest path should be absolute")
dest = "%s@%s:%s" % (self.config.data['login'], self.addr_variable,
dest)
j.sal.fs.copyDirTree(
source,
dest,
keepsymlinks=True,
deletefirst=False,
overwriteFiles=True,
ignoredir=[".egg-info", ".dist-info", "__pycache__"],
ignorefiles=[".egg-info"],
rsync=True,
ssh=True,
sshport=self.port_variable,
recursive=recursive)
def rsync_down(self, source, dest, source_prefix="", recursive=True):
if source[0] != "/":
raise j.exceptions.RuntimeError("source path should be absolute")
source = "%s@%s:%s" % (self.config.data['login'], self.addr_variable,
source)
j.sal.fs.copyDirTree(source,
dest,
keepsymlinks=True,
deletefirst=False,
overwriteFiles=True,
ignoredir=[".egg-info", ".dist-info"],
ignorefiles=[".egg-info"],
rsync=True,
ssh=True,
sshport=self.port_variable,
recursive=recursive)
@property
def prefab(self):
if self._prefab:
return self._prefab
ex = j.tools.executor
executor = ex.getSSHViaProxy(
self.addr_variable) if self.config.data['proxy'] else ex.ssh_get(
self)
if self.config.data["login"] != "root":
executor.state_disabled = True
self._prefab = executor.prefab
return self._prefab
# 1 - Preparing nodes
command = "rm -rf * && wget https://raw.githubusercontent.com/tonymorony/komodo-cctools-python/master/scripts/dexp2p/multi-server/prepare_dexp2p_node_ms.sh " \
"&& chmod u+x prepare_dexp2p_node_ms.sh && ./prepare_dexp2p_node_ms.sh"
client = ParallelSSHClient(hosts, user="******")
output = client.run_command(command, sudo=True)
for node in output:
for line in output[node]['stdout']:
print(line)
# 2 - Preparing "started nodes" file on each server
i = 0
for host in hosts:
print("Preparing file on node " + str(i + 1))
non_parallel_client = SSHClient(host, user="******")
if i == 0:
non_parallel_client.run_command("touch ip_list")
else:
line_with_hosts = ""
for host in hosts[:i]:
line_with_hosts += host + "\n"
non_parallel_client.run_command("echo -e " + line_with_hosts +
" >> ip_list")
i = i + 1
print("Test nodes software prepared. Starting network.")
# 3 - Starting network (need to do one by one)
i = 0
for host in hosts:
print("Starting network on node " + str(i + 1))
from pssh.clients import SSHClient
client = SSHClient(host="192.168.56.105", user="******", password="******")
ch, host, stdout, stderr, stdin = client.run_command("uptime")
for line in stdout:
print(line)
class SSH:
"""
This class contains all methods to retieve information from the remote system
"""
conn = False
client = None
def __init__(self, ip, user, password):
self.HOST = ip
self.USER = user
self.PASSWD = password
self.last_idle = 0.0
self.last_total = 0.0
print(utils.get_time() + "\tConnecting to " + ip)
try:
self.client = SSHClient(self.HOST,
self.USER,
self.PASSWD,
timeout=1,
num_retries=2)
print(utils.get_time() + "\tSession started")
self.conn = True
except Exception:
print(utils.get_time() + "\tSession not stablisehd. Closing")
exit(-1)
def finalize_conn(self):
"""
Finalizes the SSH connection
"""
self.client.session.disconnect()
print(utils.get_time() + "\tClosing session")
def get_GPU_temp(self):
"""
Obtains the GPU temperature via the vcgencmd measure_temp command
"""
host_out = self.client.run_command("vcgencmd measure_temp")
for line in host_out.stdout:
return line[5:9]
def get_CPU_temp(self):
"""
Obtains the CPU temperature
"""
host_out = self.client.run_command(
"cat /sys/class/thermal/thermal_zone0/temp")
for line in host_out.stdout:
return (int(line) / 1000).__round__(1)
def get_CPU_percentage(self, line):
"""
Calculates the usage of a CPU based on the /proc/stat content
:param line: line from /proc/stat for the CORE
:return: Usage percentage for the CORE
"""
fields = [float(column) for column in line.strip().split()[1:]]
idle, total = fields[3], sum(fields)
idle_delta, total_delta = idle - self.last_idle, total - self.last_total
self.last_idle = idle
self.last_total = total
usage = 100.0 * (1.0 - (idle_delta / total_delta))
return usage.__round__(2)
def get_CPU_use(self):
"""
Obtains the CPU usage
"""
output = []
host_out = self.client.run_command("cat /proc/stat")
contline = 0
for line in host_out.stdout:
if contline == 1:
break
output.append(self.get_CPU_percentage(line))
contline += 1
return output[0]
def get_status(self):
"""
Returns the status of the system, indicating if there are under voltage warnings or any other events
The string is already reversed, so byte 0 in documentation is position 0 of the string (easier parsing)
"""
host_out = self.client.run_command("vcgencmd get_throttled")
for line in host_out.stdout:
"""
Bit Hex value Meaning
0 1 Under-voltage detected
1 2 Arm frequency capped
2 4 Currently throttled
3 8 Soft temperature limit active
16 10000 Under-voltage has occurred
17 20000 Arm frequency capping has occurred
18 40000 Throttling has occurred
19 80000 Soft temperature limit has occurred
"""
# Convert the hexadecimal value into binary (filling with 0 up to 20 positions if needed) and reverse it
return bin(int(line[10:], 16))[2:].zfill(20)[::-1]
def get_codecs(self):
"""
Returns a list of the codecs and its status (enable for HW of disabled for SW processing)
"""
output = []
for codec in utils.CODECS:
host_out = self.client.run_command("vcgencmd codec_enabled " +
codec)
for line in host_out.stdout:
output.append(line)
return output
def get_kernel_info(self, arch=False):
"""
Returns kernel name, release and processor architecture (optional)
:param arch: flag to get the architecture
"""
if arch:
command = "uname -smr"
else:
command = "uname -sr"
host_out = self.client.run_command(command)
for line in host_out.stdout:
return line
def get_hostname(self):
"""
Returns hostname
"""
host_out = self.client.run_command("hostname")
for line in host_out.stdout:
return line
def get_RAM_usage(self):
"""
Returns % of RAM usage
"""
host_out = self.client.run_command("free -m")
aux2 = []
for line in host_out.stdout:
if str.startswith(line, "Mem"):
aux2 = line.split(" ")
break
aux = []
for elem in aux2:
if elem != "":
aux.append(elem)
# Used RAM is the sum of Used and Shared columns
value = (int(aux[2]) + int(aux[4])) / int(aux[1])
return str((value * 100).__round__(2)) + " %"
def get_CPU_freq(self):
"""
Returns current CPU frequency
"""
host_out = self.client.run_command("vcgencmd measure_clock arm")
for line in host_out.stdout:
return str(int(int(line[14:]) / 1000000))
def get_disk_usage(self):
"""
Returns mounted disks usage (excluding tmpfs)
"""
if utils.HUMAN_DISK_INFO:
host_out = self.client.run_command("df -mh")
else:
host_out = self.client.run_command("df -m")
aux2 = []
cont = 0
for line in host_out.stdout:
if cont == 0:
cont += 1
continue
if str.startswith(line, "tmpfs") or str.startswith(
line, "devtmpfs"):
continue
else:
aux2.append(line)
aux = []
for elem in aux2:
aux3 = elem.split(" ")
for item in aux3:
if item != "":
aux.append(item)
output = "Mount\tTotal\tUsed\tFree\t% Used\n"
for cont in range(0, len(aux) - 1, 6):
output += str(aux[cont + 5]) + "\t" + str(aux[cont + 1]) + "\t" + str(aux[cont + 2]) + "\t" + \
str(aux[cont + 3]) + "\t" + str(aux[cont + 4]) + "\n"
return output
def get_avg_load(self):
"""
Returns avg CPU load for the last 1, 5 and 15 minutes
"""
host_out = self.client.run_command("cat /proc/loadavg")
aux2 = []
for line in host_out.stdout:
aux2 = line.split(" ")
aux = []
for elem in aux2:
if elem != "":
aux.append(elem)
return aux
def get_uptime(self):
"""
Returns the uptime in human readable format
1 day, 5 hours, 5 minutes
"""
host_out = self.client.run_command("uptime -p")
for line in host_out.stdout:
return line[3:]
def get_governors(self):
"""
Returns a list of the available governors as well as the current one
"""
host_out = self.client.run_command(
"cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_available_governors"
)
out = []
for line in host_out.stdout:
out = line.split(" ")
host_out = self.client.run_command(
"cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor")
governor = ""
for line in host_out.stdout:
governor = line
return out, governor
def get_GPU_memory(self):
"""
Returns memory reserved for the GPU
"""
host_out = self.client.run_command("vcgencmd get_mem gpu")
for line in host_out.stdout:
return line.split("=")[1]
from pssh.utils import load_private_key
from pssh.clients import SSHClient
import yaml
with open("host_config.yml") as yaml_file:
host_config = yaml.load(yaml_file, Loader=yaml.CLoader)
clients = []
for host, info in host_config.items():
client = SSHClient(host=host,
port=info["port"],
user=info["user"],
password=info["password"])
for command in info["commands"]:
output = client.run_command(command)
clients.append(output)
print(clients)
for channel, host, stdout, stderr, stdin in clients:
for line in stdout:
print(host, line)
class ClusterConnector:
def __init__(
self,
workspace,
cluster_name,
ssh_key,
vm_type,
admin_username="******",
):
"""Thin wrapper class around azureml.core.compute.AmlCluster
Provides parallel ssh objects and helper for master node and all node commands
and file copies.
Usage:
>>> cc = ClusterConnector(workspace, "MyCluster", sshkey, "Standard_ND40rs_v2")
>>> cc.initialize(min_nodes=0, max_nodes=4, idle_timeout_secs=30)
>>> cluster = cc.cluster
>>> [print(node['name']) for node in cc.cluster.list_nodes()]
"""
self.cluster_name = cluster_name
self.workspace = workspace
self.ssh_key = ssh_key
self.vm_type = vm_type
self.admin_username = admin_username
enable_host_logger()
hlog = logging.getLogger("pssh.host_logger")
tstr = datetime.now().isoformat(timespec="minutes")
[
hlog.removeHandler(h) for h in hlog.handlers
if isinstance(h, logging.StreamHandler)
]
os.makedirs("clusterlogs", exist_ok=True)
self.logfile = "clusterlogs/{}_{}.log".format(self.workspace.name,
tstr)
hlog.addHandler(logging.FileHandler(self.logfile))
self.cluster = None
self._master_scp = None
self._master_ssh = None
self._all_ssh = None
def initialise(self, min_nodes=0, max_nodes=0, idle_timeout_secs=1800):
"""Initialise underlying AmlCompute cluster instance"""
self._create_or_update_cluster(min_nodes, max_nodes, idle_timeout_secs)
def _check_logs_emessage(self, host, port):
msg = "Remote command failed on {}:{}. For details see {}".format(
host, port, self.logfile)
return msg
def terminate(self):
print('Attempting to terminate cluster "{}"'.format(
colored(self.cluster_name, "green")))
try:
self.cluster.update(min_nodes=0,
max_nodes=0,
idle_seconds_before_scaledown=10)
self.cluster.wait_for_completion()
except ComputeTargetException as err:
raise RuntimeError(
"Failed to terminate cluster nodes ({})".format(err))
if len(self.cluster.list_nodes()):
raise RuntimeError(
"Failed to terminate cluster nodes (nodes still running)")
@property
def cluster_nodes(self):
self.cluster.refresh_state()
return sorted(self.cluster.list_nodes(), key=lambda n: n["port"])
def _create_or_update_cluster(self, min_nodes, max_nodes,
idle_timeout_secs):
try:
self.cluster = AmlCompute(workspace=self.workspace,
name=self.cluster_name)
print('Updating existing cluster "{}"'.format(
colored(self.cluster_name, "green")))
self.cluster.update(
min_nodes=min_nodes,
max_nodes=max_nodes,
idle_seconds_before_scaledown=idle_timeout_secs,
)
except ComputeTargetException:
print('Creating new cluster "{}"'.format(
colored(self.cluster_name, "green")))
cluster_config = AmlCompute.provisioning_configuration(
vm_size=self.vm_type,
min_nodes=min_nodes,
max_nodes=max_nodes,
idle_seconds_before_scaledown=idle_timeout_secs,
admin_username=self.admin_username,
admin_user_ssh_key=self.ssh_key,
remote_login_port_public_access="Enabled",
)
self.cluster = AmlCompute.create(self.workspace, self.cluster_name,
cluster_config)
self.cluster.wait_for_completion()
if len(self.cluster_nodes) < min_nodes:
sleep(30)
if len(self.cluster_nodes) < min_nodes:
raise RuntimeError("Failed to provision sufficient nodes")
def _copy_nodefile_to_nodes(self):
if len(self.cluster_nodes) == 1:
cprint("Single node cluster -- skipping IB config", "yellow")
return
print("Collecting cluster IB info")
outputs = self._all_ssh.run_command(
r'ifconfig ib0 | grep -oe "inet[^6][adr: ]*[0-9.]*" | cut -d" " -f2',
shell="bash -c",
)
self._all_ssh.join(outputs)
ibaddrs = []
for output in outputs:
host = output.host
port = output.client.port
if output.exit_code != 0:
print(list(output.stdout))
print(list(output.stderr))
raise RuntimeError("Failed to get IB ip for {}:{}".format(
host, port))
try:
ibaddr = list(output.stdout)[0].split()[0]
except IndexError:
raise RuntimeError("Failed to get IB ip for {}:{} - "
"No ib interface found!".format(host, port))
print("Mapping {}:{} -> {}".format(host, port, ibaddr))
if port == self._master_scp.port:
cprint("IB Master: {}".format(ibaddr), "green")
ibaddrs = [ibaddr] + ibaddrs
else:
ibaddrs.append(ibaddr)
with NamedTemporaryFile(delete=False, mode="wt") as nfh:
self.nodefile = nfh.name
for addr in ibaddrs:
nfh.write("{}\n".format(addr))
self.ibaddrs = ibaddrs
self.copy_to_all_nodes(self.nodefile, "./nodefile")
def _create_cluster_ssh_conns(self):
hostips = [n["publicIpAddress"] for n in self.cluster_nodes]
hostconfigs = [HostConfig(port=n["port"]) for n in self.cluster_nodes]
self._all_ssh = ParallelSSHClient(hostips,
host_config=hostconfigs,
user=self.admin_username)
self._master_ssh = ParallelSSHClient(hostips[:1],
host_config=hostconfigs[:1],
user=self.admin_username)
self._master_scp = SSHClient(hostips[0],
port=hostconfigs[0].port,
user=self.admin_username)
def copy_to_all_nodes(self, source, dest):
copy_jobs = self._all_ssh.copy_file(source, dest)
joinall(copy_jobs, raise_error=True)
def copy_to_master_node(self, source, dest):
self._master_scp.copy_file(source, dest)
def copy_from_master_node(self, source, dest):
self._master_scp.copy_remote_file(source, dest)
def run_on_all_nodes(self, command):
outputs = self._all_ssh.run_command(command, shell="bash -c")
self._all_ssh.join(outputs, consume_output=True)
for output in outputs:
if int(output.exit_code) != 0:
host = output.host
port = output.client.port
raise RuntimeError(self._check_logs_emessage(host, port))
def run_on_master_node(self, command):
outputs = self._master_ssh.run_command(command, shell="bash -c")
self._master_ssh.join(outputs)
for output in outputs:
if int(output.exit_code) != 0:
host = output.host
port = output.client.port
raise RuntimeError(self._check_logs_emessage(host, port))
def attempt_termination(self):
try:
self.terminate()
except RuntimeError as err:
print(colored("ERROR: {}\n\n", "red", attrs=["bold"]).format(err))
self.warn_unterminated()
def warn_unterminated(self):
print(
colored("WARNING: {}", "red", attrs=["bold"]).format(
colored(
"Cluster {} is still running - terminate manually to avoid "
"additional compute costs".format(
colored(self.cluster_name, "green")),
"red",
)))
from pssh.clients import SSHClient
host = '192.168.56.101'
client = SSHClient(host, user="******", password="******")
channel, host, stdout, stderr, stdin = client.run_command('ls /usr/local/src')
for line in stdout:
print(line)
from pssh.clients import SSHClient
client = SSHClient("192.168.56.101", user="******", password="******")
client.copy_remote_file('/etc/passwd', 'p.txt')
def connect_to_server():
from pssh.clients import SSHClient
conn = SSHClient("192.168.56.101", user="******", password="******")
return conn
