def useSCP(args):
enable_host_logger()
hosts = utils.collect_hostnames(args.servers)
client = ParallelSSHClient(hosts,
user='******',
pkey=args.key,
allow_agent=False)
upload_file(client, args.file, args.destination)
def test_enabling_host_logger(self):
self.assertTrue(len([h for h in utils.host_logger.handlers
if isinstance(h, NullHandler)]) == 1)
utils.enable_host_logger()
# And again to test only one non-null handler is attached
utils.enable_host_logger()
self.assertTrue(len([h for h in utils.host_logger.handlers
if not isinstance(h, NullHandler)]) == 1)
def test_enabling_host_logger(self):
self.assertTrue(len([h for h in utils.host_logger.handlers
if isinstance(h, NullHandler)]) == 1)
utils.enable_host_logger()
# And again to test only one non-null handler is attached
utils.enable_host_logger()
self.assertTrue(len([h for h in utils.host_logger.handlers
if not isinstance(h, NullHandler)]) == 1)
utils.host_logger.handlers = [NullHandler()]
def __init__(
self,
workspace,
cluster_name,
ssh_key,
vm_type,
admin_username="******",
):
"""Thin wrapper class around azureml.core.compute.AmlCluster
Provides parallel ssh objects and helper for master node and all node commands
and file copies.
Usage:
>>> cc = ClusterConnector(workspace, "MyCluster", sshkey, "Standard_ND40rs_v2")
>>> cc.initialize(min_nodes=0, max_nodes=4, idle_timeout_secs=30)
>>> cluster = cc.cluster
>>> [print(node['name']) for node in cc.cluster.list_nodes()]
"""
self.cluster_name = cluster_name
self.workspace = workspace
self.ssh_key = ssh_key
self.vm_type = vm_type
self.admin_username = admin_username
enable_host_logger()
hlog = logging.getLogger("pssh.host_logger")
tstr = datetime.now().isoformat(timespec="minutes")
[
hlog.removeHandler(h) for h in hlog.handlers
if isinstance(h, logging.StreamHandler)
]
os.makedirs("clusterlogs", exist_ok=True)
self.logfile = "clusterlogs/{}_{}.log".format(self.workspace.name,
tstr)
hlog.addHandler(logging.FileHandler(self.logfile))
self.cluster = None
self._master_scp = None
self._master_ssh = None
self._all_ssh = None
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
"""Connect to SSH server on localhost,
attempt to perform an `ls` and copy a test file with both SSHClient
and ParallelSSHClient.
"""
from pssh import SSHClient, ParallelSSHClient, utils
import logging
from pprint import pprint
utils.enable_host_logger()
utils.enable_logger(utils.logger)
def test():
"""Perform ls and copy file with SSHClient on localhost"""
client = SSHClient('localhost')
channel, host, stdout, stderr = client.exec_command('ls -ltrh')
for line in stdout:
pprint(line.strip())
client.copy_file('../test', 'test_dir/test')
def test_parallel():
"""Perform ls and copy file with ParallelSSHClient on localhost.
Two identical hosts cause the same command to be executed
twice on the same host in two parallel connections.
def test_enabling_host_logger(self):
utils.enable_host_logger()
# And again to test only one handler is attached
utils.enable_host_logger()
self.assertTrue(len(utils.host_logger.handlers) == 1)
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
"""Connect to SSH server on localhost,
attempt to perform an `ls` and copy a test file with both SSHClient
and ParallelSSHClient.
"""
from pssh import SSHClient, ParallelSSHClient, utils
import logging
from pprint import pprint
utils.enable_host_logger()
utils.enable_logger(utils.logger)
def test():
"""Perform ls and copy file with SSHClient on localhost"""
client = SSHClient('localhost')
channel, host, stdout, stderr = client.exec_command('ls -ltrh')
for line in stdout:
pprint(line.strip())
client.copy_file('../test', 'test_dir/test')
def test_parallel():
"""Perform ls and copy file with ParallelSSHClient on localhost.
def test_enabling_host_logger(self):
utils.enable_host_logger()
# And again to test only one handler is attached
utils.enable_host_logger()
self.assertTrue(len(utils.host_logger.handlers)==1)
dest_path = get_server_config_path(f.split('/')[-1])
local_path = os.path.join(SERVER_CONFIG_DIR, f)
fileUpload.upload_file_no_pssh([server], key, local_path, dest_path)
print(f'Wrote file {local_path} to {server}')
output = client.run_command('bash ~/server_config/setup_new_server.sh')
utils.print_pssh_output(output)
with open(utils.get_project_path('all_servers.txt')) as f:
hostnames = {host.strip() for host in f.readlines()}
hostnames.add(server)
hostnames = sorted(hostnames)
with open(utils.get_project_path('all_servers.txt'), 'w') as f:
f.write('\n'.join(hostnames) + '\n')
print(
f"Added {server} to the fleet. Don't forget to commit autogenerated changes to all_servers.txt."
)
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='Start/Stop server in multiple nodes.')
parser.add_argument('server', type=str, help='server hostname')
args = parser.parse_args()
enable_host_logger()
server = args.server.strip()
key = local_config.get_planetlab_key()
setup_server(server, key)
