def get_ssh_client(auth: dict) -> SSHClient:
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
client.connect(auth['host'], auth['port'], auth['user'], auth['password'])
return client
def _connect(self):
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
client.load_system_host_keys()
# 0. If host in SUCCESS_CACHE try this first
for cred in find_creds(username=self.user,
schema='ssh',
port=self.port,
address=self.host):
if cred.password:
try:
client.connect(password=cred.password,
hostname=self.host,
port=self.port,
username=cred.user,
allow_agent=False,
look_for_keys=False,
gss_auth=False,
compress=not self._interactive,
timeout=self.timeout)
self._client = client
self._success_args = cred.as_dict()
self._success_args.update({
'hostname': self.host,
'port': self.port,
})
return True
except SSHException:
client.close()
elif cred.agent_socket:
SSH_AUTH_SOCK_bak = environ.get('SSH_AUTH_SOCK', None)
environ['SSH_AUTH_SOCK'] = cred.agent_socket
try:
client.connect(hostname=self.host,
port=self.port,
username=cred.user,
allow_agent=True,
look_for_keys=False,
password=None,
compress=not self._interactive,
timeout=self.timeout)
self._client = client
self._success_args = cred.as_dict()
self._success_args.update({
'hostname': self.host,
'port': self.port
})
return True
except SSHException:
client.close()
finally:
if SSH_AUTH_SOCK_bak is None:
del environ['SSH_AUTH_SOCK']
else:
environ['SSH_AUTH_SOCK'] = SSH_AUTH_SOCK_bak
elif cred.pkey:
try:
client.connect(hostname=self.host,
port=self.port,
username=cred.user,
pkey=cred.pkey,
allow_agent=False,
look_for_keys=False,
gss_auth=False,
compress=not self._interactive,
timeout=self.timeout)
self._client = client
self._success_args = cred.as_dict()
self._success_args.update({
'hostname': self.host,
'port': self.port
})
return True
except SSHException:
client.close()
current_user = getuser()
# 1. If password try password
if self.passwords:
for password in self.passwords:
username = self.user or current_user
try:
client.connect(password=password,
hostname=self.host,
port=self.port,
username=username,
allow_agent=False,
look_for_keys=False,
gss_auth=False,
compress=not self._interactive,
timeout=self.timeout)
self._client = client
self._success_args = {
'hostname': self.host,
'port': self.port,
'user': username,
'password': password,
'auto': True,
'cached': False,
}
return True
except SSHException:
client.close()
# 2. Try agent, default methods etc
for username, SSH_AUTH_SOCK in self._find_agent_sockets():
SSH_AUTH_SOCK_bak = environ.get('SSH_AUTH_SOCK', None)
environ['SSH_AUTH_SOCK'] = SSH_AUTH_SOCK
username = self.user or username
try:
client.connect(hostname=self.host,
port=self.port,
username=username,
allow_agent=True,
look_for_keys=False,
gss_auth=GSSAPI_MIC_SUPPORT,
password=None,
compress=not self._interactive,
timeout=self.timeout)
self._client = client
self._success_args = {
'hostname': self.host,
'port': self.port,
'user': username,
'agent_socket': SSH_AUTH_SOCK,
'auto': False,
'cached': False
}
return True
except SSHException:
client.close()
finally:
if SSH_AUTH_SOCK_bak is None:
del environ['SSH_AUTH_SOCK']
else:
environ['SSH_AUTH_SOCK'] = SSH_AUTH_SOCK_bak
# 3. Try all found pkeys
for key_file, key_data in self._iter_private_keys:
username = self.user or current_user
key_passwords = list(self.key_passwords)
key_passwords.insert(0, None)
found_key_password = None
key_data = self._convert(key_data, key_passwords)
pkey_obj = BytesIO(str(key_data))
pkey = None
for klass in KEY_CLASSES:
for key_password in key_passwords:
try:
pkey_obj.seek(0)
pkey = klass.from_private_key(pkey_obj,
password=key_password)
found_key_password = key_password
break
except SSHException:
continue
if pkey is None:
continue
try:
client.connect(hostname=self.host,
port=self.port,
username=username,
pkey=pkey,
allow_agent=False,
look_for_keys=False,
gss_auth=False,
compress=not self._interactive,
timeout=self.timeout)
self._client = client
self._success_args = {
'hostname': self.host,
'port': self.port,
'user': username,
'key': key_data,
'key_file': key_file,
'key_password': found_key_password,
'pkey': pkey,
'auto': False,
'cached': False
}
return True
except SSHException:
client.close()
if not self._client and client:
client.close()
)
except ModuleNotFoundError:
sys.exit("[-] You need to create admin_conf.py, see admin_conf.py_example")
urllib3.disable_warnings()
NUM_THREADS = 5
POSTPROCESS = "systemctl restart cape-processor; systemctl status cape-processor"
log = logging.getLogger("Cluster admin")
log.setLevel(logging.INFO)
logging.info("-")
servers = []
jumpbox_used = False
CI = False
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
def color(text, color_code):
if sys.platform == "win32" and os.getenv("TERM") != "xterm" or CI:
return text
return "\x1b[%dm%s\x1b[0m" % (color_code, text)
def red(text):
return color(text, 31)
def green(text):
import random
import sys
import os
import select
import paramiko
from paramiko import SSHClient
import subprocess
import time
client_master = SSHClient()
client_master.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client_master.load_system_host_keys()
client_master.connect("10.10.1.89", username="******", password="******")
client_slave1 = SSHClient()
client_slave1.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client_slave1.load_system_host_keys()
client_slave1.connect("10.10.1.90", username="******", password="******")
client_slave2 = SSHClient()
client_slave2.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client_slave2.load_system_host_keys()
client_slave2.connect("10.10.1.51", username="******", password="******")
def write_stop(file, diff, misconfig_code, client, write_dir):
file_lines = []
ftp = client.open_sftp()
remote_file = ftp.open("/home/ubuntu/Neha_Shreya/" + file)
for line in remote_file:
file_lines.append(line)
file_lines.append("TIME:" + str(diff) + ",CODE:" + str(misconfig_code) +
"\n")
def __init__(self, ip, uName, uPass):
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(hostname=ip, port=22, username=uName, password=uPass)
def create_ssh_client(host,
port,
username,
password,
private_key,
passphrase,
timeout=DEFAULT_TIMEOUT,
auth_timeout=DEFAULT_AUTH_TIMEOUT,
banner_timeout=DEFAULT_BANNER_TIMEOUT):
"""
Creates and returns a connected SSHClient.
If a password is supplied the connection is created using this password.
If no password is supplied a valid private key must be present. If this private key is encrypted the associated
passphrase must be supplied.
:param host: The host to connect to
:param username: The username which is used to connect to the ssh host
:param port: The port number to connect to
:param password: The password to authenticate
:param private_key: A valid private RSA key as string
:param passphrase: A passphrase to decrypt the private key, if the private key is encrypted
:param timeout: The timeout for the tcp connection. Defaults to 120.
:param auth_timeout: The authentication timeout in seconds. Defaults to 60.
:param banner_timeout: The banner timeout for the client in seconds. Defaults to 60.
:return: A connected paramiko.SSHClient
:raise ConnectionError: If the connection to the remote host failed or if neither password nor pkey are specified
:raise InvalidAuthenticationError: If the authentication to the remote host failed
"""
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
if password is not None:
try:
client.connect(host,
port=port,
username=username,
password=password,
timeout=timeout,
auth_timeout=auth_timeout,
banner_timeout=banner_timeout)
except socket.gaierror:
raise ConnectionError(
'Could not connect to remote host "{}"'.format(host))
except AuthenticationException:
raise InvalidAuthenticationError(
'Could not connect to remote host "{}". Invalid username/password.'
.format(host))
elif private_key is not None:
with create_temp_file(private_key) as key_file:
try:
pkey = RSAKey.from_private_key(key_file, password=passphrase)
except SSHException:
raise InvalidAuthenticationError(
'Could not connect to remote host "{}". Invalid key.'.
format(host))
try:
client.connect(host, username=username, pkey=pkey)
except socket.gaierror:
raise ConnectionError(
'Could not connect to remote host "{}"'.format(host))
else:
raise ConnectionError(
'At least password or private_key must be present.')
return client
'-u',
action='store',
help='The username to authenticate as',
dest='username',
required=True)
parser.add_argument('--branch',
'-b',
action='store',
help='Name of current git branch',
dest='branch',
required=True)
parser.add_argument('--plan',
'-pl',
action='store',
help='Name of current bamboo plan',
dest='plan',
required=True)
args = parser.parse_args()
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.load_system_host_keys()
ssh.connect(args.hostname, port=args.port, username=args.username)
download_specific_or_develop(ssh, args.plan, args.branch)
ssh.close()
def __init__(self): self.server = "192.168.0.2" self.client = SSHClient() self.client.load_system_host_keys() self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) self.client.connect(username='******', password='******', hostname=self.server)
def configureSSH(self):
"""
Configure ssh for use by scp.
"""
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
def push(args):
"""Push a file or files to a remote server.
Usage:
paver push [--password] [user@]hostname[:target_dir] file1, file2, ...
Uses pythonic paramiko-based SCP to copy files to the remote server.
If --password is provided at the command line, the user will be prompted
for a password. This is sometimes required when the remote's private key
requires a password to decrypt.
If a target username is not provided ([user@]...), the current user's username
used for the transfer.
If a target directory is not provided (hostname[:target_dir]), the current
directory of the target user is used.
"""
import paramiko
from paramiko import SSHClient
from scp import SCPClient
ssh = SSHClient()
ssh.load_system_host_keys()
# Clean out all of the user-configurable options flags.
config_opts = []
for argument in args:
if argument.startswith('--'):
config_opts.append(argument)
args.remove(argument)
use_password = '******' in config_opts
try:
destination_config = args[0]
except IndexError:
print "ERROR: destination config must be provided"
return
files_to_push = args[1:]
if len(files_to_push) == 0:
print "ERROR: At least one file must be given"
return
# ASSUME WE'RE ONLY DOING ONE HOST PER PUSH
# split apart the configuration string.
# format:
# [user@]hostname[:directory]
if '@' in destination_config:
username = destination_config.split('@')[0]
destination_config = destination_config.replace(username + '@', '')
else:
username = getpass.getuser()
if ':' in destination_config:
target_dir = destination_config.split(':')[-1]
destination_config = destination_config.replace(':' + target_dir, '')
else:
# just use the SCP default
target_dir = None
# hostname is whatever remains of the dest config.
hostname = destination_config
# start up the SSH connection
if use_password:
password = getpass.getpass()
else:
password = None
try:
ssh.connect(hostname, username=username, password=password)
except paramiko.BadAuthenticationType:
print 'ERROR: incorrect password or bad SSH key.'
return
except paramiko.PasswordRequiredException:
print 'ERROR: password required to decrypt private key on remote. Use --password flag'
return
scp = SCPClient(ssh.get_transport())
for transfer_file in files_to_push:
file_basename = os.path.basename(transfer_file)
if target_dir is not None:
target_filename = os.path.join(target_dir, file_basename)
else:
target_filename = file_basename
print 'Transferring %s -> %s:%s ' % (transfer_file, hostname,
target_filename)
scp.put(transfer_file, target_filename)
def discover_hardware(ip):
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(SilentPolicy())
client.connect(ip, username='******', timeout=2.0)
stdout = client.exec_command('cat /proc/partitions')[1].read()
disks = []
for line in stdout.split('\n'):
if not line: continue
line = [x for x in line.split(' ') if x]
if not line[0].isdigit(): continue
if not re.match('^[hs]d[a-z]$', line[3]): continue
name = line[3]
blocks = int(line[2])
blocks *= 1024
hdinfo = {
'osname': name,
'size': str(blocks),
}
# Query info from hdparm (IDE and SATA)
stdout = client.exec_command('hdparm -I /dev/%s' % name)[1].read()
useful = ('model', 'serial', 'firmware')
for field in stdout.split('\n'):
field = field.strip(' \t')
for u in useful:
if field.lower().startswith(u):
value = field.split(':', 1)[1]
hdinfo[u] = value.strip(' \t')
# Attempt a SCSI query
if not [x for x in useful if x in hdinfo]:
stdout = client.exec_command('/usr/bin/sg_inq /dev/%s' %
name)[1].read()
scsi_useful = {
'Product identification': 'model',
'Product revision level': 'firmware',
'Unit serial number': 'serial',
}
for field in [x.strip(' \t') for x in stdout.split('\n')]:
for u in scsi_useful:
if field.startswith(u):
key = scsi_useful[u]
value = field.split(':', 1)[1]
hdinfo[key] = value.strip(' \t')
if [x for x in useful if not x in hdinfo]:
sys.stdout.write('%s:missing ' % name)
disks.append(hdinfo)
xen = False
stdout = client.exec_command('uname -r')[1].read()
if stdout.lower().find('-xen-') != -1:
xen = True
stdout = client.exec_command('dmidecode -t memory')[1].read()
memory = []
mem = {}
for line in stdout.split('\n'):
if not line and mem:
memory.append(mem)
mem = {}
continue
if not line.startswith('\t'): continue
key, value = line.lstrip('\t').split(': ', 1)
if key in ('Locator', 'Type', 'Speed', 'Size'):
mem[key.lower()] = value
processors = []
cpu = {}
if xen:
sys.stdout.write('xen ')
sys.stdout.flush()
stdout = client.exec_command('/usr/sbin/xm info')[1].read()
for line in stdout.split('\n'):
line = line.split(':', 1)
if len(line) != 2:
continue
key, value = line
key = key.strip(' \t')
value = value.strip(' \t')
if key == 'nr_cpus':
cpucount = int(value)
if key == 'total_memory':
kmem = int(value)
else:
stdout = client.exec_command('/usr/bin/free -m')[1].read()
stdout = [x for x in stdout.split('\n')[1].split(' ') if x]
kmem = int(stdout[1])
stdout = client.exec_command('cat /proc/cpuinfo')[1].read()
for line in stdout.split('\n'):
if not line and cpu:
processors.append(cpu)
cpu = {}
continue
if not line: continue
key, value = line.split(':', 1)
key = key.strip(' \t')
if key in ('model name', 'cpu MHz', 'cache size', 'vendor_id'):
key = key.lower().replace(' ', '-').replace('_', '-')
cpu[key] = value.strip(' ')
cpucount = len(processors)
serial = client.exec_command(
'/usr/sbin/dmidecode --string=system-serial-number')[1].read().rstrip(
'\r\n')
hostname = client.exec_command('/bin/hostname -s')[1].read().rstrip('\r\n')
stdout = client.exec_command('/sbin/ifconfig -a')[1].read()
iface = {}
for line in stdout.split('\n'):
line = line.rstrip('\r\n')
if not line: continue
line = line.split(' ')
if line[0]:
name = line[0]
iface[name] = []
del line[0]
line = [x for x in line if x]
iface[name] += line
for name in iface:
attribs = {}
value = None
for attr in iface[name]:
value = None
if attr.startswith('Link encap') or \
attr.startswith('inet addr') or \
attr.startswith('Bcast') or \
attr.startswith('Mask') or \
attr.startswith('MTU') or \
attr.startswith('Metric'):
key, value = attr.split(':', 1)
if attr.startswith('HWaddr'):
key, value = attr.split(' ', 1)
if attr.startswith('inet6 addr'):
key, value = attr.split(': ', 1)
if not value: continue
attribs[key.lower()] = value
iface[name] = attribs
client.close()
return {
'disk':
disks,
'memory':
memory,
'processor':
processors,
'network':
iface,
'system': [{
'serial': serial,
'cpucount': cpucount,
'hostname': hostname,
'memory': kmem,
'disk': sum([int(x['size'][:-9]) for x in disks])
}],
}
def connect(self,
host,
username,
keyFile="",
password='',
port_forward=False,
port=""):
try:
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(AutoAddPolicy())
if self.port_forward:
if self.keyFile != '':
try:
if password != "":
client.connect(host,
username=username,
key_filename=keyFile,
passphrase=password,
look_for_keys=True,
timeout=5000)
else:
client.connect(host,
username=username,
key_filename=keyFile,
look_for_keys=True,
timeout=5000)
sshTunnel = SSHTunnelForwarder(
(host, 22),
ssh_pkey=keyFile,
ssh_username=username,
ssh_private_key_password=password,
remote_bind_address=('127.0.0.1', port))
except Exception as e:
self.error = e
return None
else:
sshTunnel = SSHTunnelForwarder(
(host, 22),
ssh_username=username,
ssh_password=password,
remote_bind_address=('127.0.0.1', port))
client.connect(host,
username=username,
password=password,
look_for_keys=True,
timeout=5000)
sshTunnel.start()
return client, sshTunnel
else:
if keyFile != '':
try:
if password != "":
client.connect(host,
username=username,
key_filename=keyFile,
passphrase=password,
look_for_keys=True,
timeout=5000)
else:
client.connect(host,
username=username,
key_filename=keyFile,
look_for_keys=True,
timeout=5000)
except Exception as e:
self.error = e
return None
else:
client.connect(host,
username=username,
password=password,
look_for_keys=True,
timeout=5000)
return client
except Exception as e:
self.error = e
return None
def download_log_file_from_cromwell(server,
file_path,
local_file_path,
type,
scp_port=22,
scp_user=None,
scp_key=None):
# Download a log file from the cromwell server
# server: name or ip of the cromwell server
# file_path: the path on the cromwell server of the file
# to be downloaded
# local_file_path: the path of the copied file on local file system
# type: how to perform the download. Currently only scp is supported
# scp_user: user name to be used to authenticate to Cromwell server
# scp_key: location of ssh key to be used by scp.
#
logging.getLogger("paramiko").setLevel(logging.WARNING)
if type == "scp":
from scp import SCPClient, SCPException
from paramiko import SSHClient, AutoAddPolicy
# Check if ssh private key exists
expanded_scp_key = os.path.expanduser(scp_key)
if not os.path.isfile(expanded_scp_key):
logstatus = ("SSH keyfile does not exist '{}'. Download of "
"log file will be skipped".format(expanded_scp_key))
# logmsgs = "LOGFILE DOWNLOAD ERROR"
return logstatus
ssh = SSHClient()
# ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(server,
port=scp_port,
username=scp_user,
key_filename=expanded_scp_key,
timeout=300)
# Create SCP client
scp = SCPClient(ssh.get_transport())
# Download the file
try:
# local_file_path = os.path.basename(file_path)
scp.get(file_path, local_file_path)
scp.close()
ssh.close()
except SCPException as err:
err.args
logstatus = ("Unable to download log file, '{}': {} ".format(
local_file_path, err.args))
# logmsgs = "LOGFILE DOWNLOAD ERROR"
return logstatus
logstatus = "True"
return logstatus
else:
# A non-supported download method was selected
logstatus = ("Download method of '{}' was specified and "
"this is not supported".format(type))
# logmsgs = "LOGFILE DOWNLOAD ERROR"
return logstatus
def __init__(self, **kwargs):
self.client = SSHClient()
self.client.set_missing_host_key_policy(AutoAddPolicy())
self.kwargs = kwargs
from paramiko import SSHClient, AutoAddPolicy
import logging
logging.basicConfig(filename="run_bc.log")
log = logging.getLogger("paramiko")
log.setLevel(logging.DEBUG)
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(AutoAddPolicy())
commands = """
scale=1000
sqrt(2)
quit
"""
client.connect('192.168.56.101', username='******', password='******')
stdin, stdout, stderr = client.exec_command('bc')
stdin.write(commands)
# for line in stdout:
# print('... ' + line.strip('\n'))
data = stdout.readlines()
client.close()
for line in data:
print(line.strip())
def main():
logging.basicConfig(level=logging.INFO)
if isfile(get_config_path(__file__, '/esxi.ini')):
config = ConfigurationINI(get_config_path(__file__, '/esxi.ini'))
elif isfile('/etc/esxi.ini'):
config = ConfigurationINI('/etc/esxi.ini')
else:
logging.critical('/etc/esxi.ini missing.')
exit(0)
logging.debug('Configuration file used : {conf}'.format(
conf=get_config_path(__file__, '/esxi.ini')))
try:
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(config['esxi']['hostip'],
username=config['esxi']['username'],
password=config['esxi']['password'])
except SocketError as e:
logging.critical('Host unreachable.')
logging.critical(e.__str__())
exit()
logging.info('vim-cmd hostsvc/firmware/sync_config')
ssh.exec_command('vim-cmd hostsvc/firmware/sync_config')
logging.info('vim-cmd hostsvc/firmware/backup_config')
stdin, stdout, stderr = ssh.exec_command(
'vim-cmd hostsvc/firmware/backup_config')
for l in stdout:
m = search('http://\*(.*)', l.strip())
if m is not None:
download = "http://{host}{position}".format(
host=config['esxi']['hostip'], position=m.group(1))
logging.info("Downloading {0}".format(download))
local_file = '{localpath}/backup-{host}-{date}.tgz'.format(
host=config['esxi']['hostdns'],
date=strftime(config['local']['dateformat']),
localpath=config['local']['savepath'])
urlretrieve(download, local_file)
if config['webdav']['enabled']:
logging.info("Uploading file on WebDAV")
comediaoc = webdav_connect(
config['webdav']['host'],
username=config['webdav']['username'],
password=config['webdav']['password'],
protocol=config['webdav']['proto'],
verify_ssl=False)
comediaoc.upload(
local_file, '{0}/backup-{1}-{2}.tgz'.format(
config['webdav']['savepath'],
config['esxi']['hostdns'],
strftime(config['local']['dateformat'])))
logging.info('Cleaning old configurations')
files = [
file for file in comediaoc.ls(config['webdav']['savepath'])
if file.contenttype == 'application/x-compressed'
]
ssh.close()
def change_port_status(request):
""" Activates or Deactivates a port.
:param port_id: The port's id.
:type port_id: int
"""
# Pull post parameters
port_id = request.data["port_id"]
port_instance = Port.objects.get(id=port_id)
# Set up paramiko ssh client
ssh_client = SSHClient()
ssh_client.set_missing_host_key_policy(AutoAddPolicy())
ssh_client.connect(str(port_instance.upstream_device.ip_address),
username=settings.RESNET_SWITCH_SSH_USER,
password=settings.RESNET_SWITCH_SSH_PASSWORD,
allow_agent=False,
look_for_keys=False)
ssh_shell = ssh_client.invoke_shell()
if ssh_shell.get_transport().is_active():
ssh_shell.send('conf t\n')
time.sleep(.5)
ssh_shell.send('interface Gi' + str(port_instance.blade_number) + '/' +
str(port_instance.port_number) + '\n')
time.sleep(.5)
else:
raise IOError('Lost connection to switch {switch}.'.format(
switch=port_instance.upstream_device.ip_address))
if ssh_shell.get_transport().is_active():
if port_instance.active:
ssh_shell.send('shutdown\n')
time.sleep(.5)
else:
ssh_shell.send('no shutdown\n')
time.sleep(.5)
buffer_size = 1024
stderr = ""
# Pull stderr from the buffer
while ssh_shell.recv_stderr_ready():
stderr += smart_str(ssh_shell.recv_stderr(buffer_size))
# Attempt to parse errors
if stderr:
raise IOError(stderr)
else:
# An error did not occur.
port_instance.active = not port_instance.active
port_instance.save()
else:
raise IOError('Lost connection to switch {switch}.'.format(
switch=port_instance.upstream_device.ip_address))
# Close ssh connection(s)
ssh_shell.close()
ssh_client.close()
return redraw_row(request, PopulatePorts, port_id)
def main(config: Config) -> None:
print_verbose(
f"Downloading backup from {config.ssh_username}@{config.ssh_hostname}:{config.ssh_port} "
f"with {config.ssh_keyfile}")
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
passphrase = ""
if config.ssh_passphrase is not None:
passphrase = config.ssh_passphrase
ssh.connect(config.ssh_hostname,
port=config.ssh_port,
username=config.ssh_username,
key_filename=config.ssh_keyfile,
passphrase=passphrase)
# list existing backups (from stdout)
existing_backups = ssh.exec_command(
f"ls {config.server_location}")[1].readlines()
# remove \n from list entries
existing_backups = list(map(lambda s: s.strip(), existing_backups))
# sort existing backups and get the directory name of the latest
backup_date = sorted(existing_backups,
key=lambda x: datetime.strptime(x, "%Y-%m-%d"))[-1]
sftp = ssh.open_sftp()
# local directory where the backup should be stored
local_path: str = config.local_location
if not local_path.endswith("/"):
local_path += "/"
local_path += backup_date
# create local directory for the backup
if not os.path.exists(local_path):
os.mkdir(local_path)
else:
# if local path exists, but is not a directory, exit
if not os.path.isdir(local_path):
print_verbose(f"{local_path} is not a directory! Exitig")
exit(1)
# backup location on the server (remote path)
server_path: str = config.server_location
if not server_path.endswith("/"):
server_path += "/"
server_path += backup_date
for filename in list(
filter(
None,
ssh.exec_command(f"ls {server_path}")[1].read().decode().split(
"\n"))):
print_verbose(
f"- {server_path}/{filename} (remote) => {local_path}/{filename} (local)"
)
sftp.get(f"{server_path}/{filename}", f"{local_path}/{filename}")
# check check sums
methods: list = ["sha512", "sha384", "sha256", "sha224", "sha1", "md5"]
best_available_method = next(
(m for m in methods if os.path.isfile(f"{local_path}/{m}sum.txt")),
None)
if best_available_method:
with open(f"{local_path}/{best_available_method}sum.txt",
"r") as check_sum_file:
for entry in check_sum_file.readlines():
checksum, filepath = entry.replace("\n", "").split("\t")
if not os.path.isfile(f"{local_path}/{filepath}"):
print_verbose(
f"There is a checksum for {os.path.basename(filepath)}, but the file does not exist"
)
continue
if checksum != getattr(hashlib, best_available_method)(open(
f"{local_path}/{filepath}", "rb").read()).hexdigest():
print(
f"Checksum of {os.path.basename(filepath)} is not correct!"
)
else:
print_verbose("Unable to find check sums, skipping integrity check!")
options['ip'] = "moonstone.unist.ac.kr"
options['path'] = ['/David/180602_median2_160515_SWiFT_60nmpx_singles/']
options['emerald'] = "/scratch/jmmoon/180602_median2_160515_SWiFT_120nmpx_singles_distribute/"
options['file_type'] = "png"
options['user'] = "******"
options['password'] = "******"
options['port'] = 17771
out_root_path = "/scratch/jmmoon/180602_median2_160515_SWiFT_120nmpx_singles_distribute_rename/"
# TargeFileList = {}
orig_name_list = []
with closing(SSHClient()) as ssh:
ssh.load_system_host_keys() #NOTE: no AutoAddPolicy()
ssh.connect(options['ip'], port=options['port'], username=options['user'], password=options['password'])
with closing(ssh.open_sftp()) as sftp:
for p in options['path']:
sftp.chdir(p)
for filename in sftp.listdir():
if filename[0] != ".":
# v = int(filename.split("v")[-1].split("_")[0])
# s = int(filename.split("s")[-1].split( "." + options['file_type'])[0])
orig_name_list.append(p + filename)
orig_name_list.sort()
def ssh_client() -> SSHClient:
ssh_client = SSHClient()
ssh_client.set_missing_host_key_policy(AutoAddPolicy())
return ssh_client
def connectToPi(ip, username='******', pw='1357'):
print('connecting to {}@{}...'.format(username, ip))
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(ip, username=username, password=pw)
return ssh
def __init__(self):
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.__rcode = 0
def handle(self, *args, **options):
"""
Gets the appropriate settings from Django and deploys the repository
to the target servers for the instance selected.
"""
# Grab the quiet settings and unique stamp
quiet = options["quiet"]
stamp = options["stamp"]
no_confirm = options["no_confirm"]
# Check to ensure the require setting is in Django's settings.
if hasattr(settings, "DEPLOYER_INSTANCES"):
instances = settings.DEPLOYER_INSTANCES
else:
raise CommandError(
"You have not configured DEPLOYER_INSTANCES in your Django settings."
)
# Grab the instance settings if they're properly set
if options["instance"] in instances:
instance = instances[options["instance"]]
else:
raise CommandError(
"The instance name you provided ('{instance}') is not configured in "
"your settings DEPLOYER_INSTANCES. Valid instance names are: "
"{instances}".format(
instance=options["instance"],
instances=", ".join(list(instances.keys())),
))
print(
"We are about to deploy the instance '{instance}' to the following "
"servers: {servers}.".format(instance=options["instance"],
servers=", ".join(
instance["servers"])))
if no_confirm:
verify = "yes"
else:
verify = input(
"Are you sure you want to do this (enter 'yes' to proceed)? ")
if verify.lower() != "yes":
print("You did not type 'yes' - aborting.")
return
# Variables for directory locations and symlinks
deployer_clone_dir_format = getattr(
settings,
"DEPLOYER_CLONE_DIR_FORMAT",
"{name}-{instance}",
)
deployer_clone_dir = deployer_clone_dir_format.format(
instance=options["instance"],
name=instance["name"],
branch=instance["branch"],
server_user=instance["server_user"],
)
git_dir_stamp = "{deployer_clone_dir}-{stamp}".format(
deployer_clone_dir=deployer_clone_dir, stamp=stamp)
install_code_path = os.path.join(
instance["code_path"],
deployer_clone_dir,
)
install_code_path_stamp = os.path.join(
instance["code_path"],
git_dir_stamp,
)
# On our first iteration, we clone the repository, build the virtual
# environment, and collect static files; this may take some time.
for index, server in enumerate(instance["servers"]):
# Open an SSH connection to the server
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(server, username=instance["server_user"])
print("Cloning code and preparing venv on node: {}...".format(
server))
# Make sure the code_path directory exists
stdin, stdout, stderr = ssh.exec_command("""
mkdir -p {directory}
""".format(directory=instance["code_path"]))
self.command_output(stdout, stderr, quiet)
stdin, stdout, stderr = ssh.exec_command("""
cd {code_path}
git clone --recursive --verbose -b {branch} {repository} {git_dir_stamp}
""".format(
code_path=instance["code_path"],
branch=instance["branch"],
repository=instance["repository"],
git_dir_stamp=git_dir_stamp,
))
self.command_output(stdout, stderr, quiet)
print("Preparing the installation...")
if instance.get("upgrade_pip", True):
pip_upgrade_text = "pip install -U pip"
else:
pip_upgrade_text = ""
print("pip will NOT be upgraded.")
if instance.get("collectstatic", True):
collectstatic_text = (
"python manage.py collectstatic --noinput --settings={s}".
format(s=instance["settings"], ))
else:
collectstatic_text = ""
print("Static files will NOT be collected.")
stdin, stdout, stderr = ssh.exec_command("""
cd {install_code_path_stamp}
{venv_python_path} -m venv venv
. venv/bin/activate
{pip_upgrade_text}
pip install -U wheel
pip install --ignore-installed -r {requirements}
{collectstatic_text}
""".format(
install_code_path_stamp=install_code_path_stamp,
venv_python_path=instance["venv_python_path"],
pip_upgrade_text=pip_upgrade_text,
requirements=instance["requirements"],
collectstatic_text=collectstatic_text,
))
self.command_output(stdout, stderr, quiet)
if "selinux" in instance and instance["selinux"]:
print(
"Setting security context for RedHat / CentOS SELinux...")
stdin, stdout, stderr = ssh.exec_command("""
chcon -Rv --type=httpd_sys_content_t {install_code_path_stamp} > /dev/null
find {install_code_path_stamp}/venv/ \( -name "*.so" -o -name "*.so.*" \) -exec chcon -Rv --type=httpd_sys_script_exec_t {{}} \; > /dev/null
""".format(
install_code_path_stamp=install_code_path_stamp))
self.command_output(stdout, stderr, quiet)
# On our second iteration, we update symlinks, keep only recent deployments,
# run any additional commands, and run migrations. This should be fast.
for index, server in enumerate(instance["servers"]):
# Open an SSH connection to the server
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(server, username=instance["server_user"])
print("")
print(
"Updating symlinks and running any additional defined commands on "
"node: {}...".format(server))
stdin, stdout, stderr = ssh.exec_command("""
ln -sfn {install_code_path_stamp} {install_code_path}
""".format(
install_code_path_stamp=install_code_path_stamp,
install_code_path=install_code_path,
))
self.command_output(stdout, stderr, quiet)
if int(instance.get("save_deploys", 0)) > 0:
print(
"Keeping the {} most recent deployments, and deleting the rest on "
"node: {}".format(instance["save_deploys"], server))
stdin, stdout, stderr = ssh.exec_command("""
ls -1trd {install_code_path}* | head -n -{save_deploys} | xargs -d '\\n' rm -rf --
""".format(
install_code_path=install_code_path,
save_deploys=instance["save_deploys"] + 1,
))
self.command_output(stdout, stderr, quiet)
if "additional_commands" in instance:
print("Performing defined additional commands...")
for additional_command in instance["additional_commands"]:
print("Running '{}'...".format(additional_command))
stdin, stdout, stderr = ssh.exec_command(
additional_command)
self.command_output(stdout, stderr, quiet)
# Only run migrations when we're on the last server.
if index + 1 == len(instance["servers"]) and instance.get(
"migrate", True):
print("Finally, running migrations...")
stdin, stdout, stderr = ssh.exec_command("""
cd {install_code_path_stamp}
. venv/bin/activate
python manage.py migrate --noinput --settings={settings}
""".format(
install_code_path_stamp=install_code_path_stamp,
settings=instance["settings"],
))
self.command_output(stdout, stderr, quiet)
else:
print("Not running migrations; migrate is set to False.")
print("All done!")
class Common(object):
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
HOSTNAME = socket.gethostname()
def __init__(self,
hostname,
port=None,
username=None,
password=None,
pkey=None,
timeout=None):
self.hostname = hostname
self.port = 22 if port is None else port
if username is None:
if "username" not in cfg.options("default"):
self.username = '******'
self.HOME = "/root"
else:
self.username = cfg.get("default", "username")
self.HOME = os.path.join('/home', self.username)
else:
if username == 'root':
self.username = '******'
self.HOME = "/root"
else:
self.username = username
self.HOME = os.path.join('/home', self.username)
if password is None:
if "password" not in cfg.options("default"):
self.password = None
else:
self.password = cfg.get("default", "password")
else:
self.password = password
if pkey is None and password is None:
self.pkey = paramiko.RSAKey.from_private_key_file(
cfg.get("default", "private_key"))
else:
self.pkey = None
self.timeout = 3 if timeout is None else timeout
self.auth(self.timeout)
def run(self, cmd, timeout=None):
# 执行远程shell命令
if timeout is None:
timeout = 600
self.ssh.connect(hostname=self.hostname,
port=self.port,
username=self.username,
password=self.password,
pkey=self.pkey,
timeout=self.timeout)
stdin, stdout, stderr = self.ssh.exec_command(cmd, timeout=timeout)
status = stdout.channel.recv_exit_status()
return self.format(self.hostname, stdout, stderr, status)
def sudo(self, cmd, timeout=None):
# 执行远程sudo命令
if timeout is None:
timeout = 600
self.ssh._transport = self.transport
# stdin,stdout,stderr = self.ssh.exec_command('sudo sh -c "{}"'.format(cmd),timeout=timeout)
stdin, stdout, stderr = self.ssh.exec_command(
'sudo -S sh -c "{}"'.format(cmd), timeout=timeout)
stdin.write("{}\n".format(self.password))
stdin.flush()
status = stdout.channel.recv_exit_status()
return self.format(self.hostname, stdout, stderr, status)
def exec_script(self,
script,
args=None,
sudo=False,
type=None,
display=True):
# 本地脚本远程执行
self.initPath()
destFile_script = os.path.join(self.ATOM_PATH,
os.path.split(script)[1])
self.sftp_upload(script, destFile_script, display=False)
if args is None:
if display:
print("[{}] Executing {} script.".format(
self.hostname, script))
if sudo:
result = self.sudo("{} {}".format(type, destFile_script))
else:
result = self.run("{} {}".format(type, destFile_script))
return result
else:
if display:
print("[{}] Executing '{} {}' script.".format(
self.hostname, script, args))
if sudo:
result = self.sudo("{} {} {}".format(type, destFile_script,
args))
else:
result = self.run("{} {} {}".format(type, destFile_script,
args))
return result
def sftp_upload(self, source, dest, display=True):
# 上传文件或者目录
self.initPath()
sftp = paramiko.SFTPClient.from_transport(self.transport)
if os.path.isdir(source):
if not os.path.isabs(source):
real_path = os.path.dirname(os.path.realpath(__file__))
source = os.path.join(real_path, source)
for root, dirs, files in os.walk(source):
for filename in files:
local_file = os.path.join(root, filename)
remote_file = ''.join(
[dest, local_file.replace(source, '')])
try:
sftp.put(local_file, remote_file)
except Exception as e:
sftp.mkdir(os.path.split(remote_file)[0])
sftp.put(local_file, remote_file)
print("[{}] upload {} to {} success.".format(
self.hostname, local_file, remote_file))
for name in dirs:
local_path = os.path.join(root, name)
remote_path = ''.join(
[dest, local_path.replace(source, '')])
try:
sftp.mkdir(remote_path)
print("[{}] mkdir path {}".format(
self.hostname, remote_path))
except Exception as error:
self.run("mkdir -p {}".format(remote_path))
print("[{}] mkdir {} success.".format(
self.hostname, remote_path))
else:
try:
sftp.put(source, dest)
if display is True:
print("[{}] upload {} to {} {} success.".format(
self.HOSTNAME, source, self.hostname, dest))
except Exception as error:
try:
sftp.mkdir(os.path.split(dest)[1])
if display is True:
print("[{}] mkdir path {}".format(
self.hostname,
os.path.split(dest)[1]))
sftp.put(source, dest)
if display is True:
print("[{}] upload {} to {} {} success.".format(
self.HOSTNAME, source, self.hostname, dest))
except Exception as error:
print("[{}] {} {}".format(self.HOSTNAME, error, dest))
def sftp_down_file(self, remotePath, localPath=None):
# 下载文件
if localPath is None:
localPath = os.path.join(os.path.abspath('.'),
os.path.split(remotePath)[1])
sftp = paramiko.SFTPClient.from_transport(self.transport)
try:
sftp.get(remotePath, localPath)
except Exception as e:
sftp.mkdir(os.path.join(localPath)[1])
sftp.get(remotePath, localPath)
def format(self, hostname, stdout, stderr, status):
if status == 0:
status = True
else:
status = False
stderr = stderr.read().replace(
"[sudo] password for {}: ".format(self.username), '')
result = {
hostname: {
"stdout": stdout.read(),
"stderr": stderr
},
"status": status
}
return result
def auth(self, timeout=None):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
if timeout is not None:
sock.settimeout(timeout)
sock.connect((self.hostname, self.port))
try:
self.transport = Transport(self.hostname, self.port)
if self.password is not None:
self.transport.connect(username=self.username,
password=self.password)
else:
self.transport.connect(username=self.username,
pkey=self.pkey)
except ssh_exception.SSHException as error:
raise Exception("{} {}".format(self.hostname, error.message))
except socket.error as error:
raise Exception("{} {}".format(self.hostname, error))
def initPath(self):
self.ATOM_PATH = os.path.join(self.HOME, '.atom')
self.run("mkdir {}".format(self.ATOM_PATH))
def __exit__(self, exc_type, exc_val, exc_tb):
self.sudo("rm -rf {}".format(self.ATOM_PATH))
self.ssh.close()
self.transport.close()
def activate_workers(logdir='train_logs',
git_dir='vaelstmpredictor',
verbose=True):
machines = [
{
"host": "172.16.50.181",
"username": "******",
"key_filename": key_filename
},
#{"host": "172.16.50.176", "username": "******",
# "key_filename": key_filename},
{
"host": "172.16.50.177",
"username": "******",
"key_filename": key_filename
},
#{"host": "172.16.50.163", "username": "******",
# "key_filename": key_filename},
{
"host": "172.16.50.182",
"username": "******",
"key_filename": key_filename
}, # not operation today
{
"host": "172.16.50.218",
"username": "******",
"key_filename": key_filename
},
{
"host": "172.16.50.159",
"username": "******",
"key_filename": key_filename
},
{
"host": "172.16.50.235",
"username": "******",
"key_filename": key_filename
},
{
"host": "172.16.50.157",
"username": "******",
"key_filename": key_filename
},
{
"host": "172.16.50.237",
"username": "******",
"key_filename": key_filename
}
]
for machine in machines:
try:
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(machine["host"], key_filename=machine['key_filename'])
except NoValidConnectionsError as error:
warning_message(error)
ssh.close()
stdin, stdout, stderr = ssh.exec_command(
'ls | grep {}'.format(git_dir))
if (len(stdout.readlines()) == 0):
git_clone(machine["host"], key_filename=machine['key_filename'])
elif verbose:
info_message('File {} exists on {}'.format(git_dir,
machine['host']))
stdin, stdout, stderr = ssh.exec_command(
'nohup python {}/RunWorker.py'.format(git_dir))
def ssh_connect(serv, **kwargs):
import sql
fullpath = get_config_var('main', 'fullpath')
for sshs in sql.select_ssh(serv=serv):
ssh_enable = sshs[3]
ssh_user_name = sshs[4]
ssh_user_password = sshs[5]
ssh_key_name = fullpath + '/keys/%s.pem' % sshs[2]
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
if ssh_enable == 1:
k = paramiko.RSAKey.from_private_key_file(ssh_key_name)
ssh.connect(hostname=serv, username=ssh_user_name, pkey=k)
else:
ssh.connect(hostname=serv,
username=ssh_user_name,
password=ssh_user_password)
if kwargs.get('check'):
return True
else:
return ssh
except paramiko.AuthenticationException:
if kwargs.get('check'):
print(
'<div class="alert alert-danger">Authentication failed, please verify your credentials</div>'
)
return False
else:
return 'Authentication failed, please verify your credentials'
pass
except paramiko.SSHException as sshException:
if kwargs.get('check'):
print(
'<div class="alert alert-danger">Unable to establish SSH connection: %s </div>'
% sshException)
return False
else:
return 'Unable to establish SSH connection: %s ' % sshException
pass
except paramiko.BadHostKeyException as badHostKeyException:
if kwargs.get('check'):
print(
'<div class="alert alert-danger">Unable to verify server\'s host key: %s </div>'
% badHostKeyException)
return False
else:
return 'Unable to verify server\'s host key: %s ' % badHostKeyException
pass
except Exception as e:
if e.args[1] == "No such file or directory":
if kwargs.get('check'):
print(
'<div class="alert alert-danger">{}. Check ssh key</div>'.
format(e.args[1]))
else:
return '{}. Check ssh key'.format(e.args[1])
pass
elif e.args[1] == "Invalid argument":
if kwargs.get('check'):
print(
'<div class="alert alert-danger">Check the IP of the new server</div>'
)
else:
error = 'Check the IP of the new server'
pass
else:
if kwargs.get('check'):
print('<div class="alert alert-danger">{}</div>'.format(
e.args[1]))
else:
error = e.args[1]
pass
if kwargs.get('check'):
return False
else:
return error
def move_ws(request):
response_data = {}
ws_name = request.POST.get('ws_name')
release = request.POST.get('release')
id_ip = request.POST.get('id_ip').encode('UTF-8')
id_path = request.POST.get('id_path').encode('UTF-8')
id_username = request.POST.get('id_username').encode('UTF-8')
id_password = request.POST.get('id_password').encode('UTF-8')
print __name__ + ".py :", sys._getframe().f_code.co_name
try:
res = models.user_workspace.objects.get(
ws_name=ws_name, added_by=request.session['users'])
ws_path = res.ws_path
except:
DatabaseError
os.system("rm -rf " + ws_path + ".tar.gz")
response_data['msg'] = "Database error..."
return HttpResponse(json.dumps(response_data),
content_type="application/json")
print("ws_name :",
(request.session['users'] + "_" + ws_name).encode("UTF-8"),
"ws_path :", ws_path.encode("UTF-8"))
make_tarfile(
settings.ws_path.encode("UTF-8") + "/" +
(request.session['users'] + "_" + ws_name).encode("UTF-8") + ".tar.gz",
ws_path.encode("UTF-8"))
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
try:
ssh.connect(id_ip, port=22, username=id_username, password=id_password)
# SCPCLient takes a paramiko transport as its only argument
# Just a no-op. Required sanitize function to allow wildcards.
scp = SCPClient(ssh.get_transport(),
sanitize=lambda x: x,
progress=progress)
try:
scp.put("" + ws_path + ".tar.gz",
remote_path=str(id_path),
recursive=False)
except Exception as x:
print(x)
os.system("rm -rf " + ws_path + ".tar.gz")
response_data['msg'] = "Workspace not moved as..." + str(
x
) + " ip = " + id_ip + " path = " + id_path + " user = "******"application/json")
except Exception as x:
print(x)
os.system("rm -rf " + ws_path + ".tar.gz")
response_data['msg'] = "Workspace not moved as..." + str(
x
) + " ip = " + id_ip + " path = " + id_path + " user = "******"application/json")
try:
res.delete()
except:
DatabaseError
try:
shutil.rmtree(ws_path, ignore_errors=True)
except:
DatabaseError
cmd = "rm -rf " + ws_path + ".tar.gz"
res1 = subprocess.check_output(cmd, shell=True)
print "\n\n>>>>>>>>", res1
if res1 == 0:
response_data['msg'] = "Workspace moved successfully..."
response_data['status'] = True
else:
response_data[
'msg'] = "Failed building ISO... \nCheck log for details."
response_data['status'] = False
#os.system()
response_data['msg'] = "Workspace moved successfully..."
return HttpResponse(json.dumps(response_data),
content_type="application/json")
def connectSSH(user, host, passwd): ssh = SSHClient() ssh.load_system_host_keys() ssh.connect(host, username=user, password=passwd) return ssh
def __init__(self):
self.client = SSHClient()
self.sftp = None
self.isActive = None
def Connect(ip, username='******', pw='basicrandompassword'):
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(ip, username=username, password=pw)
return ssh
