def _get_ssh_connection(self):
"""Returns an ssh connection to the specified host"""
_timeout = True
ssh = SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
_start_time = time.time()
saved_exception = exceptions.StandardError()
#doing this because the log file fills up with these messages
#this way it only logs it once
log_attempted = False
socket_error_logged = False
auth_error_logged = False
ssh_error_logged = False
while not self._is_timed_out(self.timeout, _start_time):
try:
if not log_attempted:
self._log.debug('Attempting to SSH connect to: ')
self._log.debug('host: %s, username: %s, password: %s' %
(self.host, self.username, self.password))
log_attempted = True
ssh.connect(hostname=self.host,
username=self.username,
password=self.password,
timeout=20,
key_filename=[],
look_for_keys=False,
allow_agent=False)
_timeout = False
break
except socket.error as e:
if not socket_error_logged:
self._log.error('Socket Error: %s' % str(e))
socket_error_logged = True
saved_exception = e
continue
except paramiko.AuthenticationException as e:
if not auth_error_logged:
self._log.error('Auth Exception: %s' % str(e))
auth_error_logged = True
saved_exception = e
time.sleep(2)
continue
except paramiko.SSHException as e:
if not ssh_error_logged:
self._log.error('SSH Exception: %s' % str(e))
ssh_error_logged = True
saved_exception = e
time.sleep(2)
continue
#Wait 2 seconds otherwise
time.sleep(2)
if _timeout:
self._log.error('SSHConnector timed out while trying to establish a connection')
raise saved_exception
#This MUST be done because the transport gets garbage collected if it
#is not done here, which causes the connection to close on invoke_shell
#which is needed for exec_shell_command
ResourceManager.register(self, ssh.get_transport())
return ssh
def __connect(self): assert self._server_pub_key and self._server_url and isinstance(self._my_host_key_policy, MissingHostKeyPolicy) client = SSHClient() client.set_missing_host_key_policy(self._my_host_key_policy) client.connect(self._server_url) # host alias from .ssh/config # self._ssh_client = client return True
def run(self):
if self.GITHUB_TOKEN is None:
logging.critical('No github OAuth token defined in the GITHUB_TOKEN env variable')
sys.exit(1)
if self.SSH_PKEY is None:
logging.critical('SSH_KEY not configured, please set it to you private SSH key file')
sys.exit(1)
github = Github(self.GITHUB_TOKEN)
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.SSH_PKEY = os.path.expanduser(self.SSH_PKEY)
orga = github.get_organization('nuxeo') # type: Organization
repo = orga.get_repo('nuxeo.com') # type: Repository
opened_pulls = [('/var/www/nuxeo.com/pr-%d.' % pull.number) + self.PREVIEW_DOMAIN for pull in repo.get_pulls()]
try:
proxy = ProxyCommand(('ssh -i %s -W 10.10.0.63:22 ' % self.SSH_PKEY) + self.BASTION_IP)
ssh.connect('10.10.0.63', username='******', sock=proxy, key_filename=self.SSH_PKEY)
_, stdout, _ = ssh.exec_command('ls -d /var/www/nuxeo.com/pr-*')
[ssh.exec_command('rm -rf ' + line.strip()) for line in stdout.readlines() if line.strip() not in opened_pulls]
ssh.close()
except SSHException, e:
logging.critical('Could work on remote: %s', e)
sys.exit(1)
class RemoteRunner(Runner):
def __init__(self, *args, **kwargs):
super(RemoteRunner, self).__init__(*args, **kwargs)
self.context
def start(self, command):
self.ssh_client = SSHClient()
self.ssh_client.load_system_host_keys()
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
self.ssh_client.connect(self.context.remote_runner.hostname, username=self.context.remote_runner.username)
self.ssh_channel = self.ssh_client.get_transport().open_session()
if self.using_pty:
self.ssh_channel.get_pty()
self.ssh_channel.exec_command(command)
def stdout_reader(self):
return self.ssh_channel.recv
def stderr_reader(self):
return self.ssh_channel.recv_stderr
def default_encoding(self):
return locale.getpreferredencoding(True)
def wait(self):
return self.ssh_channel.recv_exit_status()
def returncode(self):
return self.ssh_channel.recv_exit_status()
class Host:
def __init__(self, name):
self.name = name
self.ip = None
self.ssh_pkey = None
self._ssh = None
def __str__(self):
return self.name
def transport(self):
"""Returns the ssh transport for this host."""
if self._ssh == None:
self._ssh = SSHClient()
# import keys using paramikos method
self._ssh.load_system_host_keys()
# import openssh system host keys
host_keys_file = '/etc/ssh/ssh_known_hosts'
if os.path.exists(host_keys_file):
self._ssh.load_system_host_keys(host_keys_file)
# import saved host keys from/for multiapt
host_keys_file = os.getenv('HOME','/')+'/.ssh/known_hosts_multiapt'
if os.path.exists(host_keys_file):
self._ssh.load_host_keys(host_keys_file)
# now set our own filename for key save purposes
self._ssh._host_keys_filename = host_keys_file
# enable our own policy for host key problems
self._ssh.set_missing_host_key_policy(SSHAskHostKeyPolicy())
if Main.debug: print 'D: ssh.connect()'
self._ssh.connect(self.ip, username=config.remote_username, pkey=self.ssh_pkey)
return self._ssh
def connect(self, instance, ssh_user, ssh_ports, cmd, ssh_key_name):
"""
execute a command on instance with ssh and return if cmd param is not None
connect to ssh if cmd is None
:param instance:
:param ssh_user:
:param ssh_ports:
:param ssh_key_name:
:param cmd: execute this command if not None
:return:
"""
# get instance public ip
ssh_ip = instance.ip
# we need to find the ssh key
try:
key_file = open(os.path.join(os.path.expanduser(self._key_path), ssh_key_name), 'r')
except FileNotFoundError:
try:
key_file = open(os.path.join(os.path.expanduser(self._key_path), ssh_key_name + '.pem'), 'r')
except FileNotFoundError:
raise CourirSshException('private key %(key_name)s nor %(key_name)s.pem not found' % {
'key_name': ssh_key_name
})
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
logger.debug('connecting to %s with port %s and user %s', ssh_ip, ssh_ports[0], ssh_user)
mykey = RSAKey.from_private_key(key_file)
# we try with each ssh_port we have
for count, ssh_port in enumerate(ssh_ports):
try:
logger.debug(ssh_ip)
logger.debug(ssh_port)
client.connect(hostname=ssh_ip, port=int(ssh_port), username=ssh_user, pkey=mykey, timeout=4)
if cmd is None:
with NamedTemporaryFile(mode='w+') as tmp_key_file:
mykey.write_private_key(tmp_key_file, password=None)
tmp_key_file.flush()
cmd = 'ssh -i %s %s@%s -p %s' % (tmp_key_file.name, ssh_user, ssh_ip, ssh_port)
logger.debug(cmd)
os.system(cmd)
else:
stdin, stdout, stderr = client.exec_command(command=cmd)
out_str = stdout.read()
out_err = stderr.read().strip(' \t\n\r')
print(out_str)
if out_err != '':
print(out_err)
sys.exit(1)
except (ConnectionRefusedError, socket.timeout):
# we will try another tcp port
if count < len(ssh_ports):
continue
else:
raise CourirSshException('connection error')
def verify(self):
client = SSHClient()
client.set_missing_host_key_policy(self)
client.connect(
self.host,
username=self.username,
password=self.password,
)
def connect_ssh(dns_name, identity_file):
'''
'''
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
client.connect(dns_name, username='******', key_filename=identity_file)
return client
def timecheck(h): cmd = "date +%s" s = SSHClient() print "> %s" % (h,) s.set_missing_host_key_policy(AutoAddPolicy()) s.connect(hostname=h[1], username=h[0]) (_in, _out, _err) = s.exec_command( cmd, bufsize=4096) print "< %s" % (h,) return (h,_out.read())
def timecheck(h): cmd = "sudo sysctl -w net.core.somaxconn=16000;" s = SSHClient() print "> %s" % (h,) s.set_missing_host_key_policy(AutoAddPolicy()) s.connect(hostname=h[1], username=h[0])#, password='') (_in, _out, _err) = s.exec_command( cmd, bufsize=4096) print "< %s" % (h,) return (h,_out.read())
def ssh_client(*args, **kwargs):
client = SSHClient()
try:
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(*args, **kwargs)
except Exception, e:
logger.warn("Could not connect to remote host.")
logger.exception(e)
raise e
class Context(object):
def __init__(self, hostname, username, password, xml=False):
self.hostname = hostname
self.username = username
self.password = password
self.xml = xml
self.ssh_client = SSHClient()
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
self.ssh_client.connect(self.hostname, username=self.username, password=self.password)
def _exec_ssh_cmd(self, cmdline):
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
client.connect(self.droplet_ip, username='******',
key_filename=self.ssh_key_path)
stdin, stdout, stderr = client.exec_command(cmdline)
for line in stdout:
logging.info(line)
for line in stderr:
logging.info(line)
def __create_ssh_client(self):
sshclient = SSHClient()
sshclient.set_missing_host_key_policy(paramiko.AutoAddPolicy())
for i in range(1, MAX_WAIT_FOR_LOOPS):
try:
sshclient.connect(hostname=self.hostname, port=22,
username=self.login_user, password=self.login_password, timeout=SLEEP_TIMEOUT)
return sshclient
except socket.error: # no route to host is expected here at first
time.sleep(SLEEP_TIMEOUT)
def install_kinto_remotely(id_alwaysdata, credentials, ssh_host, prefixed_username, status_handler):
logs = StringIO()
# SSH Login
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(ssh_host, username=prefixed_username, password=credentials[1], look_for_keys=False)
# Install pip
retry = 30
error = None
while retry > 0:
try:
stdin, stdout, stderr = ssh.exec_command(
"PYTHONPATH=~/.local/ easy_install-2.6 --install-dir=~/.local -U pip"
)
retry = 0
except ssh_exception.AuthenticationException as e:
error = e
sleep(5)
retry -= 1
if retry == 0 and error is not None:
logs.write(error)
logs.write(stdout.read())
logs.write(stderr.read())
status_handler.ssh_logs = logs
# Install virtualenv
stdin, stdout, stderr = ssh.exec_command(
"PYTHONPATH=~/.local/ ~/.local/pip install --user --no-binary --upgrade "
"setuptools virtualenv virtualenvwrapper"
)
logs.write(stdout.read())
logs.write(stderr.read())
status_handler.ssh_logs = logs
# Create virtualenv
stdin, stdout, stderr = ssh.exec_command("~/.local/bin/virtualenv kinto/venv/ --python=python2.7")
logs.write(stdout.read())
logs.write(stderr.read())
status_handler.ssh_logs = logs
# Install Kinto in the virtualenv
stdin, stdout, stderr = ssh.exec_command("kinto/venv/bin/pip install kinto[postgresql] kinto-attachment flup")
logs.write(stdout.read())
logs.write(stderr.read())
status_handler.ssh_logs = logs
# Run kinto migration to setup the database.
stdin, stdout, stderr = ssh.exec_command("kinto/venv/bin/kinto --ini kinto/kinto.ini migrate")
logs.write(stdout.read())
logs.write(stderr.read())
status_handler.ssh_logs = logs
ssh.close()
def run(self, project):
if not self.vm.start_VM():
return -1, ''
if not os.path.exists(os.path.join(project.tempdir, project.target)):
raise FileNotFoundError('Error: Executable file has not been created!')
copy_to_vm = [os.path.join(project.tempdir, project.target)]
copy_from_vm = ['CUnitAutomated-Results.xml']
print('Connecting to remote machine...')
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
client.load_system_host_keys()
client.connect(self.host, username=self.username, password=self.password, timeout=10)
return_code = 0
data = ''
with client.open_sftp() as sftp:
try:
self.rmtree(sftp, self.remote_path)
except FileNotFoundError:
pass
try:
sftp.mkdir(self.remote_path)
except OSError:
pass
for f in copy_to_vm:
remote_file = os.path.join(self.remote_path, os.path.basename(f))
sftp.put(f, remote_file)
sftp.chmod(remote_file, 0o777)
stdin, stdout, stderr = client.exec_command('cd {}; timeout {}s {}'.format(self.remote_path, self.timeout, remote_file))
return_code = stdout.channel.recv_exit_status()
print('[Remote] Error code: {}'.format(return_code))
stdout_string = '[Remote] ' + ''.join(stdout)
if stdout_string:
print('[Remote] STDOUT:')
print(stdout_string)
stderr_string = '[Remote] ' + ''.join(stderr)
if stderr_string:
print('[Remote] STDERR:')
print(stderr_string)
for f in copy_from_vm:
# get all result files
remote_file = os.path.join(self.remote_path, os.path.basename(f))
try:
with tempfile.TemporaryFile() as local_file:
sftp.getfo(remote_file, local_file)
local_file.seek(0)
data = local_file.read()
except FileNotFoundError:
print('Remote file not found!')
# delete all files in home directory
self.rmtree(sftp, self.remote_path)
client.close()
if self.shutdown_vm_after:
self.vm.stop_VM()
return return_code, data
def _exec_ssh_cmd(self, cmdline):
pkey_buf = StringIO(self.state.pkey)
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
client.connect(self.droplet_ip, username='******',
pkey=RSAKey.from_private_key(pkey_buf))
stdin, stdout, stderr = client.exec_command(cmdline)
for line in stdout:
logging.info(line)
for line in stderr:
logging.info(line)
def _connect(self):
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(AutoAddPolicy())
try:
client.connect(hostname=self.host, port=self.port, username=self.user, password=self.passwd, timeout=3)
self.closed = False
return client
except AuthenticationException, e:
return e
def newSshClient(hostname, username="******", localPrivKey=None, timeout=None):
from paramiko.client import SSHClient, AutoAddPolicy
localPrivKey = localPrivKey or loadRsaPrivKey()
missingHostKeyPolicy = AutoAddPolicy()
ssh = SSHClient()
ssh.set_missing_host_key_policy(missingHostKeyPolicy)
ssh.connect(hostname, username=username, pkey=localPrivKey, timeout=timeout)
return ssh
def createClient(host, username=None, pkeyPath=None):
"""
Creates an SSH client object that can be used to perform SSH-related
operations
"""
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
pkey = RSAKey.from_private_key_file(os.path.expanduser(pkeyPath)) if pkeyPath else None
client.connect(host, username=username, pkey=pkey)
return client
def test_credentials(hostname, username, password, port): """ Returns True if the credentials work """ client = SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) try: client.connect(hostname, username=username, password=password, port=port) return True except Exception as e: print(e) return False
class datastore_sftp(datastore):
""" collect of independent functions, not really a Class """
def __init__(self):
# Remote path for saving all resources
self.base_folder = settings.SFTP_DATASTORE_REMOTEBASEFOLDER
# Local base folder for saving temporary files before upload
self.tmp_folder = settings.SFTP_DATASTORE_LOCALTMPFOLDER
# url for donwloading resources
self.public_base_url = settings.SFTP_BASE_URL
self.buckets = []
self.connection = None
self.ssh_client = SSHClient()
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
self.ssh_client.load_system_host_keys()
self.sftp = None
def connect(self):
""" don't use at INIT because it hangs all application"""
logger = logging.getLogger(__name__)
logger.error('Connecting SFTP %s:%s (%s, %s)' %(
settings.SFTP_DATASTORE_HOSTNAME,
settings.SFTP_DATASTORE_PORT,
settings.SFTP_DATASTORE_USER,
settings.SFTP_DATASTORE_PASSWORD)
)
# TODO: Remove
con = sftp.Connection(
host=settings.SFTP_DATASTORE_HOSTNAME,
port=settings.SFTP_DATASTORE_PORT,
username=settings.SFTP_DATASTORE_USER,
password=settings.SFTP_DATASTORE_PASSWORD,
log=True
)
self.connection = con
#
self.ssh_client.connect(
settings.SFTP_DATASTORE_HOSTNAME,
port=settings.SFTP_DATASTORE_PORT,
username=settings.SFTP_DATASTORE_USER,
password=settings.SFTP_DATASTORE_PASSWORD
)
self.sftp = self.ssh_client.open_sftp()
# list all buckets (folders)
try:
self.buckets = self.sftp.listdir(path=self.base_folder)
logger.error('Buckets: %s' %str(self.buckets))
except Exception, e:
logger.error('Error Connecting SFTP %s' % str(e))
self.sftp.close()
class Shared(object):
def __init__(self, args):
self.args = args
self.client = Client(
'http://{0}{1}'.format(self.args.address, ':{0}'.format(self.args.port) if self.args.port else ''),
'/api/v2.0/',
username=self.args.username,
password=self.args.password,
)
self.ssh_client = SSHClient()
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
self.ssh_client.connect(args.address, port=args.sshport, username=args.username, password=args.password)
def test_keyfile(hostname, username, key_filename, port=22, timeout=15):
""" Tests if a key file works
"""
client = SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
client.connect(
hostname, port=port, username=username,
key_filename=key_filename, timeout=10)
return True
except Exception as e:
print(e)
return False
def recvr(h): s = SSHClient() print "> %s" % (h,) s.set_missing_host_key_policy(AutoAddPolicy()) s.connect(hostname=h[1], username=h[0]) (_in, _out, _err) = s.exec_command(client_cmd, bufsize=4096) _in.write(bytecode) _in.flush() print "< %s" % (h,) err = _err.read() if(err): print err return (h,_out.read())
def executar_comando_remoto(com):
hosts = ["192.168.0.2"]
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
for h in hosts:
try:
ssh.connect(h)
stdin,stdout,stderr = ssh.exec_command(com)
if stderr.channel.recv_exit_status() != 0:
print stderr.read()
else:
print stdout.read()
except Exception as e:
print "Nao conseguiu conectar ao servidor %s"%e
class SshOps:
def __init__(self):
self.client = SSHClient()
self.client.load_system_host_keys()
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
config = ConfigParser.ConfigParser()
config.read("/home/forlinux/www/4803-Python/ProjetoDexter/config.cfg")
self.client.connect(config.get("docker","address"))
def runCommand(self,command):
stdin,stdout,stderr = self.client.exec_command(command)
if stderr.channel.recv_exit_status() != 0:
return {"status":1,"message":stderr.read()}
else:
return {"status":0,"message":stdout.read()}
def configure_cfme(ipaddr, ssh_username, ssh_password, region, db_password):
cmd = "appliance_console_cli --region %s --internal --force-key -p %s" % (region, db_password)
client = SSHClient()
try:
client.set_missing_host_key_policy(AutoAddPolicy())
client.connect(ipaddr, username=ssh_username, password=ssh_password, allow_agent=False)
print "Will run below command on host: %s" % (ipaddr)
print cmd
stdin, stdout, stderr = client.exec_command(cmd)
status = stdout.channel.recv_exit_status()
out = stdout.readlines()
err = stderr.readlines()
return status, out, err
finally:
client.close()
def stop_instance(instance):
try:
stop_script_location = instance.tags['stop_script']
except KeyError:
stop_script_location = '~/shutdown.sh'
client = SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.load_system_host_keys()
client.connect(instance.ip_address, username="******")
stdin, stdout, stderr = client.exec_command('%s "%s" "%s"' % (stop_script_location, app.config["API_KEY"], app.config["MY_URL"] + "/api/v1/stats"))
try:
for s in [stdin, stdout, stderr]:
app.logger.info(s.read())
except IOError:
pass
client.close()
def executar_comando(servidor,senha):
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect("localhost",username="******",password=senha)
stdin, stdout, stderr = ssh.exec_command("ls -la")
if stderr.channel_recv_exit_status() != 0:
print stderr.read()
else:
print stdout.read()
except Exception as e:
print "A conexao falhou"
def _ssh_connect(self) -> "SSHClient":
parent = self.parent
client = SSHClient()
client.set_missing_host_key_policy(
RejectPolicy if parent.host_key_checking else AutoAddPolicy)
try:
LOG.debug(
f"connecting, host={parent.host}, user={parent.user}, key={parent.private_key_file}"
)
client.connect(
parent.host,
username=parent.user,
key_filename=_expand_path(parent.private_key_file),
timeout=parent.timeout,
)
except NoValidConnectionsError as exc:
raise BindingConnectionError(
binding_name=parent.binding.name) from exc
except SSHException as exc:
raise BindingConnectionError(
binding_name=parent.binding.name) from exc
return client
def run_command(self, command_str):
""" Uses paramiko ssh client to execute `command_str` on this remote Instance.
Args:
command_str (str): The remote shell command to execute.
"""
ssh_client = SSHClient()
ssh_client.set_missing_host_key_policy(AutoAddPolicy)
print("Connecting to %s:%i " % (self.ssh_host, self.ssh_port))
try:
ssh_client.connect(self.ssh_host,
port=int(self.ssh_port),
username='******',
key_filename=self.client._get_ssh_key_file())
print("Running command '%s'" % command_str)
stdin, stdout, stderr = ssh_client.exec_command(command_str)
print(stdout.read().decode('utf-8'))
print(stderr.read().decode('utf-8'))
# except NoValidConnectionsError as err:
# raise InstanceError(self.id, err.errors)
finally:
ssh_client.close()
def check(dict):
for key in dict['client']:
clients.append(dict['client'][key])
for client in clients:
mfsClientVM = SSHClient()
hostname = client
mfsClientVM.set_missing_host_key_policy(paramiko.AutoAddPolicy())
mfsClientVM.load_system_host_keys()
#.ssh/cs6620Key101.pem
# mfsClientVM.connect(hostname=hostname, username='******', key_filename='/home/centos/cs6620Key101.pem')
mfsClientVM.connect(hostname=hostname,
username='******',
key_filename='cs6620Key101.pem')
# if client == '10.0.0.241':
# print("The client ip/name is: cl2")
# else:
# print("The client ip/name is: ", client)
print("Test result:")
stdin, stdout, stderr = mfsClientVM.exec_command(
'cd /mnt/mfs/test7; grep -c "B" testfile.txt')
outlines = stdout.readlines()
stdin.close()
resp = ''.join(outlines)
resp = int(resp)
print(resp)
# resultList.append(resp)
# for i in range(0,len(resultList) - 1):
if resp != 0:
print("Test failure")
return
print("Test7 passed")
return
class MySshClient():
def __init__(self):
self.ssh_client = SSHClient()
# 此函数用于输入用户名密码登录主机
def ssh_login(self, host_ip, username, password):
try:
# 设置允许连接known_hosts文件中的主机（默认连接不在known_hosts文件中的主机会拒绝连接抛出SSHException）
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
self.ssh_client.connect(host_ip,
port=22,
username=username,
password=password)
except AuthenticationException:
logging.warning('username or password error')
return 1001
except NoValidConnectionsError:
logging.warning('connect time out')
return 1002
except:
logging.warning('unknow error')
print("Unexpected error:", sys.exc_info()[0])
return 1003
return 1000
# 此函数用于执行command参数中的命令并打印命令执行结果
def execute_some_command(self, command):
stdin, stdout, stderr = self.ssh_client.exec_command(command)
str1 = stdout.read().decode()
f1 = str1[0:5]
print(f1)
f = open(f1, 'r+')
print(f.read())
# 此函数用于退出登录
def ssh_logout(self):
logging.warning('will exit host')
self.ssh_client.close()
def check_servers():
cursor.execute('SELECT * FROM appconfig')
app_conf = cursor.fetchone()
if app_conf['monitoring']:
cursor.execute('SELECT * FROM server')
servers = cursor.fetchall()
for server in servers:
ssh_client = SSHClient()
ssh_client.set_missing_host_key_policy(AutoAddPolicy())
ssh_client.load_system_host_keys()
print "********** Checking server {0} **********".format(
server['hostname'])
print "Making SSH Connection to server {0}".format(
server['hostname'])
ssh_client.connect(server['hostname'], port=22, username='******')
print "SSH connection is successful."
get_server_time(ssh_client)
check_data_generated_on_server(ssh_client)
check_database_populated(ssh_client)
check_data_could_fetched(ssh_client)
check_influxdb(server['hostname'])
print "-" * 50, '\n'
def check(dict):
for key in dict['client']:
clients.append(dict['client'][key])
for client in clients:
mfsClientVM = SSHClient()
hostname = client
mfsClientVM.set_missing_host_key_policy(paramiko.AutoAddPolicy())
mfsClientVM.load_system_host_keys()
#.ssh/cs6620Key101.pem
# mfsClientVM.connect(hostname=hostname, username='******', key_filename='/home/centos/cs6620Key101.pem')
mfsClientVM.connect(hostname=hostname,
username='******',
key_filename='./ssh_scp/real_key.pem')
# if client == '10.0.0.241':
# print("The client ip/name is: cl2")
# else:
# print("The client ip/name is: ", client)
print("File name is:")
stdin, stdout, stderr = mfsClientVM.exec_command(
'cd ..; cd ..; cd mnt; cd mfs; ls')
outlines = stdout.readlines()
stdin.close()
resp = ''.join(outlines)
print(resp)
resultList.append(resp)
for i in range(1, len(resultList)):
if resultList[i - 1] != resultList[i]:
print("Test failure")
return
print("Test success")
return
class RemoteSession():
def __init__(self, ip, username=REMOTE_USER):
self.username = username
self.client = SSHClient()
self.client.set_missing_host_key_policy(IgnoreHostKeyPolicy)
self.client.connect(ip, username=self.username)
self.sftp = self.client.open_sftp()
def execute(self, command):
stdin, stdout, stderr = self.client.exec_command(command)
rc = stdout.channel.recv_exit_status()
out = stdout.read().decode('utf-8').splitlines()
err = stderr.read().decode('utf-8').splitlines()
if rc != 0 and not quiet:
log = logging.getLogger('TPCH')
log.error("ssh command returned %d" % rc)
log.error("ssh -l %s %s %s" % (self.username, ip, command))
print(command)
for line in out:
print(line)
for line in err:
print(line)
print()
return
def download(self, src, dst):
self.sftp.get(src, dst)
return
def close(self):
self.client.close()
def _inner(ssh_env, no_cache=False, *args, **kwargs):
_rename_kwargs(ssh_env)
key = (ssh_env['hostname'],
ssh_env.get('port', 22),
ssh_env['username'])
with connect_lock:
if key not in cache or no_cache:
client = SSHClient()
client.set_missing_host_key_policy(WarningPolicy())
client.connect(**ssh_env)
if no_cache:
return f(*args, **kwargs)
proxy = NetstringMultiCtxProxy()
transport = client.get_transport()
port = transport.request_port_forward('127.0.0.1', 0,
proxy.handle)
proxy.proxy_url = 'netstring://127.0.0.1:{0}'.format(port)
cache[key] = (client, proxy)
kwargs['client'], kwargs['proxy'] = cache[key]
try:
return f(*args, **kwargs)
finally:
if no_cache:
client.close()
def handle(self, *labels, **options):
init_logging(logger, 3)
try:
# Make sure the directory and the file exist
dirname = os.path.dirname(settings.SSH_KNOWN_HOSTS)
os.makedirs(dirname, exist_ok=True)
open(settings.SSH_KNOWN_HOSTS, 'a')
client = SSHClient()
client.load_host_keys(settings.SSH_KNOWN_HOSTS)
client.set_missing_host_key_policy(AutoAddPolicy())
client.set_log_channel('paramiko')
# Filter Paramiko message to only show host key messages
logging.getLogger('paramiko').addFilter(ParamikoLogFilter())
hostnames = [
settings.V4_HOST,
settings.V6_HOST,
settings.NAT64_HOST
]
for hostname in hostnames:
logger.info("Connecting to {}".format(hostname))
client.connect(hostname, key_filename=settings.SSH_PRIVATE_KEY, allow_agent=False, look_for_keys=False)
stdin, stdout, stderr = client.exec_command('phantomjs --version')
stdout_lines = stdout.readlines()
if stdout_lines:
logger.info("PhantomJS version: {}".format(stdout_lines[0].strip()))
else:
logger.error(''.join([line.strip() for line in stderr.readlines()][:1]))
client.close()
except Exception as e:
logger.critical(str(e))
def verify_moosefs_drive_content(self, remote_host_ip: str) -> list:
try:
result_list = list()
mfsClientVM = SSHClient()
mfsClientVM.set_missing_host_key_policy(paramiko.AutoAddPolicy())
mfsClientVM.load_system_host_keys()
# mfsClientVM.connect(hostname=remote_host_ip,
# username=self.remote_host_username,
# key_filename='cs6620Key101.pem')
mfsClientVM.connect(hostname=remote_host_ip,
username=self.remote_host_username)
print("Verifying file content on VM with IP: " + remote_host_ip)
# Count Bs in file on client VM
stdin, stdout, stderr = mfsClientVM.exec_command(
'cd /mnt/mfs/test7; grep -c "B" testfile.txt')
outlines = stdout.readlines()
stdin.close()
count = ''.join(outlines)
# count = int(count)
result_list.append(count)
print('File contains ' + count + ' Bs')
mfsClientVM.close()
return result_list
except Exception as e:
print(
"Something went wrong while verifying the moosefs drive content"
)
print("Error Details: " + str(e))
return None
class Client:
context = {}
def __init__(self, hostname, configpath=None, dry_run=False):
ssh_config = SSHConfig()
if not hostname:
print(red('"hostname" must be defined'))
sys.exit(1)
parsed = self.parse_host(hostname)
hostname = parsed.get('hostname')
username = parsed.get('username')
if configpath:
if not isinstance(configpath, (list, tuple)):
configpath = [configpath]
for path in configpath:
self._load_config(path, hostname)
with (Path.home() / '.ssh/config').open() as fd:
ssh_config.parse(fd)
ssh_config = ssh_config.lookup(hostname)
self.dry_run = dry_run
self.hostname = config.hostname or ssh_config['hostname']
self.username = (username or config.username
or ssh_config.get('user', getuser()))
self.formatter = Formatter()
self.key_filenames = []
if config.key_filename:
self.key_filenames.append(config.key_filename)
if 'identityfile' in ssh_config:
self.key_filenames.extend(ssh_config['identityfile'])
self.sudo = ''
self.cd = None
self.screen = None
self.env = {}
self._sftp = None
self.proxy_command = ssh_config.get('proxycommand',
config.proxy_command)
self.open()
def open(self):
self._client = SSHClient()
self._client.load_system_host_keys()
self._client.set_missing_host_key_policy(WarningPolicy())
print(f'Connecting to {self.username}@{self.hostname}')
if self.proxy_command:
print('ProxyCommand:', self.proxy_command)
sock = (paramiko.ProxyCommand(self.proxy_command)
if self.proxy_command else None)
try:
self._client.connect(hostname=self.hostname,
username=self.username,
sock=sock,
key_filename=self.key_filenames)
except paramiko.ssh_exception.BadHostKeyException:
sys.exit('Connection error: bad host key')
self._transport = self._client.get_transport()
def close(self):
print(f'\nDisconnecting from {self.username}@{self.hostname}')
self._client.close()
def _load_config(self, path, hostname):
with Path(path).open() as fd:
conf = yaml.load(fd)
if hostname in conf:
conf.update(conf[hostname])
config.update(conf)
def parse_host(self, host_string):
user_hostport = host_string.rsplit('@', 1)
hostport = user_hostport.pop()
user = user_hostport[0] if user_hostport and user_hostport[0] else None
# IPv6: can't reliably tell where addr ends and port begins, so don't
# try (and don't bother adding special syntax either, user should avoid
# this situation by using port=).
if hostport.count(':') > 1:
host = hostport
port = None
# IPv4: can split on ':' reliably.
else:
host_port = hostport.rsplit(':', 1)
host = host_port.pop(0) or None
port = host_port[0] if host_port and host_port[0] else None
if port is not None:
port = int(port)
return {'username': user, 'hostname': host, 'port': port}
def _build_command(self, cmd, **kwargs):
prefix = ''
if self.cd:
cmd = f'cd {self.cd}; {cmd}'
if self.env:
prefix = ' '.join(f'{k}={v}' for k, v in self.env.items())
if self.sudo:
prefix = f'{self.sudo} {prefix}'
cmd = self.format(f"{prefix} sh -c $'{cmd}'")
if self.screen:
cmd = f'screen -UD -RR -S {self.screen} {cmd}'
return cmd.strip().replace(' ', ' ')
def _call_command(self, cmd, **kwargs):
channel = self._transport.open_session()
try:
size = os.get_terminal_size()
except IOError:
channel.get_pty() # Fails when ran from pytest.
else:
channel.get_pty(width=size.columns, height=size.lines)
channel.exec_command(cmd)
channel.setblocking(False) # Allow to read from empty buffer.
stdout = channel.makefile('r', -1)
stderr = channel.makefile_stderr('r', -1)
proxy_stdout = b''
buf = b''
while True:
while sys.stdin in select.select([sys.stdin], [], [], 0)[0]:
# TODO compute bytes_to_read like in invoke?
data = sys.stdin.read(1)
if data:
channel.sendall(data)
else:
break
if not channel.recv_ready():
if buf: # We may have read some buffer yet, let's output it.
sys.stdout.write(buf.decode())
sys.stdout.flush()
buf = b''
if channel.exit_status_ready():
break
continue
try:
data = stdout.read(1)
except Exception: # Not sure how to catch socket.timeout properly.
pass
else:
proxy_stdout += data
buf += data
if data == b'\n':
sys.stdout.write(buf.decode())
sys.stdout.flush()
buf = b''
continue
time.sleep(paramiko.io_sleep)
channel.setblocking(True) # Make sure we now wait for stderr.
ret = Status(proxy_stdout.decode(),
stderr.read().decode().strip(),
channel.recv_exit_status())
channel.close()
if ret.code:
self.exit(ret.stderr, ret.code)
return ret
def exit(self, msg, code=1):
print(red(msg))
sys.exit(code)
def __call__(self, cmd, **kwargs):
cmd = self._build_command(cmd, **kwargs)
print(gray(cmd))
if self.dry_run:
return Status('¡DRY RUN!', '¡DRY RUN!', 0)
with character_buffered():
return self._call_command(cmd, **kwargs)
def format(self, tpl):
try:
return self.formatter.vformat(tpl, None, self.context)
except KeyError as e:
print(red(f'Missing key {e}'))
sys.exit(1)
@property
def sftp(self):
if not self._sftp:
self._sftp = self._client.open_sftp()
return self._sftp
# -*- coding: utf-8 -*-
"Not used"
import os
import time
from paramiko.client import SSHClient, AutoAddPolicy, socket
hosts_filename = os.path.expanduser("~/.ssh/paramiko_known_hosts")
#print hosts_filename
client = SSHClient()
if os.path.isfile(hosts_filename):
client.load_host_keys(hosts_filename) #'~/.ssh/known_hosts'
client.set_missing_host_key_policy(AutoAddPolicy())
print client.get_host_keys()
client.connect('localhost', 2200, username='******', password='******')
client.save_host_keys(hosts_filename)
#print client.get_host_keys()
channel = client.invoke_shell()
channel.settimeout(0)
#time.sleep(1)
def datas(data=''):
while True:
try:
res = channel.recv(10000)
if len(res) == 0:
exit(0)
data += res
cmds = data.split('\n')
def start_server_over_ssh(self):
try:
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(WarningPolicy())
client.connect(
self.ssh_host,
port=self.ssh_port,
username=self.ssh_user,
password=self.text,
)
transport = client.get_transport()
ip, _ = transport.getpeername()
if ip:
self.update_ip_linedt_signal.emit(ip)
logger.info(f"IP for {self.ssh_host} detected as {ip}.")
ws_name = self.run_config["workspace_name"]
server_port = self.run_config["server_port"]
# TODO Check if the server port is already in use
logger.info(
f"Checking if server port: {server_port} at ip: {ip} is already in use."
)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
result = sock.connect_ex((ip, int(server_port)))
if result == 0:
logger.error(f"Port {server_port} is already open.")
self.button_message_signal.emit([
f"Port: {server_port} at ip: {ip} is already in use!",
"maroon", 3
])
self.error_signal.emit()
sock.close()
client.close()
return
else:
logger.info(f"Port {server_port} is not open.")
sock.close()
cuda_command = "module load cuda/10.1\n"
command = (
"/dls_sw/apps/SuRVoS2/s2_conda/bin/python -u "
"/dls/science/groups/das/SuRVoS/s2/s2_dec/SuRVoS2/survos.py "
f"start_server {ws_name} {server_port} > {date.today()}_survos2.log &\n"
)
logger.info(f"Running command on remote machine: {command}")
session = transport.open_session()
session.setblocking(0) # Set to non-blocking mode
session.get_pty()
session.invoke_shell()
# Send commands
session.send(cuda_command)
session.send(command)
# Loop for 15 seconds
self.button_message_signal.emit([
f"Starting server on {self.ssh_host}. Please Wait!", "navy", 14
])
start = time.time()
while time.time() - start < 15:
if session.recv_ready():
data = session.recv(512)
print(data.decode(), flush=True)
time.sleep(1) # Yield CPU so we don't take up 100% usage...
self.finished.emit()
except AuthenticationException:
logger.error("SSH Authentication failed!")
self.button_message_signal.emit(
["Incorrect Password!", "maroon", 3])
self.error_signal.emit()
class MySshClient():
def __init__(self):
self.config = configparser.ConfigParser()
self.filename = os.path.join(os.path.dirname(__file__),
args.filename).replace("\\", "/")
self.fileyaml = os.path.join(os.path.dirname(__file__),
args.fileyaml).replace("\\", "/")
self.f = open(self.fileyaml)
self.y = yaml.load(self.f, Loader=yaml.FullLoader)
self.config.read(self.filename)
self.ssh_client = SSHClient()
self.target = self.y['pushgateway']['targets'][0]
self.shell = None
self.registry = CollectorRegistry()
def ssh_login(self):
try:
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
self.ssh_client.connect(
hostname=self.config.get('ssh', 'host'),
port=self.config.get('ssh', 'port'),
username=self.config.get('ssh', 'username'),
password=self.config.get('ssh', 'password'))
except AuthenticationException:
logging.warning('username or password error')
return False
except NoValidConnectionsError:
logging.warning('connect time out')
return False
except:
logging.warning('unknow error')
return False
return True
def execute_some_command(self, command):
try:
stdin, stdout, stderr = self.ssh_client.exec_command(command)
res, err = stdout.readlines(), stderr.readlines()
result = res if res else err
li_out = []
tem_dict = {}
flag = 0
for line in result:
if 'obdfilter' in line:
flag += 1
if flag == 1:
try:
res = re.search("((?:[0-9]{1,3}\.){3}[0-9]{1,3})",
line)
if res:
server_IP = res.group(1)
except Exception as e:
logging.error(str(e))
if flag > 1:
tem_dict = {}
li_out.append(tem_dict)
store = line.split('.')[1]
tem_dict['store'] = store
res = re.search("((?:[0-9]{1,3}\.){3}[0-9]{1,3})",
line)
if res:
client_IP = res.group(1)
tem_dict['client_IP'] = client_IP
store = line.split('.')[1]
tem_dict['store'] = store
tem_dict['server_IP'] = server_IP
elif 'snapshot_time' in line:
snapshot_time = line.split(' ')[-1].strip('\r\n')
tem_dict['snapshot_time'] = snapshot_time
elif 'read_bytes' in line:
read_bytes = line.split(' ')[-1].split(' ')[-1].strip(
'\r\n')
tem_dict['read_bytes'] = read_bytes
elif 'write_bytes' in line:
write_bytes = line.split(' ')[-1].split(' ')[-1].strip(
'\r\n')
tem_dict['write_bytes'] = write_bytes
else:
tem_dict = {}
return li_out
except Exception as e:
logging.error(str(e))
def push2gateway(self):
self.read_metric = Gauge('read_push',
'status', [
'server_IP', 'store_name', 'client_IP',
'snapshot_time', 'read_bytes'
],
registry=self.registry)
self.write_metric = Gauge('write_push',
'status', [
'server_IP', 'store_name', 'client_IP',
'snapshot_time', 'write_bytes'
],
registry=self.registry)
dict_list = self.execute_some_command(command)
try:
for dict_tmp in dict_list:
server_IP = dict_tmp['server_IP']
store = dict_tmp['store']
client_IP = dict_tmp['client_IP']
snapshot_time = dict_tmp['snapshot_time']
read_bytes = dict_tmp['read_bytes']
write_bytes = dict_tmp['write_bytes']
self.read_metric.labels(server_IP, store, client_IP,
snapshot_time,
read_bytes).set(read_bytes)
self.write_metric.labels(server_IP, store, client_IP,
snapshot_time,
write_bytes).set(write_bytes)
pushadd_to_gateway(self.target,
job='gene_pushgateway',
registry=self.registry,
timeout=200)
except Exception as e:
logging.error(str(e))
def ssh_logout(self):
logging.warning('will exit host')
self.ssh_client.close()
#!/usr/bin/python2.7
# -*- coding: UTF-8 -*-
from paramiko.client import SSHClient
import paramiko
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect("172.16.65.100", username='', password='')
stdin, stdout, stderr = client.exec_command("lss -la")
if stderr.channel.recv_exit_status() != 0:
print stderr.channel.recv_exit_status()
print stderr.read()
else:
print stdout.read()
class RemoteClient(object):
"""Remote Client is a wrapper over SSHClient with utility functions.
Args:
host (string): The hostname of the server to connect. It can be an IP
address of the server also.
user (string, optional): The user to connect to the remote server. It
defaults to root
Attributes:
host (string): The hostname passed in as a the argument
user (string): The user to connect as to the remote server
client (:class:`paramiko.client.SSHClient`): The SSHClient object used
for all the communications with the remote server.
sftpclient (:class:`paramiko.sftp_client.SFTPClient`): The SFTP object
for all the file transfer operations over the SSH.
"""
def __init__(self, host, user='******'):
self.host = host
self.user = user
self.client = SSHClient()
self.sftpclient = None
self.client.set_missing_host_key_policy(AutoAddPolicy())
self.client.load_system_host_keys()
def startup(self):
"""Function that starts SSH connection and makes client available for
carrying out the functions.
"""
self.client.connect(self.host, port=22, username=self.user)
self.sftpclient = self.client.open_sftp()
def download(self, remote, local):
"""Downloads a file from remote server to the local system.
Args:
remote (string): location of the file in remote server
local (string): path where the file should be saved
"""
if not self.sftpclient:
raise ClientNotSetupException(
'Cannot download file. Client not initialized')
try:
self.sftpclient.get(remote, local)
except OSError:
return "Error: Local file %s doesn't exist." % local
except IOError:
return "Error: Remote location %s doesn't exist." % remote
finally:
return "Download successful. File at: {0}".format(local)
def upload(self, local, remote):
"""Uploads the file from local location to remote server.
Args:
local (string): path of the local file to upload
remote (string): location on remote server to put the file
"""
if not self.sftpclient:
raise ClientNotSetupException(
'Cannot upload file. Client not initialized')
try:
self.sftpclient.put(local, remote)
except OSError:
return "Error: Local file %s doesn't exist." % local
except IOError:
return "Error: Remote location %s doesn't exist." % remote
finally:
return "Upload successful. File at: {0}".format(remote)
def exists(self, filepath):
"""Returns whether a file exists or not in the remote server.
Args:
filepath (string): path to the file to check for existance
Returns:
True if it exists, False if it doesn't
"""
if not self.client:
raise ClientNotSetupException(
'Cannot run procedure. Client not initialized')
cin, cout, cerr = self.client.exec_command('stat {0}'.format(filepath))
if len(cout.read()) > 5:
return True
elif len(cerr.read()) > 5:
return False
def run(self, command):
"""Run a command in the remote server.
Args:
command (string): the command to be run on the remote server
Returns:
tuple of three strings containing text from stdin, stdout an stderr
"""
if not self.client:
raise ClientNotSetupException(
'Cannot run procedure. Client not initialized')
buffers = self.client.exec_command(command)
output = []
for buf in buffers:
try:
output.append(buf.read())
except IOError:
output.append('')
return tuple(output)
def close(self):
"""Close the SSH Connection
"""
self.client.close()
class SSH:
def __init__(self,
hostname,
port=SSH_PORT,
username='******',
pkey=None,
password=None,
connect_timeout=10):
if pkey is None and password is None:
raise Exception(
'public key and password must have one is not None')
self.client = None
self.arguments = {
'hostname':
hostname,
'port':
port,
'username':
username,
'password':
password,
'pkey':
RSAKey.from_private_key(StringIO(pkey))
if isinstance(pkey, str) else pkey,
'timeout':
connect_timeout,
}
@staticmethod
def generate_key():
key_obj = StringIO()
key = RSAKey.generate(2048)
key.write_private_key(key_obj)
return key_obj.getvalue(), 'ssh-rsa ' + key.get_base64()
# def add_public_key(self, public_key):
# command = f'mkdir -p -m 700 ~/.ssh && \
# echo {public_key!r} >> ~/.ssh/authorized_keys && \
# chmod 600 ~/.ssh/authorized_keys'
# code, out = self.exec_command(command)
# if code != 0:
# raise Exception(out)
def ping(self):
with self:
return True
def get_client(self):
if self.client is not None:
return self.client
self.client = SSHClient()
self.client.set_missing_host_key_policy(AutoAddPolicy)
self.client.connect(**self.arguments)
return self.client
# def put_file(self, local_path, remote_path):
# with self as cli:
# sftp = cli.open_sftp()
# sftp.put(local_path, remote_path)
# sftp.close()
#
# def exec_command(self, command, timeout=1800, environment=None):
# command = 'set -e\n' + command
# with self as cli:
# chan = cli.get_transport().open_session()
# chan.settimeout(timeout)
# chan.set_combine_stderr(True)
# if environment:
# str_env = ' '.join(f"{k}='{v}'" for k, v in environment.items())
# command = f'export {str_env} && {command}'
# chan.exec_command(command)
# out = chan.makefile("r", -1)
# return chan.recv_exit_status(), out.read()
#
# def exec_command_with_stream(self, command, timeout=1800, environment=None):
# command = 'set -e\n' + command
# with self as cli:
# chan = cli.get_transport().open_session()
# chan.settimeout(timeout)
# chan.set_combine_stderr(True)
# if environment:
# str_env = ' '.join(f"{k}='{v}'" for k, v in environment.items())
# command = f'export {str_env} && {command}'
# chan.exec_command(command)
# stdout = chan.makefile("r", -1)
# out = stdout.readline()
# while out:
# yield chan.exit_status, out
# out = stdout.readline()
# yield chan.recv_exit_status(), out
#
# def put_file_by_fl(self, fl, remote_path, callback=None):
# with self as cli:
# sftp = cli.open_sftp()
# sftp.putfo(fl, remote_path, callback=callback)
#
# def list_dir_attr(self, path):
# with self as cli:
# sftp = cli.open_sftp()
# return sftp.listdir_attr(path)
#
# def remove_file(self, path):
# with self as cli:
# sftp = cli.open_sftp()
# sftp.remove(path)
def __enter__(self):
if self.client is not None:
raise RuntimeError('Already connected')
return self.get_client()
def __exit__(self, exc_type, exc_val, exc_tb):
self.client.close()
self.client = None
class RemoteClient(object):
"""Remote Client is a wrapper over SSHClient with utility functions.
Args:
host (string): The hostname of the server to connect. It can be an IP
address of the server also.
user (string, optional): The user to connect to the remote server. It
defaults to root
Attributes:
host (string): The hostname passed in as a the argument
user (string): The user to connect as to the remote server
client (:class:`paramiko.client.SSHClient`): The SSHClient object used
for all the communications with the remote server.
sftpclient (:class:`paramiko.sftp_client.SFTPClient`): The SFTP object
for all the file transfer operations over the SSH.
"""
def __init__(self, host, ip=None, user='******'):
self.host = host
self.ip = ip
self.user = user
self.client = SSHClient()
self.sftpclient = None
self.client.set_missing_host_key_policy(AutoAddPolicy())
self.client.load_system_host_keys()
logging.debug("RemoteClient created for host: %s", host)
def startup(self):
"""Function that starts SSH connection and makes client available for
carrying out the functions. It tries with the hostname, if it fails
it tries with the IP address if supplied
"""
try:
logging.debug("Trying to connect to remote server %s", self.host)
self.client.connect(self.host, port=22, username=self.user)
self.sftpclient = self.client.open_sftp()
except (SSHException, socket.error):
if self.ip:
logging.warning("Connection with hostname failed. Retrying "
"with IP")
self._try_with_ip()
else:
logging.error("Connection to %s failed.", self.host)
raise ClientNotSetupException('Could not connect to the host.')
def _try_with_ip(self):
try:
logging.debug("Connecting to IP:%s User:%s", self.ip, self.user)
self.client.connect(self.ip, port=22, username=self.user)
self.sftpclient = self.client.open_sftp()
except (SSHException, socket.error):
logging.error("Connection with IP (%s) failed.", self.ip)
raise ClientNotSetupException('Could not connect to the host.')
def download(self, remote, local):
"""Downloads a file from remote server to the local system.
Args:
remote (string): location of the file in remote server
local (string): path where the file should be saved
"""
if not self.sftpclient:
raise ClientNotSetupException(
'Cannot download file. Client not initialized')
try:
self.sftpclient.get(remote, local)
return "Download successful. File at: {0}".format(local)
except OSError:
return "Error: Local file %s doesn't exist." % local
except IOError:
return "Error: Remote location %s doesn't exist." % remote
def upload(self, local, remote):
"""Uploads the file from local location to remote server.
Args:
local (string): path of the local file to upload
remote (string): location on remote server to put the file
"""
if not self.sftpclient:
raise ClientNotSetupException(
'Cannot upload file. Client not initialized')
try:
self.sftpclient.put(local, remote)
return "Upload successful. File at: {0}".format(remote)
except OSError:
return "Error: Local file %s doesn't exist." % local
except IOError:
return "Error: Remote location %s doesn't exist." % remote
def exists(self, filepath):
"""Returns whether a file exists or not in the remote server.
Args:
filepath (string): path to the file to check for existance
Returns:
True if it exists, False if it doesn't
"""
if not self.client:
raise ClientNotSetupException(
'Cannot run procedure. Client not initialized')
cin, cout, cerr = self.client.exec_command('stat {0}'.format(filepath))
if len(cout.read()) > 5:
return True
elif len(cerr.read()) > 5:
return False
def run(self, command):
"""Run a command in the remote server.
Args:
command (string): the command to be run on the remote server
Returns:
tuple of three strings containing text from stdin, stdout an stderr
"""
if not self.client:
raise ClientNotSetupException(
'Cannot run procedure. Client not initialized')
#buffers = self.client.exec_command(command, timeout=30)
buffers = self.client.exec_command(command)
output = []
for buf in buffers:
try:
output.append(buf.read())
except IOError:
output.append('')
return tuple(output)
def get_file(self, filename):
"""Reads content of filename on remote server
Args:
filename (string): name of file to be read from remote server
Returns:
tuple: True/False, file like object / error
"""
f = StringIO.StringIO()
try:
r = self.sftpclient.getfo(filename, f)
f.seek(0)
return r, f
except Exception as err:
return False, err
def put_file(self, filename, filecontent):
"""Puts content to a file on remote server
Args:
filename (string): name of file to be written on remote server
filecontent (string): content of file
Returns:
tuple: True/False, file size / error
"""
f = StringIO.StringIO()
f.write(filecontent)
f.seek(0)
try:
r = self.sftpclient.putfo(f, filename)
return True, r.st_size
except Exception as err:
return False, err
def mkdir(self, dirname):
"""Creates a new directory.
Args:
dirname (string): the full path of the directory that needs to be
created
Returns:
a tuple containing the success or failure of operation and dirname
on success and error on failure
"""
try:
self.sftpclient.mkdir(dirname)
return True, dirname
except Exception as err:
return False, err
def listdir(self, dirname):
"""Lists all the files and folders in a directory.
Args:
dirname (string): the full path of the directory that needs to be
listed
Returns:
a list of the files and folders in the directory
"""
try:
r = self.sftpclient.listdir(dirname)
return True, r
except Exception as err:
return False, err
def close(self):
"""Close the SSH Connection
"""
self.client.close()
def __repr__(self):
return "RemoteClient({0}, ip={1}, user={2})".format(
self.host, self.ip, self.user)
class SystemVM(object):
def __init__(self, host='default', vagrantDir=None, controlVagrant=True):
global _defaultVagrantDir
self.host = host
self._controlVagrant = controlVagrant
if vagrantDir is None:
vagrantDir = _defaultVagrantDir
self._vagrant = Vagrant(root=vagrantDir)
self._startedVagrant = False
self._sshClient = None
self._sshConfigStr = None
self._sshConfig = None
self._sshHostConfig = None
def maybeUp(self):
if not self._controlVagrant:
return
state = self._vagrant.status(vm_name=self.host)[0].state
if state == Vagrant.NOT_CREATED:
self._vagrant.up(vm_name=self.host)
self._startedVagrant = True
elif state in [Vagrant.POWEROFF, Vagrant.SAVED, Vagrant.ABORTED]:
raise Exception(
"SystemVM testing does not support resume(), do not use vagrant suspend/halt"
)
elif state == Vagrant.RUNNING:
self._startedVagrant = False
else:
raise Exception("Unrecognized vagrant state %s" % state)
def maybeDestroy(self):
if not self._controlVagrant or not self._startedVagrant:
return
self._vagrant.destroy(vm_name=self.host)
if self._sshClient is not None:
self._sshClient.close()
def loadSshConfig(self):
if self._sshConfig is None:
self._sshConfigStr = self._vagrant.ssh_config(vm_name=self.host)
configObj = StringIO(self._sshConfigStr)
self._sshConfig = SSHConfig()
# noinspection PyTypeChecker
self._sshConfig.parse(configObj)
self._sshHostConfig = self._sshConfig.lookup(self.host)
@property
def sshConfig(self):
if self._sshConfig is None:
self.loadSshConfig()
return self._sshConfig
@property
def sshConfigStr(self):
if self._sshConfigStr is None:
self.loadSshConfig()
return self._sshConfigStr
@property
def sshClient(self):
if self._sshClient is None:
self.loadSshConfig()
self._sshClient = SSHClient()
self._sshClient.set_missing_host_key_policy(AutoAddPolicy())
self._sshClient.connect(self.hostname,
self.sshPort,
self.sshUser,
key_filename=self.sshKey,
timeout=10)
return self._sshClient
@property
def hostname(self):
return self._sshHostConfig.get('hostname', self.host)
@property
def sshPort(self):
return int(self._sshHostConfig.get('port', 22))
@property
def sshUser(self):
return self._sshHostConfig.get('user', 'root')
@property
def sshKey(self):
return self._sshHostConfig.get('identityfile', '~/.ssh/id_rsa')
class sshWidget(QtWidgets.QWidget):
signalShowText = QtCore.Signal()
def __init__(self, addr, usr, pwd, pk = None, kf = None):
super().__init__()
self.initialize(addr, usr, pwd, pk, kf)
def initialize(self, addr, usr, pwd, pk, kf):
try:
self.signalShowText.connect(self.ShowText)
self.hasEndedScreen = False
self.interactiveWidget = QtWidgets.QTextEdit(self)
self.interactiveWidget.setFont(QtGui.QFont({
'win32': 'Consolas',
'linux': 'Monospace',
'darwin': 'Andale Mono'
}.get(sys.platform, 'Courier'), 10))
#self.interactiveWidget.setFocusPolicy(QtCore.Qt.ClickFocus)
self.interactiveWidget.setStyleSheet("background-color : green; color : #cccccc;")
self.interactiveWidget.installEventFilter(self)
self.interactiveWidget.selectionChanged.connect(self.interactiveWidget.copy)
self.interactiveWidget.setContextMenuPolicy(QtCore.Qt.NoContextMenu)
self.interactiveWidget.setCursorWidth(5)
lay = QtWidgets.QHBoxLayout()
lay.addWidget(self.interactiveWidget)
self.setLayout(lay)
# proc vt stream
self.defaultLines = 240
self._vt = pyte.Screen(self.defaultLines, 120)
self.stream = pyte.Stream()
self.stream.attach(self._vt)
# connect to ssh
(host, port) = addr.split(':')
self.ssh = SSHClient()
self.ssh.set_missing_host_key_policy(AutoAddPolicy())
self.ssh.connect(host, port=int(port), username=usr, password=pwd, pkey = pk, key_filename=kf)
self.shell = self.ssh.invoke_shell()
self.thread = threading.Thread(target = ThreadRun, args = (self, ), daemon = True)
self.thread.start()
return True
except Exception as e:
traceback.print_exc()
return False
def run(self):
time.sleep(0.5)
while True:
try:
buf = self.shell.recv(999)
if len(buf) < 1: break
self.stream.feed(buf.decode("utf-8"))
self.signalShowText.emit()
except Exception as e:
self.stream.feed("the network is wrong for: %s"%(traceback.format_exc()))
self.signalShowText.emit()
break
@QtCore.Slot()
def ShowText(self):
try:
self.interactiveWidget.setPlainText("\n".join(self._vt.display))
self.hasEndedScree = self.hasEndedScreen or (self._vt.cursor.y + 1) >= self._vt.lines
tc = self.interactiveWidget.textCursor()
tc.setPosition(self._vt.cursor.y * (self._vt.columns + 1) + self._vt.cursor.x, QtGui.QTextCursor.MoveAnchor)
self.interactiveWidget.setTextCursor(tc)
self.interactiveWidget.setFocus()
except Exception as e:
traceback.print_exc()
def mousePressEvent(self, QMouseEvent):
if QMouseEvent.button() == QtCore.Qt.RightButton:
clipText = QtWidgets.QApplication.clipboard().text()
if clipText: # send the clip text to ssh server when right click
self.Send(clipText)
else:
pass
def resizeEvent(self, event):
if hasattr(self, "_vt"):
sz = QtGui.QFontMetrics(self.font()).size(0, " ")
columns = int(event.size().width() / sz.width())
self._vt.resize(self.defaultLines, columns)
self.ShowText()
super().resizeEvent(event)
def eventFilter(self, obj, event):
if obj == self.interactiveWidget:
if event.type() == QtCore.QEvent.KeyPress:
self.Send(
{
QtCore.Qt.Key_Tab: "\t",
QtCore.Qt.Key_Backspace: "\x7f",
QtCore.Qt.Key_Up: "\033[A",
QtCore.Qt.Key_Down: "\033[B",
QtCore.Qt.Key_Left: "\033[D",
QtCore.Qt.Key_Right:"\033[C"
}.get(event.key(), event.text()))
return True
return False
def Send(self, text):
try:
self.shell.send(text)
except Exception as e:
traceback.print_exc()
def destroy(self, v1, v2):
try:
self.shell.close()
self.ssh.close()
self.thread.join()
except Exception as e:
traceback.print_exc()
return super().destroy(v1, v2)
class USIEngine:
def __init__(self,
name,
host,
engine_path,
nodes=None,
multiPV=1,
threads=1,
delay=0,
delay2=0):
self.name = name
self.nodes = nodes
self.multiPV = multiPV
self.quit_event = threading.Event()
self.client = SSHClient()
self.client.set_missing_host_key_policy(paramiko.client.WarningPolicy)
#self.client.load_system_host_keys()
keys = self.client.get_host_keys()
keys.clear()
self.client.connect(host)
dirname = os.path.dirname(engine_path)
command = f'cd {dirname} && {engine_path}'
self.stdin, self.stdout, self.stderr = \
self.client.exec_command(command, bufsize=0)
self.queue = queue.Queue()
self.watcher_thread = threading.Thread(target=self.stream_watcher,
name='engine_watcher',
args=(self.stdout, ))
self.watcher_thread.start()
self.pvs = [[]] * multiPV
self.status = 'wait'
self.position = 'startpos'
self.send('usi')
self.wait_for('usiok')
self.set_option('Threads', threads)
self.set_option('USI_Ponder', 'false')
self.set_option('NetworkDelay', delay)
self.set_option('NetworkDelay2', delay2)
self.set_option('MultiPV', multiPV)
if nodes:
self.set_option('NodesLimit', nodes)
#self.send('isready')
#self.wait_for('readyok')
def stream_watcher(self, stream):
# for line in iter(stream.readline, b''):
prog = re.compile('.*score cp (-?\d+) (?:multipv (\d+))? .*pv (.+)$')
#for line in iter(stream.readline, b''):
while (not self.quit_event.isSet()) and (not stream.closed):
line = stream.readline().strip()
if len(line):
logging.debug(f'{self.name} > {line}')
print(f'info string {self.name} > {line}', flush=True)
match = prog.match(line)
if match:
logging.debug(f'match: {match.group(1, 2, 3)}')
if match.group(2):
# multi PV
num = int(match.group(2)) - 1
else:
# single PV
num = 0
logging.debug(f'{self.name}: Found score of pv {num}')
self.pvs[num] = [int(match.group(1)), match.group(3)]
# bestmove
if line.startswith('bestmove'):
self.status = 'wait'
self.queue.put(line)
logging.debug(f'{self.name}: terminating the engine watcher thread')
def set_option(self, name, value):
self.send(f'setoption name {name} value {value}')
def __del__(self):
pass
#self.terminate()
def terminate(self):
self.stop()
self.quit_event.set()
self.send('usi')
self.watcher_thread.join(1)
self.send('quit')
self.status = 'quit'
#self.client.close()
def send(self, command):
logging.debug(f'sending {command} to {self.name}')
print(f'info string sending {command} to {self.name}', flush=True)
self.stdin.write((command + '\n').encode('utf-8'))
self.stdin.flush()
def wait_for(self, command):
logging.debug(f'{self.name}: waiting for {command}')
lines = ""
while self.client.get_transport().is_active():
line = self.queue.get()
lines += f'{line}\n'
if (line == command):
logging.debug(f'{self.name}: found {command}')
self.status = 'wait'
return lines
def wait_for_bestmove(self):
logging.debug(f'{self.name}: waiting for bestmove...')
infostr(f'{self.name}: waiting for bestmove...')
while self.client.get_transport().is_active():
line = self.queue.get()
if (line.startswith('bestmove')):
logging.debug(f'{self.name}: found bestmove')
infostr(f'{self.name}: found bestmove')
bestmove = line[9:].split()[0].strip()
self.status = 'wait'
return bestmove
def set_position(self, pos):
self.position = pos
self.send(f'position {pos}')
def clear_queue(self):
while True:
try:
line = self.queue.get_nowait()
print(f'info string {self.name}: clearing queue: {line}',
flush=True)
except queue.Empty:
break
def ponder(self, command):
infostr(f'{self.name}: in ponder()')
self.go_command = command
if 'ponder' not in command:
command = command.replace('go', 'go ponder')
self.send(command)
self.status = 'ponder'
infostr(f'{self.name}: end of ponder()')
def stop(self):
infostr(f'{self.name}: in stop()')
if self.status in ['go', 'ponder']:
self.send('stop')
self.wait_for_bestmove()
self.status = 'wait'
class GLEAM(Base):
def __init__(self, filename=None):
Base.__init__(self, filename)
# set default spatial reference
sr = osr.SpatialReference()
sr.ImportFromEPSG(4326)
self.srs = sr.ExportToWkt()
self.doy = 0 # day of the year to retrieve
self.ssh = None
def open(self, filename):
self.nc = nc.Dataset(filename)
return self.nc is not None
def close(self):
self.nc = None
def getcolrow(self, ds, lon, lat):
col = (lon - LEFT) / SIZE
row = (TOP - lat) / SIZE
return (col, row)
def extractdate(self, name):
''' extract year from filename'''
#SMroot_2017_GLEAM_v3.2b.nc
pat = r'(?P<var>\w+)_(?P<year>\d{4})_GLEAM_v(?P<ver>\d\.\d\w)\.nc$'
m = re.search(pat, name)
if m is not None:
year = int(m.group('year'))
return datetime.date(year, 1, 1)
return None
def get_dataset(self, name):
return self.nc.variables[name]
def transbox(self, ds, bbox, topix=False, clip=False):
x1, y1, x2, y2 = bbox
if clip or topix:
px1, py1 = self.getcolrow(ds, x1, y1)
px2, py2 = self.getcolrow(ds, x2, y2)
if clip:
px1 = max(0, min(px1, WIDTH - 1))
px2 = max(0, min(px2, WIDTH - 1))
py1 = max(0, min(py1, HEIGHT - 1))
py2 = max(0, min(py2, HEIGHT - 1))
if topix:
x1 = int(px1)
x2 = int(px2)
y1 = int(py1)
y2 = int(py2)
if y1 > y2:
return (x1, y2, x2, y1)
else:
return (x1, y1, x2, y2)
def get_data(self, ds, bbox=None):
if isinstance(ds, str):
ds = self.get_dataset(ds)
if bbox is None:
# no bounding box: return entire tile
data = ds[self.doy]
else:
# clip bounding box
x1, y1, x2, y2 = self.transbox(ds, bbox, topix=True, clip=True)
data = ds[self.doy, int(x1):int(x2), int(y1):int(y2)]
# need to transpose data for gdal: y is first index
return np.transpose(data)
def iter_data(self, ds, bbox=None):
''' yield a tile for every day in this dataset '''
if isinstance(ds, str):
ds = self.get_dataset(ds)
times = self.nc.variables['time']
self.doy = 0
for time in times:
date = DAY0 + timedelta(days=int(time))
yield (date, self.get_data(ds, bbox))
self.doy += 1
def connect(self, host, port=0, timeout=-999):
if self.ssh is None:
self.ssh = SSHClient()
self.ssh.set_missing_host_key_policy(AutoAddPolicy())
self.ssh.connect(hostname=host,
port=port,
username=GLEAM_USERNAME,
password=GLEAM_PASSWORD)
self.ftp = self.ssh.open_sftp()
return self.ftp
def download(self, filename, folder, overwrite=True):
print(filename)
localpath = os.path.join(folder, filename)
if not os.path.exists(folder):
os.makedirs(folder)
if os.path.exists(localpath):
if not overwrite:
print(localpath + ' exists')
return
self.ftp.get(filename, localpath)
def download_tile(self, folder, tile, dest, overwrite=True):
self.ftp.chdir(folder)
files = self.ftp.listdir()
for filename in files:
if tile is None or tile in filename:
self.download(filename, dest, overwrite)
break
def download_dataset(self, name, years, version, dest, overwrite=True):
if self.ssh is None:
self.connect(GLEAM_HOST, GLEAM_PORT)
if not dest.endswith('/'):
dest += '/'
for year in years:
folder = GLEAM_PATH.format(version=version, year=year)
self.download_tile(folder, name, dest, overwrite)
def create_tif(self, filename, extent, data, template, etype):
if os.path.exists(filename):
os.remove(filename)
else:
dirname = os.path.dirname(filename)
if not os.path.exists(dirname):
os.makedirs(dirname)
ysize, xsize = data.shape
tif = gdal.GetDriverByName('GTiff').Create(filename,
xsize,
ysize,
eType=etype)
tif.SetProjection(self.srs)
tif.SetGeoTransform([extent[0], SIZE, 0, extent[3], 0, -SIZE])
band = tif.GetRasterBand(1)
band.WriteArray(data)
class SshClusterConnection(AbstractConnection):
def __init__(self, girder_token, cluster):
self._girder_token = girder_token
self._cluster = cluster
def _load_rsa_key(self, path, passphrase):
return RSAKey.from_private_key_file(path, password=passphrase)
def __enter__(self):
self._client = SSHClient()
self._client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
username = parse('config.ssh.user').find(self._cluster)[0].value
hostname = parse('config.host').find(self._cluster)[0].value
port = parse('config.port').find(self._cluster)
if port:
port = port[0].value
else:
port = 22
passphrase \
= parse('config.ssh.passphrase').find(self._cluster)
if passphrase:
passphrase = passphrase[0].value
else:
passphrase = None
key_name = parse('config.ssh.key').find(self._cluster)[0].value
key_path = os.path.join(cumulus.config.ssh.keyStore, key_name)
private_key = self._load_rsa_key(key_path, passphrase)
self._client.connect(hostname=hostname,
port=port,
username=username,
pkey=private_key)
return self
def __exit__(self, type, value, traceback):
self._client.close()
def execute(self, command, ignore_exit_status=False, source_profile=True):
if source_profile:
command = 'source /etc/profile && %s' % command
chan = self._client.get_transport().open_session()
chan.exec_command(command)
stdout = chan.makefile('r', -1)
stderr = chan.makefile_stderr('r', -1)
output = stdout.readlines() + stderr.readlines()
exit_code = chan.recv_exit_status()
if ignore_exit_status and exit_code != 0:
raise SshCommandException(command, exit_code, output)
return output
@contextmanager
def get(self, remote_path):
sftp = None
file = None
try:
sftp = self._client.get_transport().open_sftp_client()
file = sftp.open(remote_path)
yield file
finally:
if file:
file.close()
sftp.close()
def isfile(self, remote_path):
with self._client.get_transport().open_sftp_client() as sftp:
try:
s = sftp.stat(remote_path)
except IOError:
return False
return stat.S_ISDIR(s.st_mode)
def mkdir(self, remote_path, ignore_failure=False):
with self._client.get_transport().open_sftp_client() as sftp:
try:
sftp.mkdir(remote_path)
except IOError:
if not ignore_failure:
raise
def makedirs(self, remote_path):
with self._client.get_transport().open_sftp_client() as sftp:
current_path = ''
if remote_path[0] == '/':
current_path = '/'
for path in remote_path.split('/'):
if not path:
continue
current_path = os.path.join(current_path, path)
try:
sftp.listdir(current_path)
except IOError:
sftp.mkdir(current_path)
def put(self, stream, remote_path):
with self._client.get_transport().open_sftp_client() as sftp:
sftp.putfo(stream, remote_path)
def stat(self, remote_path):
with self._client.get_transport().open_sftp_client() as sftp:
return sftp.stat(remote_path)
def remove(self, remote_path):
with self._client.get_transport().open_sftp_client() as sftp:
return sftp.remove(remote_path)
def list(self, remote_path):
with self._client.get_transport().open_sftp_client() as sftp:
for path in sftp.listdir_iter(remote_path):
yield {
'name': path.filename,
'user': path.st_uid,
'group': path.st_gid,
'mode': path.st_mode,
# For now just pass mtime through
'date': path.st_mtime,
'size': path.st_size
}
def __init__(
self,
host,
user=None,
port=None,
config=None,
gateway=None,
forward_agent=None,
connect_timeout=None,
connect_kwargs=None,
inline_ssh_env=None,
):
"""
Set up a new object representing a server connection.
:param str host:
the hostname (or IP address) of this connection.
May include shorthand for the ``user`` and/or ``port`` parameters,
of the form ``user@host``, ``host:port``, or ``user@host:port``.
.. note::
Due to ambiguity, IPv6 host addresses are incompatible with the
``host:port`` shorthand (though ``user@host`` will still work
OK). In other words, the presence of >1 ``:`` character will
prevent any attempt to derive a shorthand port number; use the
explicit ``port`` parameter instead.
.. note::
If ``host`` matches a ``Host`` clause in loaded SSH config
data, and that ``Host`` clause contains a ``Hostname``
directive, the resulting `.Connection` object will behave as if
``host`` is equal to that ``Hostname`` value.
In all cases, the original value of ``host`` is preserved as
the ``original_host`` attribute.
Thus, given SSH config like so::
Host myalias
Hostname realhostname
a call like ``Connection(host='myalias')`` will result in an
object whose ``host`` attribute is ``realhostname``, and whose
``original_host`` attribute is ``myalias``.
:param str user:
the login user for the remote connection. Defaults to
``config.user``.
:param int port:
the remote port. Defaults to ``config.port``.
:param config:
configuration settings to use when executing methods on this
`.Connection` (e.g. default SSH port and so forth).
Should be a `.Config` or an `invoke.config.Config`
(which will be turned into a `.Config`).
Default is an anonymous `.Config` object.
:param gateway:
An object to use as a proxy or gateway for this connection.
This parameter accepts one of the following:
- another `.Connection` (for a ``ProxyJump`` style gateway);
- a shell command string (for a ``ProxyCommand`` style style
gateway).
Default: ``None``, meaning no gatewaying will occur (unless
otherwise configured; if one wants to override a configured gateway
at runtime, specify ``gateway=False``.)
.. seealso:: :ref:`ssh-gateways`
:param bool forward_agent:
Whether to enable SSH agent forwarding.
Default: ``config.forward_agent``.
:param int connect_timeout:
Connection timeout, in seconds.
Default: ``config.timeouts.connect``.
.. _connect_kwargs-arg:
:param dict connect_kwargs:
Keyword arguments handed verbatim to
`SSHClient.connect <paramiko.client.SSHClient.connect>` (when
`.open` is called).
`.Connection` tries not to grow additional settings/kwargs of its
own unless it is adding value of some kind; thus,
``connect_kwargs`` is currently the right place to hand in paramiko
connection parameters such as ``pkey`` or ``key_filename``. For
example::
c = Connection(
host="hostname",
user="******",
connect_kwargs={
"key_filename": "/home/myuser/.ssh/private.key",
},
)
Default: ``config.connect_kwargs``.
:param bool inline_ssh_env:
Whether to send environment variables "inline" as prefixes in front
of command strings (``export VARNAME=value && mycommand here``),
instead of trying to submit them through the SSH protocol itself
(which is the default behavior). This is necessary if the remote
server has a restricted ``AcceptEnv`` setting (which is the common
default).
The default value is the value of the ``inline_ssh_env``
:ref:`configuration value <default-values>` (which itself defaults
to ``False``).
.. warning::
This functionality does **not** currently perform any shell
escaping on your behalf! Be careful when using nontrivial
values, and note that you can put in your own quoting,
backslashing etc if desired.
Consider using a different approach (such as actual
remote shell scripts) if you run into too many issues here.
.. note::
When serializing into prefixed ``FOO=bar`` format, we apply the
builtin `sorted` function to the env dictionary's keys, to
remove what would otherwise be ambiguous/arbitrary ordering.
.. note::
This setting has no bearing on *local* shell commands; it only
affects remote commands, and thus, methods like `.run` and
`.sudo`.
:raises ValueError:
if user or port values are given via both ``host`` shorthand *and*
their own arguments. (We `refuse the temptation to guess`_).
.. _refuse the temptation to guess:
http://zen-of-python.info/
in-the-face-of-ambiguity-refuse-the-temptation-to-guess.html#12
.. versionchanged:: 2.3
Added the ``inline_ssh_env`` parameter.
"""
# NOTE: parent __init__ sets self._config; for now we simply overwrite
# that below. If it's somehow problematic we would want to break parent
# __init__ up in a manner that is more cleanly overrideable.
super(Connection, self).__init__(config=config)
#: The .Config object referenced when handling default values (for e.g.
#: user or port, when not explicitly given) or deciding how to behave.
if config is None:
config = Config()
# Handle 'vanilla' Invoke config objects, which need cloning 'into' one
# of our own Configs (which grants the new defaults, etc, while not
# squashing them if the Invoke-level config already accounted for them)
elif not isinstance(config, Config):
config = config.clone(into=Config)
self._set(_config=config)
# TODO: when/how to run load_files, merge, load_shell_env, etc?
# TODO: i.e. what is the lib use case here (and honestly in invoke too)
shorthand = self.derive_shorthand(host)
host = shorthand["host"]
err = "You supplied the {} via both shorthand and kwarg! Please pick one." # noqa
if shorthand["user"] is not None:
if user is not None:
raise ValueError(err.format("user"))
user = shorthand["user"]
if shorthand["port"] is not None:
if port is not None:
raise ValueError(err.format("port"))
port = shorthand["port"]
# NOTE: we load SSH config data as early as possible as it has
# potential to affect nearly every other attribute.
#: The per-host SSH config data, if any. (See :ref:`ssh-config`.)
self.ssh_config = self.config.base_ssh_config.lookup(host)
self.original_host = host
#: The hostname of the target server.
self.host = host
if "hostname" in self.ssh_config:
# TODO: log that this occurred?
self.host = self.ssh_config["hostname"]
#: The username this connection will use to connect to the remote end.
self.user = user or self.ssh_config.get("user", self.config.user)
# TODO: is it _ever_ possible to give an empty user value (e.g.
# user='')? E.g. do some SSH server specs allow for that?
#: The network port to connect on.
self.port = port or int(self.ssh_config.get("port", self.config.port))
# Gateway/proxy/bastion/jump setting: non-None values - string,
# Connection, even eg False - get set directly; None triggers seek in
# config/ssh_config
#: The gateway `.Connection` or ``ProxyCommand`` string to be used,
#: if any.
self.gateway = gateway if gateway is not None else self.get_gateway()
# NOTE: we use string above, vs ProxyCommand obj, to avoid spinning up
# the ProxyCommand subprocess at init time, vs open() time.
# TODO: make paramiko.proxy.ProxyCommand lazy instead?
if forward_agent is None:
# Default to config...
forward_agent = self.config.forward_agent
# But if ssh_config is present, it wins
if "forwardagent" in self.ssh_config:
# TODO: SSHConfig really, seriously needs some love here, god
map_ = {"yes": True, "no": False}
forward_agent = map_[self.ssh_config["forwardagent"]]
#: Whether agent forwarding is enabled.
self.forward_agent = forward_agent
if connect_timeout is None:
connect_timeout = self.ssh_config.get("connecttimeout",
self.config.timeouts.connect)
if connect_timeout is not None:
connect_timeout = int(connect_timeout)
#: Connection timeout
self.connect_timeout = connect_timeout
#: Keyword arguments given to `paramiko.client.SSHClient.connect` when
#: `open` is called.
self.connect_kwargs = self.resolve_connect_kwargs(connect_kwargs)
#: The `paramiko.client.SSHClient` instance this connection wraps.
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
self.client = client
#: A convenience handle onto the return value of
#: ``self.client.get_transport()``.
self.transport = None
if inline_ssh_env is None:
inline_ssh_env = self.config.inline_ssh_env
#: Whether to construct remote command lines with env vars prefixed
#: inline.
self.inline_ssh_env = inline_ssh_env
class SshMixin(UploadDownloadMixin):
def __init__(self, host, user, password, interpreter, cwd):
# UploadDownloadMixin.__init__(self)
try:
import paramiko
from paramiko.client import SSHClient, AutoAddPolicy
except ImportError:
print(
"\nThis back-end requires an extra package named 'paramiko'."
" Install it from 'Tools => Manage plug-ins' or via your system package manager.",
file=sys.stderr,
)
sys.exit()
self._host = host
self._user = user
self._password = password
self._remote_interpreter = interpreter
self._cwd = cwd
self._proc = None # type: Optional[RemoteProcess]
self._sftp = None # type: Optional[paramiko.SFTPClient]
self._client = SSHClient()
self._client.load_system_host_keys()
self._client.set_missing_host_key_policy(paramiko.client.AutoAddPolicy())
# TODO: does it get closed properly after process gets killed?
self._connect()
def _connect(self):
from paramiko.ssh_exception import AuthenticationException
import socket
from paramiko import SSHException
try:
self._client.connect(
hostname=self._host,
username=self._user,
password=self._password,
passphrase=self._password,
)
except (SSHException, OSError) as e:
print(
"\nCan't connect to '%s' with user '%s': %s" % (self._host, self._user, str(e)),
file=sys.stderr,
)
print("Re-check your host, authentication method, password or keys.", file=sys.stderr)
delete_stored_ssh_password()
sys.exit(1)
def _create_remote_process(self, cmd_items: List[str], cwd: str, env: Dict) -> RemoteProcess:
# Before running the main thing:
# * print process id (so that we can kill it later)
# http://redes-privadas-virtuales.blogspot.com/2013/03/getting-hold-of-remote-pid-through.html
# * change to desired directory
#
# About -onlcr: https://stackoverflow.com/q/35887380/261181
cmd_line_str = (
"echo $$ ; stty -echo ; stty -onlcr ; "
+ (" cd %s 2> /dev/null ;" % shlex.quote(cwd) if cwd else "")
+ (" exec " + " ".join(map(shlex.quote, cmd_items)))
)
stdin, stdout, _ = self._client.exec_command(
cmd_line_str, bufsize=0, get_pty=True, environment=env
)
# stderr gets directed to stdout because of pty
pid = stdout.readline().strip()
channel = stdout.channel
return RemoteProcess(self._client, channel, stdin, stdout, pid)
def _handle_immediate_command(self, cmd: ImmediateCommand) -> None:
if cmd.name == "kill":
self._kill()
elif cmd.name == "interrupt":
self._interrupt()
else:
raise RuntimeError("Unknown immediateCommand %s" % cmd.name)
def _kill(self):
if self._proc is None or self._proc.poll() is not None:
return
self._proc.kill()
def _interrupt(self):
pass
def _get_sftp(self, fresh: bool):
if fresh and self._sftp is not None:
self._sftp.close()
self._sftp = None
if self._sftp is None:
import paramiko
# TODO: does it get closed properly after process gets killed?
self._sftp = paramiko.SFTPClient.from_transport(self._client.get_transport())
return self._sftp
def _read_file(
self, source_path: str, target_fp: BinaryIO, callback: Callable[[int, int], None]
) -> None:
self._perform_sftp_operation_with_retry(
lambda sftp: sftp.getfo(source_path, target_fp, callback)
)
def _write_file(
self,
source_fp: BinaryIO,
target_path: str,
file_size: int,
callback: Callable[[int, int], None],
) -> None:
self._perform_sftp_operation_with_retry(
lambda sftp: sftp.putfo(source_fp, target_path, callback)
)
def _perform_sftp_operation_with_retry(self, operation) -> Any:
try:
return operation(self._get_sftp(fresh=False))
except OSError:
# It looks like SFTPClient gets stale after a while.
# Try again with fresh SFTPClient
return operation(self._get_sftp(fresh=True))
def _get_stat_mode_for_upload(self, path: str) -> Optional[int]:
try:
return self._perform_sftp_operation_with_retry(lambda sftp: sftp.stat(path).st_mode)
except OSError as e:
return None
def _mkdir_for_upload(self, path: str) -> None:
self._perform_sftp_operation_with_retry(lambda sftp: sftp.mkdir(path, NEW_DIR_MODE))
from paramiko.client import SSHClient,AutoAddPolicy
from getpass import getpass
Covid77 = getpass()
sshc = SSHClient()
sshc.set_missing_host_key_policy(AutoAddPolicy())
sshconnect("10.0.2.100", 22, "rafii", Covid77)
while(True)
sstdin, stdout, stderr = ssch.exec_command('python3 program2.py')
print(stdout.readlines())
print(stderr.readlines())
class SFTP(Communicator):
ssh_client = None
@property
def client(self):
if self.ssh_client and self.ssh_client.get_transport().is_active():
return self._active_client
self._active_client = self._client()
return self._active_client
def _client(self):
logger.info(u'Conecting through SSH to the server (%s:%s)', self.host,
self.port)
try:
self.ssh_client = SSHClient()
self.ssh_client.set_missing_host_key_policy(
paramiko.AutoAddPolicy())
self.ssh_client.connect(self.host,
username=self.user,
password=self.password,
compress=True)
except ssh_exception.AuthenticationException:
logger.error(
u'Fail while connecting through SSH. Check your creadentials.')
return None
except ssh_exception.NoValidConnectionsError:
logger.error(
u'Fail while connecting through SSH. Check your credentials or the server availability.'
)
return None
else:
return self.ssh_client.open_sftp()
def mkdir(self, path):
logger.info(u'Creating directory (%s)', path)
try:
self.client.mkdir(path)
logger.debug(u'Directory has being created (%s)', path)
except IOError as e:
try:
self.client.stat(path)
logger.warning(u'Directory already exists (%s)', path)
except IOError as e:
logger.error(u'Fail while creating directory (%s): %s', path,
e.strerror)
raise (e)
def chdir(self, path):
logger.info(u'Changing to directory (%s)', path)
try:
self.client.chdir(path)
except IOError as e:
logger.error(u'Fail while accessing directory (%s): %s', path,
e.strerror)
raise (e)
def put(self, from_fl, to_fl):
logger.info(u'Copying file from (%s) to (%s)', from_fl, to_fl)
try:
self.client.put(from_fl, to_fl)
logger.debug(u'File has being copied (%s)', to_fl)
except OSError as e:
logger.error(u'Fail while copying file (%s), file not found',
to_fl)
except IOError as e:
logger.error(u'Fail while copying file (%s): %s', to_fl,
e.strerror)
def __init__(
self,
host,
user=None,
port=None,
config=None,
gateway=None,
forward_agent=None,
connect_timeout=None,
connect_kwargs=None,
):
"""
Set up a new object representing a server connection.
:param str host:
the hostname (or IP address) of this connection.
May include shorthand for the ``user`` and/or ``port`` parameters,
of the form ``user@host``, ``host:port``, or ``user@host:port``.
.. note::
Due to ambiguity, IPv6 host addresses are incompatible with the
``host:port`` shorthand (though ``user@host`` will still work
OK). In other words, the presence of >1 ``:`` character will
prevent any attempt to derive a shorthand port number; use the
explicit ``port`` parameter instead.
.. note::
If ``host`` matches a ``Host`` clause in loaded SSH config
data, and that ``Host`` clause contains a ``Hostname``
directive, the resulting `.Connection` object will behave as if
``host`` is equal to that ``Hostname`` value.
In all cases, the original value of ``host`` is preserved as
the ``original_host`` attribute.
Thus, given SSH config like so::
Host myalias
Hostname realhostname
a call like ``Connection(host='myalias')`` will result in an
object whose ``host`` attribute is ``realhostname``, and whose
``original_host`` attribute is ``myalias``.
:param str user:
the login user for the remote connection. Defaults to
``config.user``.
:param int port:
the remote port. Defaults to ``config.port``.
:param config:
configuration settings to use when executing methods on this
`.Connection` (e.g. default SSH port and so forth).
Should be a `.Config` or an `invoke.config.Config`
(which will be turned into a `.Config`).
Default is an anonymous `.Config` object.
:param gateway:
An object to use as a proxy or gateway for this connection.
This parameter accepts one of the following:
- another `.Connection` (for a ``ProxyJump`` style gateway);
- a shell command string (for a ``ProxyCommand`` style style
gateway).
Default: ``None``, meaning no gatewaying will occur (unless
otherwise configured; if one wants to override a configured gateway
at runtime, specify ``gateway=False``.)
.. seealso:: :ref:`ssh-gateways`
:param bool forward_agent:
Whether to enable SSH agent forwarding.
Default: ``config.forward_agent``.
:param int connect_timeout:
Connection timeout, in seconds.
Default: ``config.timeouts.connect``.
:param dict connect_kwargs:
Keyword arguments handed verbatim to
`SSHClient.connect <paramiko.client.SSHClient.connect>` (when
`.open` is called).
`.Connection` tries not to grow additional settings/kwargs of its
own unless it is adding value of some kind; thus,
``connect_kwargs`` is currently the right place to hand in
parameters such as ``pkey`` or ``key_filename``.
Default: ``config.connect_kwargs``.
:raises ValueError:
if user or port values are given via both ``host`` shorthand *and*
their own arguments. (We `refuse the temptation to guess`_).
.. _refuse the temptation to guess:
http://zen-of-python.info/
in-the-face-of-ambiguity-refuse-the-temptation-to-guess.html#12
"""
# NOTE: parent __init__ sets self._config; for now we simply overwrite
# that below. If it's somehow problematic we would want to break parent
# __init__ up in a manner that is more cleanly overrideable.
super(Connection, self).__init__(config=config)
#: The .Config object referenced when handling default values (for e.g.
#: user or port, when not explicitly given) or deciding how to behave.
if config is None:
config = Config()
# Handle 'vanilla' Invoke config objects, which need cloning 'into' one
# of our own Configs (which grants the new defaults, etc, while not
# squashing them if the Invoke-level config already accounted for them)
elif not isinstance(config, Config):
config = config.clone(into=Config)
self._set(_config=config)
# TODO: when/how to run load_files, merge, load_shell_env, etc?
# TODO: i.e. what is the lib use case here (and honestly in invoke too)
shorthand = self.derive_shorthand(host)
host = shorthand["host"]
err = "You supplied the {} via both shorthand and kwarg! Please pick one." # noqa
if shorthand["user"] is not None:
if user is not None:
raise ValueError(err.format("user"))
user = shorthand["user"]
if shorthand["port"] is not None:
if port is not None:
raise ValueError(err.format("port"))
port = shorthand["port"]
# NOTE: we load SSH config data as early as possible as it has
# potential to affect nearly every other attribute.
#: The per-host SSH config data, if any. (See :ref:`ssh-config`.)
self.ssh_config = self.config.base_ssh_config.lookup(host)
self.original_host = host
#: The hostname of the target server.
self.host = host
if "hostname" in self.ssh_config:
# TODO: log that this occurred?
self.host = self.ssh_config["hostname"]
#: The username this connection will use to connect to the remote end.
self.user = user or self.ssh_config.get("user", self.config.user)
# TODO: is it _ever_ possible to give an empty user value (e.g.
# user='')? E.g. do some SSH server specs allow for that?
#: The network port to connect on.
self.port = port or int(self.ssh_config.get("port", self.config.port))
# Non-None values - string, Connection, even eg False - get set
# directly; None triggers seek in config/ssh_config
if gateway is None:
# SSH config wins over Invoke-style config
if "proxyjump" in self.ssh_config:
# Reverse hop1,hop2,hop3 style ProxyJump directive so we start
# with the final (itself non-gatewayed) hop and work up to
# the front (actual, supplied as our own gateway) hop
hops = reversed(self.ssh_config["proxyjump"].split(","))
prev_gw = None
for hop in hops:
# Happily, ProxyJump uses identical format to our host
# shorthand...
if prev_gw is None:
# TODO: this isn't persisting config! which among other
# things can lead to not honoring skipping ssh config
# file loads...
cxn = Connection(hop)
else:
cxn = Connection(hop, gateway=prev_gw)
prev_gw = cxn
gateway = prev_gw
elif "proxycommand" in self.ssh_config:
# Just a string, which we interpret as a proxy command..
gateway = self.ssh_config["proxycommand"]
else:
# Neither of those? Our config value please.
gateway = self.config.gateway
#: The gateway `.Connection` or ``ProxyCommand`` string to be used,
#: if any.
self.gateway = gateway
# NOTE: we use string above, vs ProxyCommand obj, to avoid spinning up
# the ProxyCommand subprocess at init time, vs open() time.
# TODO: make paramiko.proxy.ProxyCommand lazy instead?
if forward_agent is None:
# Default to config...
forward_agent = self.config.forward_agent
# But if ssh_config is present, it wins
if "forwardagent" in self.ssh_config:
# TODO: SSHConfig really, seriously needs some love here, god
map_ = {"yes": True, "no": False}
forward_agent = map_[self.ssh_config["forwardagent"]]
#: Whether agent forwarding is enabled.
self.forward_agent = forward_agent
if connect_timeout is None:
connect_timeout = self.ssh_config.get("connecttimeout",
self.config.timeouts.connect)
if connect_timeout is not None:
connect_timeout = int(connect_timeout)
#: Connection timeout
self.connect_timeout = connect_timeout
#: Keyword arguments given to `paramiko.client.SSHClient.connect` when
#: `open` is called.
self.connect_kwargs = self.resolve_connect_kwargs(connect_kwargs)
#: The `paramiko.client.SSHClient` instance this connection wraps.
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
self.client = client
#: A convenience handle onto the return value of
#: ``self.client.get_transport()``.
self.transport = None
"""
批量登录服务器(ssh)并安装带认证的squid代理服务
"""
from logging_conf import logger
from paramiko import client
import paramiko
from paramiko.client import SSHClient
import pandas as pd
AUTH_FILE = 'files/passwords'
SSH_SERVER_FILE = 'files/server.tsv'
SQUID_CONF = 'files/squid.conf'
ssh = SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
def load_servers(tsv_file):
"""
从tsv文件中加载要连接ssh服务器
"""
df = pd.read_csv(tsv_file, sep='\t', header=0, quoting=3, dtype=str)
for i in range(df.shape[0]):
yield i, df.loc[i].to_dict()
for i, server in load_servers(SSH_SERVER_FILE):
logger.info('No.%d, ip: %s' % (i, server['hostname']))
try:
# 1. 建立ssh连接
def get_ssh_client(self):
'''Prepare default ssh client and set missing host key policy'''
ssh_client = SSHClient()
ssh_client.set_missing_host_key_policy(
self.missing_host_key_policy_cls())
return ssh_client
