def get_client(self):
" Return an instance of a connected and active `paramiko.SSHClient`. "
# Use existing connection if active
if self._client:
return self._client
# Create the client and connect to proxy server
client = SSHClient()
client.load_host_keys(hosts_file)
client.connect(
settings.PROXY_HOSTNAME,
port=settings.PROXY_PORT,
username=settings.PROXY_USERNAME,
key_filename=key_filename
)
# Get the transport and find create a `direct-tcpip` channel
transport = client.get_transport()
dest_addr = ('0.0.0.0', dest_port)
src_addr = ('127.0.0.1', local_port)
channel = transport.open_channel("direct-tcpip", dest_addr, src_addr)
self._client = SSHClient()
target_client.load_host_keys(hosts_file)
target_client.connect('localhost', port=local_port, username=dest_username, password=dest_password, sock=channel)
return target_client
class SSHLatentBuildSlave(AbstractLatentBuildSlave):
"""
Build slave that runs a command over SSH to start and stop the build slave.
"""
def __init__(self, name, password, ssh_host, username, key_path, host_key_file=os.path.expanduser("~/.ssh/known_hosts"), **kwargs):
"""
Creates a new SSH latent build slave with the given parameters.
"""
AbstractLatentBuildSlave.__init__(self, name, password, **kwargs)
self.client = SSHClient()
self.client.load_system_host_keys(host_key_file)
self.hostname = ssh_host
self.username = username
self.key_path = key_path
self.started = False
def is_connected(self):
self.client.get_transport() != None and self.client.get_transport().is_active()
def start_instance(self, build):
if self.started:
raise ValueError('already started')
return threads.deferToThread(self._start_instance)
def _start_instance(self):
log.msg("connecting to SSH server")
self.client.connect(self.hostname, username=self.username, key_filename=self.key_path)
self.client.get_transport().set_keepalive(True)
log.msg("executing start command")
stdin, stdout, stderr = self.client.exec_command("~/bbvenv/bin/env_exec buildslave start ~/bbslave")
self.started = True
return True
def stop_instance(self, fast=False):
if not self.started:
return defer.succeed(None)
return self._stop_instance()
def _stop_instance(self):
if not self.is_connected(): self.client.connect(self.hostname, username=self.username, key_filename=self.key_path)
log.msg("stopping build slave")
self.client.exec_command("~/bbvenv/bin/env_exec buildslave stop ~/bbslave")
log.msg("closing connection")
self.client.close()
log.msg("finished")
self.started = False
def buildFinished(self, sb):
log.msg("finished build. insubstantiating")
self.insubstantiate()
def copySCP(self, datum_original, datum_destination, server, username, option_type):
"""Copy data via SCP function
"""
util.log_to_file('ssh.log')
ssh = SSHClient()
ssh.load_system_host_keys()
logging.info("connection to %s" % server)
ssh.connect(server,
username=username,
compress=True,
timeout=60)
# SCPCLient takes a paramiko transport as its only argument
client = scp.SCPClient(ssh.get_transport())
logging.info(option_type + ": %s to %s" % (datum_original['location'],
datum_destination['location']))
if option_type == 'upload':
client.put(datum_original['location'],
datum_destination['location'],
recursive=True)
else:
client.get(datum_original['location'],
datum_destination['location'],
recursive=True)
client.close()
def get_file(fname):
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(file_server)
scp = SCPClient(ssh.get_transport())
scp.get(fname)
scp.close()
def send_file(local_file, remote_path, ip, username, password, logger=None):
# Отсылает файл local_file в remote_path удаленной машины по scp
if remote_path[len(remote_path) - 1] != '/': remote_path += '/'
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.load_system_host_keys()
if logger is not None: logger.info("SCP SEND: connecting to %s" % (ip))
try:
ssh.connect(ip, username=username, password=password)
except:
if logger is not None: logger.info("SCP SEND: failed to connect to %s" % (ip))
return False
else:
if logger is not None: logger.info("SCP SEND: connected to %s" % (ip))
try:
if logger is not None: logger.info("SCP SEND: sending file %s" % (local_file))
scp = SCPClient(ssh.get_transport())
scp.put(local_file, remote_path)
except:
if logger is not None: logger.error("SCP SEND: error: failed to send file %s" % (local_file))
ssh.close()
return False
else:
if logger is not None: logger.info("SCP SEND: file sent to %s@%s:%s " % (username, ip, remote_path))
ssh.close()
return True
class Sender(object):
def __init__(self,server,username,password,dest_path,from_path):
self.dest_path = dest_path
self.from_path = from_path
self.recorder = Recorder.Recorder()
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
self.ssh.connect(server,username=username,password=password)
self.sftp = self.ssh.open_sftp()
self.scp = SCPClient(self.ssh.get_transport())
def send(self,file_path):
if op.exists(file_path):
file_modify_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(os.stat(file_path).st_mtime))
if not self.recorder.isSynced(file_path,file_modify_time):
new_file_path = os.path.join(self.dest_path+'/',file_path.split(self.from_path+os.sep)[1])
new_file_path = new_file_path.replace('\\','/')
new_file_dir,new_file = op.split(new_file_path)
if not rexists(self.sftp,new_file_dir):
rmkdir(self.sftp,new_file_dir)
print 'uploading %s .....' % (file_path)
self.scp.put(file_path,new_file_path)
self.recorder.record(file_path,file_modify_time)
else:
pass
def get_rawdata(server, month, tmpfile):
serv = server.objects.get(name=server)
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(serv.ipaddress, username=serv.username, password=serv.password)
client = SCPClient(ssh.get_transport())
client.get(serv.path + '/volume.' + month+ '.csv', tmpfile)
class SSH(object):
"""docstring for SSH"""
def __init__(self):
self.client = SSHClient()
self.client.set_missing_host_key_policy(AutoAddPolicy())
self.client.load_system_host_keys()
def connect(self, server, user, key):
logging.info("Connecting to {u}@{s} [with key:{k}]".format(
u=user, s=server, k=key
))
self.client.connect(server, username=user,
key_filename=key, timeout=2.0)
def close():
self.client.close()
@property
def transport(self):
return self.client.get_transport()
@staticmethod
def SSHFactory(config):
logging.debug("Creating SSH Object.")
sClient = SSH()
sClient.connect(config.staging,
config.stagingUser,
config.key
)
return sClient
class SshArchiver(Archiver):
"""
Archives artifacts using ssh.
"""
def start(self):
"""
Opens the ssh connection.
"""
self.ssh_client = SSHClient()
self.ssh_client.set_missing_host_key_policy(WarningPolicy())
self.ssh_client.connect(
self.target.host,
pkey=self.target.ssh_credentials.get_pkey())
self.sftp_client = SFTPClient.from_transport(
self.ssh_client.get_transport())
def end(self):
"""
Closes the ssh connection.
"""
self.ssh_client.close()
def archive_artifact(self, artifact_url, destination):
"""
Uploads the artifact_url to the destination on
the remote server, underneath the target's basedir.
"""
destination = os.path.join(self.target.basedir, destination)
_, stdout, _ = self.ssh_client.exec_command(
"mkdir -p `dirname %s`" % destination)
# TODO: raise exception if the command fails
_ = stdout.channel.recv_exit_status() # noqa
artifact = urllib2.urlopen(artifact_url)
self.sftp_client.stream_file_to_remote(artifact, destination)
def push_cfg_files():
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
user, server, path = check_connection()
ssh.connect(server, username=user)
repo = Repo(os.path.curdir)
current_branch = repo.active_branch.name
with SCPClient(ssh.get_transport()) as scp:
files = [
'versions.cfg',
'distribution-qa.cfg',
'release.cfg',
'sources.cfg',
'qa.cfg',
'solr.cfg',
]
remote_files = []
for filename in files:
remote_filename = filename
if current_branch == '4.x':
parts = filename.split('.')
parts[0] += '4'
remote_filename = '.'.join(parts)
remote_files.append(remote_filename)
scp.put(
filename,
remote_path='{0}/{1}'.format(path, remote_filename)
)
logger.debug('Files uploaded: ')
logger.debug('\n'.join(remote_files))
def put(self, files, remote_path=None, out_stream=sys.stdout, verbose=False):
"""
Copy a file from the local system to the remote system.
:param files:
:param remote_path:
:param out_stream:
:param verbose:
:return: :rtype:
"""
if remote_path is None:
remote_path = files
self.display("scp '{src}' '{dest}'".format(src=files, dest=remote_path),
out_stream=out_stream, verbose=verbose)
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(Project.address, Project.port, Project.user, Project.password)
scp = SCPClient(ssh.get_transport())
# scp = SCPClient(self.ssh.get_transport())
# noinspection PyBroadException
try:
info("\nfiles: %s" % repr(files))
info("remote_path: %s" % remote_path)
output = scp.put(files, '"{dest}"'.format(dest=remote_path), recursive=True) or ''
except Exception:
try:
output = scp.put(files, remote_path, recursive=True) or ''
except Exception as ex:
output = str(ex)
self.display("\n" + output, out_stream=out_stream, verbose=verbose)
return output
def upload_ssh(args):
if os.path.exists(os.path.expanduser('~/.netrc')):
auth_data = netrc.netrc(os.path.expanduser('~/.netrc')).authenticators(cfg.get_contest_host())
else:
auth_data = None
if not auth_data:
login = input('login: '******'password: '******'problems/' + (
# cfg.get_problem_param('system name') or cfg.get_problem_param('system_name')))
path_prefix = cfg.get_server_contest_path() + 'problems/' + (
cfg.get_problem_param('system name') or cfg.get_problem_param('system_name'))
if args['checker']:
print('Uploading checker')
checker_path = cfg.get_problem_param('checker', True) or 'check.cpp'
checker_path = os.path.normpath(checker_path)
checker_out_path = os.path.join(path_prefix, os.path.basename(checker_path))
scp.put(checker_path, checker_out_path)
if args['validator']:
print('Uploading validator')
validator_path = cfg.get_problem_param('validator', True) or 'validator.cpp'
validator_path = os.path.normpath(validator_path)
validator_out_path = os.path.join(path_prefix, os.path.basename(validator_path))
scp.put(validator_path, validator_out_path)
if args['testlib']:
print('Uploading testlib')
scp.put('../../lib/testlib.h', os.path.join(path_prefix, 'testlib.h'))
if args['tests']:
print('Uploading tests')
for t in Test.test_gen('tests'):
print('uploading {} and {}'.format(t.inf_name(), t.ans_name()))
scp.put(t.inf_path(), os.path.join(path_prefix, 'tests', t.inf_name()))
scp.put(t.ans_path(), os.path.join(path_prefix, 'tests', t.ans_name()))
if args['statement']:
print('Uploading statement.xml')
scp.put('statement/statement.xml', os.path.join(path_prefix, 'statement.xml'))
if args['gvaluer']:
print('Uploading valuer.cfg')
scp.put('valuer.cfg', os.path.join(path_prefix, 'valuer.cfg'))
def executeOnMaster(self, cmd):
"""
Execute command on the current master leader
"""
self.log.debug("Executing on master: " + cmd)
if self._hostnameResolves(self.getManagementEndpoint()):
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
self.getManagementEndpoint(),
username = self.config.get('ACS', "username"),
port = 2200,
key_filename = os.path.expanduser(self.config.get('SSH', "privatekey")))
session = ssh.get_transport().open_session()
AgentRequestHandler(session)
stdin, stdout, stderr = ssh.exec_command(cmd)
stdin.close()
result = ""
for line in stdout.read().splitlines():
self.log.debug(line)
result = result + line.decode("utf-8") + "\n"
for line in stderr.read().splitlines():
self.log.error(line)
else:
self.log.error("Endpoint " + self.getManagementEndpoint() + " does not exist, cannot SSH into it.")
result = "Exception: No cluster is available at " + self.getManagementEndpoint()
ssh.close()
return result
def main(file, key, encrypt, host, username, password):
"""
The main function. Takes multiple parameters which are prompted if not given on the command line.
:param file: The paht of the file to encrypt or decrypt and send.
:param key: The key to encrypt or decrypt.
:param encrypt: Tells if the operation is an encryption or a decryption.
:param host: The host where to send the file.
:param username: Username on the host.
:param password: Password if needed. If not needed '-' should be used to tell that there is no password needed.
"""
ssh = SSHClient()
ssh.load_system_host_keys()
if password != "-":
ssh.connect(host, username=username, password=password)
else:
ssh.connect(host, username=username)
scp = SCPClient(ssh.get_transport())
if encrypt:
print("Encrypting... ", end="")
to_send = encrypt_file(file, key)
print("Done.")
print("Sending to {}...".format(host), end="")
scp.put(to_send)
print("Done.")
else:
print(decrypt_file(file, key))
def get_file(remote_file, local_path, ip, username, password, logger=None):
# Получает с удаленной машины файл remote_file с помощью scp и сохраняет его в local_path.
if local_path[len(local_path) - 1] != '/': local_path += '/'
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.load_system_host_keys()
if logger is not None: logger.info("SCP GET: connecting to %s" % (ip))
try:
ssh.connect(ip, username=username, password=password)
except:
if logger is not None: logger.info("SCP GET: failed to connect to %s" % (ip))
return False
else:
if logger is not None: logger.info("SCP GET: connected to %s" % (ip))
try:
if logger is not None: logger.info("SCP GET: retrieving file %s" % (remote_file))
scp = SCPClient(ssh.get_transport())
scp.get(remote_file, local_path)
except:
if logger is not None: logger.error("SCP GET: error: failed to retrieve file %s" % (remote_file))
ssh.close()
return False
else:
if logger is not None: logger.info("SCP GET: file saved to %s folder" % (local_path))
ssh.close()
return True
def run(self):
try:
# run a command and wait for it to finish
def run(command):
_, stdout, _ = ssh.exec_command(command)
stdout.channel.recv_exit_status()
# send the webapp a textual progress update
listener = Listener()
def log(text):
listener.publish(('project', self.project.name, 'deploy', 'status'), { 'text': text })
# prevent accidentally deploying on the server itself
# (this doesn't prevent deploying to the server's public
# IP address, so there's still a security problem here)
if self.host in ['127.0.0.1', '0.0.0.0']:
raise Exception('you\'re trying to deploy to the server!')
# create temporary build file
path = os.path.join(tempfile.mkdtemp(), 'deploy.launch')
xml = roslaunch_xml_for_file(self.project.project_file_path, self.remote_deploy_path)
open(path, 'w').write(xml)
log('Created file: deploy.launch')
# create a ssh session
log('Connecting to %s@%s...' % (self.user, self.host))
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(self.host, username=self.user, password=self.password)
log('Connected to remote machine')
# clean the deploy location
run('rm -fr %s' % self.remote_deploy_path)
run('mkdir -p %s' % self.remote_deploy_path)
log('Cleaned deploy location')
# copy the launch file over
scp = SCPClient(ssh.get_transport())
scp.put(path, remote_path=self.remote_deploy_path)
log('Copied file: deploy.launch')
# run the file
command = 'cd %s && roslaunch deploy.launch' % self.remote_deploy_path
channel = ssh.invoke_shell()
channel.send(command + '\nexit\n')
log('Ran command: roslaunch deploy.launch')
# wait for the command to finish
import time
while not self.die:
time.sleep(0.1)
except:
import traceback
log('Exception:\n' + traceback.format_exc())
ssh.close()
log('Connection was closed')
def scp(self, filename, remote_path):
config = SSHConfig()
config.parse(open(os.path.expanduser('~/.ssh/config')))
o = config.lookup('geodata')
ssh_client = SSHClient()
ssh_client.load_system_host_keys()
ssh_client.connect(o['hostname'], username=o['user'])
scp = SCPClient(ssh_client.get_transport())
scp.put(filename, remote_path=remote_path)
def copy(hostname, username, filepath, remotefile):
ssh = SSHClient()
# ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname, username=username)
ssh.exec_command("mkdir -p " + remotefile[0:remotefile.rindex('/')])
# SCPCLient takes a paramiko get_transportt as its only argument
scp = SCPClient(ssh.get_transport())
scp.put(filepath, remotefile)
class SecureShell(AbstractRemoteShell):
def __init__(self, hostname, username, password=None, port=22, **kwargs):
super(SecureShell, self).__init__(hostname, **kwargs)
self._hostname = hostname
self._port = port
self._username = username
self._password = password
self.connect()
def do_connect(self):
self._client = SSHClient()
self._client.load_system_host_keys()
self._client.set_missing_host_key_policy(AutoAddPolicy())
self._client.connect(hostname=self._hostname, port=self._port, username=self._username, password=self._password)
self._scp_client = SCPClient(self._client.get_transport())
def do_disconnect(self):
self._client.close()
def execute_command(self, command, env={}, wait=True, check_err=False, cwd=None):
for var, val in env.items():
command = "%s=%s; " % (var, val) + command
chan = self._client.get_transport().open_session()
chan.exec_command( (("cd \"%s\"; " % cwd) if cwd else "") + command)
queue = Queue()
StandardStreamReader(chan.makefile("r"), 1, queue)
StandardStreamReader(chan.makefile_stderr("r"), 2, queue)
def post_process_exit_code():
queue.put( (0, chan.recv_exit_status()) )
queue.put( (0, None) )
Thread(target=post_process_exit_code).start()
return ShellResult(self, command, queue, wait, check_err)
def do_pull(self, local_path, remote_path):
self._scp_client.get(remote_path, local_path)
def do_push(self, local_path, remote_path):
self._scp_client.put(local_path, remote_path)
def do_reboot(self):
self("reboot > /dev/null 2>&1 &")
sleep(.3)
def backup_to_server(servers, tarfile): ssh = SSHClient() ssh.load_system_host_keys() ssh.set_missing_host_key_policy(AutoAddPolicy()) for server in servers: ssh.connect(server['host'], port=server['port'], username=server['user'], password=server['password']) # SCPCLient takes a paramiko transport as its only argument scp = SCPClient(ssh.get_transport()) scp.put(tarfile, server['dir'])
def scp(file, path, ip, user, password):
"""Uses paramiko and scp module built on top of it to SCP vEOS to the VM.
Inputs are the .swi filename from arguments, and a path."""
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(hostname=ip, username=user, password=password)
scp = SCPClient(ssh.get_transport())
print "\nWriting " + file + " to " + path
scp.put(file, remote_path=path)
scp.close()
def handle_click(self, reason):
if reason != QSystemTrayIcon.Trigger:
return
QGuiApplication.primaryScreen().grabWindow(0).save('scr.jpg', 'jpg')
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(hostname=self.ssh_hostname, port=self.ssh_port, username=self.ssh_username)
scp = SCPClient(ssh.get_transport())
dest_name = time.strftime("screenshot_%Y%m%d_%H%M%S.jpg", time.localtime())
scp.put('scr.jpg', self.ssh_remote_path + '/' + dest_name)
def upload_file(self, source_file_path, dest_filename, sub_directory=None, callback=None):
try:
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(self.server.server_url, username=self.server.username, password=self.server.password)
with SCPClient(ssh.get_transport(), socket_timeout=15.0) as scp:
scp.put(source_file_path, os.path.join(self.server.server_directory, dest_filename))
except Exception as e:
logger.exception('SCPServer hit exception - %s' % e.message)
def sshkey_login(self, auth_host, auth_port):
"""Attempt to gain a session ID from beoweb using SSH public key
Requires the paramiko library
Connects to the given beoweb auth host/port and sends the system
username and any found SSH public keys.
Return the Session ID upon success -- this auth method will not
result in beoweb setting the Session ID in a cookie, so the Session ID
is automatically added to the param string of all subsequent reqeusts
to Beoweb
:param auth_host: hostname of SSH server to connect to
:param auth_port: TCP port of SSH server to connect to
:returns: new session ID from Beoweb
"""
try:
from paramiko import SSHClient, AutoAddPolicy
from paramiko import AuthenticationException
except ImportError:
LOG.critical("Unable to import Paramiko library")
raise beowebexc.BeowebAuthError(
"Unable to import Paramiko library.")
try:
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
LOG.debug("SSH Auth connecting to %s:%s, user: %s",
auth_host, auth_port, self.user)
client.connect(auth_host,
port=auth_port,
username=self.user)
msg = client.get_transport().global_request("session_id",
wait=True)
client.close()
if msg:
sid = msg.get_string()
self.client.params.update({'session_id': sid})
else:
LOG.critical(("Authentication was successful, "
"but no session ID returned"))
raise beowebexc.BeowebAuthError(
"Unable to get session ID from beoweb")
return sid
except AuthenticationException:
raise beowebexc.BeowebAuthError(
"SSH Key authentication failed")
finally:
try:
client.close()
except:
pass
def release_success(self, build_info):
def progress(filename, transferred, total):
print(
colorize('fuchsia', ' * %s transfer in progress: %02d%%.\r' % (filename, transferred * 100 / total)),
end='\r'
)
ssh = SSHClient()
try:
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(self.ssh['server'], username=self.ssh['user'], key_filename=self.ssh['pkey'])
except SSHException:
raise XUtilsError('Unable to establish a SSH connection to %s' % self.ssh['server'])
dest_dir = '/'.join([
self.ssh['base_dir'], build_info['category'], build_info['pkg_name'], build_info['version'],
build_info['arch']
])
stdin, stdout, stderr = ssh.exec_command('mkdir -p %s' % dest_dir) # pylint: disable=unused-variable
if stderr.read():
raise XUtilsError('Unable to create directory %s on server %s' % (dest_dir, self.ssh['server']))
src_dir = str()
for f in listdir(self.cfg['build']['workdir']):
if f.endswith('_root.tar.gz'):
src_dir = self.cfg['build']['workdir']
if not src_dir:
src_dir = self.cfg['release']['archive_dir']
files = [
'%s-%s_root.tar.gz' % (build_info['pkg_name'], build_info['version']),
'%s-%s_debuginfo.tar.gz' % (build_info['pkg_name'], build_info['version']),
'%s-%s_root.tar.gz.gpg' % (build_info['pkg_name'], build_info['version']),
]
try:
sftp = SFTPClient.from_transport(ssh.get_transport())
sftp.chdir(dest_dir)
except SSHException:
raise XUtilsError('Unable to negotiate a SFTP session for %s' % self.ssh['server'])
for f in files:
filepath = realpath(src_dir + '/' + f)
if not exists(filepath):
self.info(colorize('yellow', '%s not found => skip it.' % f))
continue
my_callback = partial(progress, f)
remote_attr = sftp.put(filepath, f, callback=my_callback)
if stat(filepath).st_size == remote_attr.st_size:
self.info('%s successfully copied on server %s.' % (f, self.ssh['server']))
else:
raise XUtilsError('Copy of %s on server %s is corrupted.' % (f, self.ssh['server']))
sftp.close()
ssh.close()
def server_connect(server, data_from):
ssh = SSHClient()
ssh.load_system_host_keys()
username='******'
data_to='/volume1/dual_back_up'
password = getpass.getpass('Password admin@nas : ')
ssh.connect(server, username=username, password=password)
with SCPClient(ssh.get_transport()) as scp:
print 'Connected to {server} and copying data'.format(server=server)
print '\t',data_from,'to',server+'@'+data_to
scp.put(data_from, data_to, recursive=True, preserve_times=True)
def _get_ssh_clients(self):
"""
Returns an SSHClient and SFTPClient configured for use.
"""
ssh_client = SSHClient()
ssh_client.set_missing_host_key_policy(WarningPolicy())
ssh_client.connect(
self.archive.host,
username=self.archive.username,
pkey=self.archive.ssh_credentials.get_pkey())
sftp_client = SFTPClient.from_transport(
ssh_client.get_transport())
return ssh_client, sftp_client
def rmt_copy(self, hostname, username=None, password=None, send=False,
fname=None, remote_path=None):
"""Remote copy function retrieves files from specified host.
:param hostname: host name or ip address
:param username: user which will have privalege to copy files to and from system.
:param password: password for defined user.
:param send: flag to send files. by default copy from remote (send=false)
:type send: bool
:param fname: file name which to transport
:param remote_path: where to place the file on the other end.
"""
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(hostname, username=username, password=password)
if not send:
with closing(Read(ssh.get_transport(), remote_path)) as scp:
scp.receive_file(fname)
else:
with closing(Write(ssh.get_transport(), remote_path)) as scp:
scp.send_file(fname, send)
def scp_connect(server, username, password, col_log, col_type):
print ('Connecting to remote host ' + server)
ssh = SSHClient()
#ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(server, 22, username, password)
scp = SCPClient(ssh.get_transport())
except:
print ('Unable to connect, please check your credentials and hostname')
sys.exit()
# Copy files to localdir
if col_log == 'None' and col_type == 'vcd':
col_log = default_vcd_debug
col_log_info = default_vcd_info
# Create local directory to hold log files
try:
os.stat(tmp_dir)
print ('Local temp folder exists')
except:
os.mkdir(tmp_dir)
print ('Creating local temp folder')
#attempting to grab debug log
print ('Connected to remote host\n')
file_path = '/tmp/vmw_eps_calc/'
try:
print ('Attempting to copy DEBUG logs to your machine. This (~11M) can take a a bit depending on your connection....\n')
scp.get(col_log, file_path)
# Pass file location to file_list() after parsing it a bit to extract the actual file name
file_name_index = col_log.rfind('/')
file_name = col_log[file_name_index:]
local_file_path = tmp_dir + file_name
print ('File copied successfully')
file_list(local_file_path, col_type)
except:
print ('Error retrieving log file ' + col_log)
try:
print ('Now doing the info level logs...\n')
scp.get(col_log_info, file_path)
# Pass file location to file_list() after parsing it a bit to extract the actual file name
file_name_index = col_log_info.rfind('/')
file_name = col_log_info[file_name_index:]
local_file_path = tmp_dir + file_name
print ('File copied successfully')
file_list(local_file_path, col_type)
except:
print ('Error retrieving log file ' + col_log_info)
scp.close()
def download(source, dest=None):
"""
Download files from a remote SCP source
@param source_info: Source information such as the source to download from.
@type source_info: a dictionary
@param local_dest_dir: The local filename
@type local_dest_dir: str
@return: True and a list of file downloaded if successful. Otherwise False.
"""
try:
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
url = urlsplit(source['url'])
username = url.username
# Use the current user if one was not specified
if not username:
username = pwd.getpwuid(os.getuid())[0]
ssh.connect(url.hostname, port=url.port or 22, username=username, password=url.password)
scp = SCPClient(ssh.get_transport())
# Download file to a local temporary file if a local file path is not specified.
if not dest:
dest = tempfile.mkdtemp()
if os.path.exists(dest) and os.path.isdir(dest):
# TODO: we should somehow support scp://dir to scp://dir
# get filename from source
filename = os.path.basename(url.path)
if not filename:
fd, dest = tempfile.mkstemp(dir=dest)
else:
dest = os.path.join(dest, filename)
scp.get(url.path, dest, recursive=False)
ssh.close()
outputfile = {'url': dest,
'name': os.path.basename(url.path),
'content_type': 'application/octet-stream'
}
return [outputfile]
except SCPException:
LOG.error("Could not SCP file %s from %s to local destination\
%s as user %s", url.path, url.hostname, dest, username, exc_info=True)
raise
def _push_single(self,
test_name="",
data=None,
timestamp=None,
test_idx=None):
"""Copy the files over SCP using the provided configuration."""
if self.target is None:
logger.error("scp.target must be specified")
return
if self.username is None and self.private_key is None:
logging.error("scp.username or scp.private_key must be provided.")
try:
(dst_host, dst_path) = self.target.split(":")
except ValueError:
logger.error("scp.target must be 'host:/path/to/destination'")
return
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy)
try:
ssh.connect(
dst_host,
int(self.port),
username=self.username,
password=self.password,
timeout=defaults.SSH_TIMEOUT,
key_filename=self.private_key,
)
with SCPClient(ssh.get_transport()) as scp:
filename = self._generate_filename(test_name, timestamp,
test_idx)
dst_path = os.path.join(dst_path, filename)
logger.info("Copying data to %s", dst_path)
buf = io.StringIO(data)
buf.seek(0)
scp.putfo(buf, dst_path)
except Exception as err:
logger.error("SCP exporter failed: %s", err)
finally:
ssh.close()
def grab(
hostname: str,
username: str = ScpGrabberDefaultValues.SSH_USER,
password: str = ScpGrabberDefaultValues.SSH_PASSWORD,
port: int = ScpGrabberDefaultValues.SSH_PORT,
) -> str or None:
"""
Grab some file from SSH with SCP
:param hostname: host to connect
:param username: username to connect with
:param password: password to connect with
:param port: port to connect
:return: str status
"""
if check_if_exist(hostname):
return
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.load_system_host_keys()
is_connect_successful = "connect not successful"
try:
ssh.connect(
hostname=hostname,
username=username,
password=password,
port=port,
auth_timeout=ScpGrabberDefaultValues.SSH_AUTH_TIMEOUT_SEC,
timeout=ScpGrabberDefaultValues.SSH_TIMEOUT_SEC,
)
is_connect_successful = "connect is successful"
with SCPClient(ssh.get_transport()) as scp:
scp.get(
remote_path=ScpGrabberDefaultValues.SCP_REMOTE_PATH,
local_path=
f"{str(ScpGrabberDefaultValues.RESULTS_DIRECTORY)}/{hostname}_{ScpGrabberDefaultValues.SCP_FILENAME_POSTFIX}",
)
return is_connect_successful
except (AuthenticationException, NoValidConnectionsError, SSHException,
timeout):
ssh.close()
return is_connect_successful
except:
ssh.close()
return is_connect_successful
def interact_with_aws(folder,
project=None,
username=None,
password=None,
get=False):
if not username and not password or not project:
logging.error(
"Make sure your username, password, and project are set.")
return
ssh_client = SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(AutoAddPolicy())
ssh_client.connect(server, username=username, password=password)
with SCPClient(ssh_client.get_transport()) as scp_client:
if get:
scp_client.get(f'/srv/www/{project}/', project, recursive=True)
else:
scp_client.put(folder, f'/srv/www/{project}/', recursive=True)
class SshRemoteExecutor(RemoteExecutor):
"""
implements RemoteExecutor using SSH as the remote execution client
"""
def _execute(self, command, block_for_response=True):
_, stdout, stderr = self.remote_client.exec_command(command)
if block_for_response:
stdout_output = stdout.read().decode().strip()
stderr_output = stderr.read().decode().strip()
return {
'exit_code': stdout.channel.recv_exit_status(),
'stdout': stdout_output,
'stderr': stderr_output,
}
return None
def connect(self):
try:
self.remote_client = SSHClient()
self.remote_client.set_missing_host_key_policy(AutoAddPolicy())
self.remote_client.connect(
self.hostname,
username=self.username,
password=self.password,
port=self.port,
pkey=PKey(self.private_key) if self.private_key else None,
key_filename=self.private_key_file_path,
)
except AuthenticationException as e:
raise RemoteExecutor.AuthenticationException(str(e))
except NoValidConnectionsError as e:
raise RemoteExecutor.NoValidConnectionException(str(e))
except Exception as e:
raise RemoteExecutor.ConnectionException(str(e))
def _close(self):
self.remote_client.close()
def is_connected(self):
return self.remote_client and self.remote_client.get_transport(
) is not None
def SFTP_script(ip, dire, origi, dest):
client = SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(ip, port=22, username='******', password=None, pkey=None, key_filename='key.pem', look_for_keys=True)
t = client.get_transport()
sftp = SFTPClient.from_transport(t)
if (dire == 1):
for i in range(len(dest)):
if (VL > 4):
print("Get. Getting files(" + origi[i] + ") from " + ip)
sftp.get(origi[i], dest[i])
else:
for i in range(len(dest)):
if (VL > 4):
print("Send. Sending files(" + origi[i] + ") to " + ip)
sftp.put(origi[i], dest[i])
def copy(self):
"""Push it !"""
from paramiko import SSHClient
from scp import SCPClient
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(self.destination.hostname,
username=self.destination.username)
LOGGER.debug('hostname %s', self.destination.hostname)
LOGGER.debug('dest path %s ', os.path.dirname(self.destination.path))
LOGGER.debug('origin %s ', self.origin)
scp = SCPClient(ssh.get_transport())
scp.put(self.origin, self.destination.path)
scp.close()
ssh.close()
def download_log():
global SSH, OLD_LOG
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
try:
ssh.connect(SSH['host'], username=SSH['user'], password=SSH['pass'])
except:
return 'SSH fail'
while True:
stdin, stdout, stderr = ssh.exec_command('ls ' + SSH['path_log'])
filelist = stdout.read().splitlines()
for filename in filelist:
filename = ''.join([chr(c) for c in filename])
if filename in [os.path.basename(i) for i in OLD_LOG]:
continue
transport = ssh.get_transport()
sftp = SFTPClient.from_transport(transport)
sftp.get(SSH['path_log'] + '/' + filename, './resource/' + filename)
print('[+] Copied ' + filename)
def fetch_remote_data(
ticker='AAPL'): # loads in all historical data for provided ticker
print("Loading price data for %s..." % ticker)
server_ip = '192.168.1.17' # ip where StockSpider is currently running
server_user = '******' # username of device running StockSpider
server_path = '/home/pi/Desktop/StockSpider' # path where StockSpider is installed on server
scp = None # client interface to remote server
server_password = input("Enter server password: "******"Took %d seconds to fetch %s.tsv" % (time() - t0, ticker))
return '%s.tsv' % ticker
def send_image_to_edge(self):
ssh = SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
#ssh.load_system_host_keys(os.path.expanduser("~/.ssh/known_hosts"))
ssh.connect(self.test_config["edge"]["hostname"],
username=self.test_config["edge"]["login"],
password=self.test_config["edge"]["password"])
# SCPCLient takes a paramiko transport as its only argument
scp = SCPClient(ssh.get_transport())
for file in os.listdir('/tmp/' + self.current_build):
if fnmatch.fnmatch(
file, "edge-imageupdate-" +
self.test_config["edge"]["model"] + "*" + ".zip"):
scp.put("/tmp/" + self.current_build + "/" + file, '~/')
print "filematch:" + file
self.edge_image_file = file
scp.close()
ssh.close()
def sync_ssh():
logger.info("Uploading backups to SSH destination")
files = glob.glob(os.path.join(settings.LOCAL_DESTINATION, "*.zip"))
client = SSHClient()
scp = None
try:
client.load_system_host_keys()
client.set_missing_host_key_policy(AutoAddPolicy())
client.connect(
settings.SSH_HOST,
port=settings.SSH_PORT,
username=settings.SSH_USER,
key_filename=settings.SSH_KEY,
look_for_keys=True,
timeout=5000,
)
stdin, stdout, stderr = client.exec_command(
f"mkdir -p {settings.SSH_DESTINATION}"
)
if stdout.channel.recv_exit_status():
raise Exception("mkdir -p failed")
scp = SCPClient(client.get_transport())
for file in files:
scp.put(file, remote_path=settings.SSH_DESTINATION)
logger.info("Cleaning up remote directory")
stdin, stdout, stderr = client.exec_command(
f"ls {settings.SSH_DESTINATION}/*.zip"
)
if stdout.channel.recv_exit_status():
raise Exception("Failed to list remote directory")
basenames = [basename(x) for x in files]
for rfile in stdout:
rfile = rfile.strip()
if basename(rfile) not in basenames:
logger.info(f"Deleting old remote backup {rfile}")
client.exec_command(f"rm {rfile}")
except Exception as e:
logger.error("SSH syncing failed! Exiting.", exc_info=e)
os._exit(1)
finally:
client.close()
if scp:
scp.close()
def deploy(ip, username, password, arduino_sketch_file=None):
print("==== Deploying to {} ====".format(ip.ljust(15)))
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
try:
ssh.connect(ip, SSH_PORT, username, password)
# Create deploy path
cmd = 'mkdir -p {}'.format(DEPLOY_PATH)
(stdin, stdout, stderr) = ssh.exec_command(cmd)
if stdout.channel.recv_exit_status() != 0:
raise DeployException('Could not execute {}'.format(cmd))
# Copy all files
print("Copying files..")
sftp = ssh.open_sftp()
def copy_file(local_path, remote_path):
print('{} --> {}:{}'.format(local_path.ljust(30), ip, remote_path))
sftp.put(local_path, remote_path, confirm=True)
for local_path in DEPLOY_FILES:
filename = path.basename(local_path)
remote_path = DEPLOY_PATH + filename
copy_file(local_path, remote_path)
if arduino_sketch_file is not None:
copy_file(path.join(path.dirname(arduino_sketch_file), 'Makefile'),
DEPLOY_PATH + 'Makefile')
copy_file(arduino_sketch_file, DEPLOY_PATH + 'sketch.ino')
print("Running setup script..")
# Run setup script, pipe output back?
channel = ssh.get_transport().open_session()
channel.exec_command(POST_DEPLOY_COMMAND)
while not channel.exit_status_ready():
if channel.recv_stderr_ready():
os.write(sys.stderr.fileno(), channel.recv_stderr(4096))
if channel.recv_ready():
os.write(sys.stdout.fileno(), channel.recv(4096))
except SSHException as e:
raise DeployException(e.value)
finally:
ssh.close()
def scp_default_video_from_tank():
"""
:author: Krishnendu Banerjee.
:date: 29/11/2019.
:description: Function that copies the default video files from the Tank to the local tool
:access: public.
:form: templates/load_default_library.html
:form_input: username, password
:return: List of Files on Success or Failure (errorchecker.py)
"""
# Taking the host/file/user credentials input from the user input form
username = request.form.get('username')
password = request.form.get('password')
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Default File Lists to be copied. Retrieving from the config.
filelist = movie_config.default_file_list
try:
ssh.connect(tank_hostname,
username=username,
password=password,
look_for_keys=False)
except:
return errorchecker.upload_authentication_error()
for filename in filelist:
try:
with SCPClient(ssh.get_transport()) as scp:
try:
scp.get(
'/ifs/PDLTankTest/Providers/BSS/Content/Distribution/TestFiles/'
+ filename,
UPLOAD_DIRECTORY + '/',
recursive=True)
except FileNotFoundError:
return errorchecker.upload_filenotfound_error(filename)
except:
return errorchecker.upload_authentication_error()
return api_media_library.list_files_checksum()
def upload_to_pi(pi):
if "user_id" not in session:
return redirect(url_for('login'))
if request.method == 'POST':
if 'file' not in request.files:
flash('No file part', category="warning")
return redirect(request.url)
pi_name = pi_func.PI[pi]["hostname"]
file = request.files['file']
if file.filename == '':
flash('No selected file', category="warning")
return redirect(request.url)
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
if not os.path.isdir(app.config['UPLOAD_FOLDER'] + '/' + pi_name):
os.mkdir(app.config['UPLOAD_FOLDER'] + '/' + pi_name)
file.save(
os.path.join(app.config['UPLOAD_FOLDER'] + '/' + pi_name,
filename))
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy)
ssh.load_system_host_keys()
ssh.connect(hostname=pi_func.PI[pi]["ip"],
username=session["user_id"],
password=session["user_id"])
scp = SCPClient(ssh.get_transport())
scp.put(
app.config['UPLOAD_FOLDER'] + '/' + pi_name + '/' + filename,
'~/' + filename)
scp.close()
ssh.close()
flash('Upload Successful', category="success")
return redirect(request.referrer)
return redirect(request.referrer)
else:
return 401
class SSH:
def __init__(self, host, user, password):
self._client = SSHClient()
self._client.set_missing_host_key_policy(AutoAddPolicy())
# self._client.load_system_host_keys()
self._client.connect(hostname=host,
password=password,
username=user)
self._scp = scp.SCPClient(self._client.get_transport())
def execute(self, cmd, wait=True):
_, stdout, stderr = self._client.exec_command(cmd)
if not wait:
return stdout, stderr
return stdout.read(), stderr.read()
def copy(self, local, remote):
self._scp.put(local, remote_path=remote, recursive=True)
def transfer_files(self, ip, files, destinations):
"""
Connects to ip address and transports specified local files to a
destination on the connected server.
"""
# Create SSH client
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
# Lock all other threads so that the pem file is being accessed one at a time
lock = Lock()
lock.acquire()
client.connect(ip,
username=self.username,
key_filename=self.pem_location)
lock.release()
# Open an SCP client with SSH connection
with SCPClient(client.get_transport()) as scp_conn:
# Transfer each file to their respective destination
for file, destination in zip(files, destinations):
scp_conn.put(file, destination)
def upload_ca():
# use sh instead of bash
env.shell = '/bin/sh -c'
# make sure /etc/ssl/certs exists
run('mkdir -p /etc/ssl/certs')
# install the root ca
# cannot use 'put' as sftp is not available
ssh = SSHClient()
ssh.load_system_host_keys()
for host in env.hosts:
if env.key_filename is None:
ssh.connect(host, username=env.user, password=env.password)
else:
ssh.connect(host)
scp = SCPClient(ssh.get_transport())
scp.put('DigiCertHighAssuranceEVRootCA.pem', '/etc/ssl/certs')
# create the symbolic links normally created by OpenSSL's c_rehash
with cd('/etc/ssl/certs'):
run('ln -s DigiCertHighAssuranceEVRootCA.pem 244b5494.0')
run('ln -s DigiCertHighAssuranceEVRootCA.pem 81b9768f.0')
def fileCheck(i):
#Check for file in the directory and transfer them to the server if it exists
fileExist = False
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect('<static ip address of your raspberry pi>',username = '******', password = '******')
# SCPCLient takes a paramiko transport as an argument
scp = SCPClient(ssh.get_transport())
command = 'ls /home/pi/Downloads/face-detect-send'
(stdin, stdout, stderr) = ssh.exec_command(command)
for line in stdout.readlines():
scp.get('/home/pi/Downloads/face-detect-send/' + 'key{}'.format(i))
scp.get('/home/pi/Downloads/face-detect-send/' + 'faceCount{}'.format(i))
scp.get('/home/pi/Downloads/face-detect-send/' + 'image{}'.format(i))
scp.get('/home/pi/Downloads/face-detect-send/' + 'associated_data{}'.format(i))
fileExist = True
scp.close()
return fileExist
def copySCPfiles(hashed_file_name):
try:
ScreenOutput("Copying Test", "To Server")
time.sleep(3)
hashed_file_path = log_files + "/" + hashed_file_name
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname, username="******")
scp = SCPClient(ssh.get_transport())
scp.put(hashed_file_path, hashed_file_path)
scp.close()
ScreenOutput('Test Copied', 'To Server')
time.sleep(1)
except Exception, e:
ScreenOutput("Copy To", "Server Failed")
time.sleep(3)
ScreenOutput("Restarting", "Speed Test")
time.sleep(2)
executeTesting()
def scp(self, src, dst):
"""Copy file to A8 node using Parallel SSH copy_file"""
result = {"0": [], "1": []}
sites = ['{}.iot-lab.info'.format(site) for site in self.groups]
for site in sites:
try:
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(site, username=self.config_ssh['user'], timeout=10)
except AuthenticationException:
raise OpenA8SshAuthenticationException(site)
except ConnectionErrorException:
result["1"].append(site)
else:
with SCPClient(ssh.get_transport()) as scp:
scp.put(src, dst)
ssh.close()
result["0"].append(site)
return _cleanup_result(result)
def apply_scp(source, destination):
''' we can use scp to copy from the remote server to the working directory '''
## source is the folder with path: we wnat to copy from
## destination is the folder with path: where we want to send the files
ssh = SSHClient()
ssh.load_system_host_keys()
#import pdb;pdb.set_trace()
ssh.connect("rackham.uppmax.uu.se",
username="******",
password="******")
# SCPCLient takes a paramiko transport as an argument
scp = SCPClient(ssh.get_transport(), progress4=progress4)
##
# Uploading the 'test' directory with its content in the
# '/home/user/dump' remote directory
scp.put(source, recursive=True, remote_path=destination)
scp.close()
def scp_to_pc(source,
target='NEWCOOL',
hostname='kimjg.astro.princeton.edu',
username='******'):
"""Function to copy files to my directory
"""
from paramiko import SSHClient
from scp import SCPClient
target = '~/Dropbox/Apps/Overleaf/{0:s}/figures'.format(target)
try:
client = SSHClient()
client.load_system_host_keys()
client.connect(hostname, username=username)
with SCPClient(client.get_transport()) as scp:
scp.put(source, target)
finally:
if client:
client.close()
def connect(ip, password):
try:
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
print('Initiating SSH to {}...'.format(ip))
ssh.connect(ip,
port=2222,
username="******",
password=password,
look_for_keys=False)
# use keepalive to keep connection open to DNAC with idle timeout=5mins
t = ssh.get_transport()
t.set_keepalive(5)
return ssh
except AuthenticationException:
print(
"Unable to SSH to the device {} due to an Authentication Exception"
.format(ip))
def secure_copy(hostname, username, password, filename, destination="~/"):
# try:
# #make sure in the above command that username and hostname are according to your server
# scp_command = 'sudo scp ./test.py '+ username+ "@" + hostname + ':~/Desktop/'
# # print(scp_command)
# child = pexpect.spawn(scp_command)
# child.expect(["assword:"])
# child.sendline(password)
# child.expect(pexpect.EOF, timeout=2)
# except Exception as e:
# print("Oops Something went wrong buddy")
# print(e)
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(WarningPolicy)
ssh.connect(hostname, username=username, password=password)
with SCPClient(ssh.get_transport()) as scp:
scp.put(filename, destination) # Copy to the server
def update(ctx):
config_dir = '/mnt/data/go/godata/config'
plugin_dir = '/mnt/data/go/godata/plugins/external'
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(os.environ['SWARM_HOST'], username='******', key_filename='./bootstrap.pem')
# deploy plugins and config
with SCPClient(ssh.get_transport()) as scp:
scp.put('cruise-config.xml', config_dir + '/cruise-config.xml')
scp.put('password.properties', config_dir + '/password.properties')
if not set(ssh.open_sftp().listdir('/mnt/data/go/godata/plugins/external'))==set(os.listdir('plugins')):
ssh.exec_command("sudo rm -r {}/*".format(plugin_dir))
for item in os.listdir('plugins'):
scp.put(os.path.join('plugins', item), os.path.join(plugin_dir + '/', item))
cmd="sudo docker kill $(sudo docker ps | grep \'{}\' | awk \'{{print $1}}\')".format(os.environ['GOCDSERVER_IMAGE'])
# print(cmd)
ssh.exec_command(cmd)
ssh.exec_command("sudo docker kill $GOID")
def scp_file_from_tank(tank_path, filename):
"""
:author: Krishnendu Banerjee.
:date: 29/11/2019.
:description: Function that copies specific files from the Tank to the local tool
:access: public
:form: templates/upload_files.html
:form_input: username, password
:param tank_path: This input parameter sends the manual override input for the directory or location of Tank
:param filename: This input parameter sends filename to the function
:return: List of Files on Success or Failure (errorchecker.py)
"""
# Taking the host/file/user credentials input from the user input form
username = request.form.get('username')
password = request.form.get('password')
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(tank_hostname,
username=username,
password=password,
look_for_keys=False)
except:
return errorchecker.upload_authentication_error()
try:
with SCPClient(
ssh.get_transport(),
sanitize=lambda x: x) as scp: # Handling Wildcard Transfers
try:
scp.get(tank_path + '/' + filename[:-1] + '*',
UPLOAD_DIRECTORY + '/',
recursive=True)
except FileNotFoundError:
return errorchecker.upload_filenotfound_error(filename)
except:
return errorchecker.upload_authentication_error()
return api_media_library.list_files_checksum()
def transfer_data(df, worker, now):
this_dir = os.path.dirname(os.path.abspath(__file__))
user_folder = os.path.join(this_dir, "user_dfs")
if not os.path.exists(user_folder):
os.makedirs(user_folder)
filename = "user_df_" + now + ".json"
filepath = os.path.join(user_folder, filename)
store_dataframe_as_json(df, filepath)
if worker != "local":
if worker == "cori":
o = subprocess.check_output(
['bash', '-c', '. ~/.bash_profile; cori_get_password'])
user = os.environ["CORI_USER"]
host = "lrc-login.lbl.gov"
elif worker == "lrc":
o = subprocess.check_output(
['bash', '-c', '. ~/.bash_profile; lrc_get_password'])
user = os.environ["LRC_USER"]
host = "lrc-login.lbl.gov"
else:
raise ValueError(f"Worker {worker} not valid!")
o_utf = o.decode("utf-8")
o_all = o_utf.split("\n")
o_all.remove("")
password = o_all[-1]
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(host,
username=user,
password=password,
look_for_keys=False)
with SCPClient(ssh.get_transport()) as scp:
scp.put(filepath,
recursive=True,
remote_path="/global/home/users/ardunn")
else:
pass
def update_database():
if os.uname()[1] in ('workstation', ):
print("Actualizando copia local de la base de datos...")
host = 'ha-remote'
ssh = SSHClient()
ssh_config_file = os.path.expanduser("~/.ssh/config")
if os.path.exists(ssh_config_file):
conf = SSHConfig()
with open(ssh_config_file) as f:
conf.parse(f)
host_config = conf.lookup(host)
proxy = ProxyCommand("ssh {}@{} -p {} nc {} 22".format(
"pi", "remote-ssh.proxy.host", 2222, 'homeassistant-host'))
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
#ssh.connect(host, username=user, pkey=pkey, sock=proxy)
ssh.connect(host, username=host_config['user'], sock=proxy)
with SCPClient(ssh.get_transport()) as scp:
src = '/home/homeassistant/devel/hamlfeeder/database.sqlite'
dst = '/home/roberto/devel/python/hamlfeeder/database.sqlite'
scp.get(src, dst)
def download():
remote_folder_name = REMOTE_PATH.split('/')[-1]
print("INFO - Connecting to Raspberry pi...")
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(hostname=HOSTNAME, username=USERNAME, password=PASSWORD)
print("INFO - Downloading {} folder on {}...".format(
remote_folder_name,
LOCAL_PATH.split('/')[-1]))
with SCPClient(ssh.get_transport()) as scp:
scp.get(remote_path=REMOTE_PATH, local_path=LOCAL_PATH, recursive=True)
print("DONE - {} downloaded").format(remote_folder_name)
print("Removing {} folder on raspberry...").format(remote_folder_name)
with ssh.open_sftp() as sftp:
rm(sftp, REMOTE_PATH)
print("DONE - {} folder deleted remotely.").format(remote_folder_name)
ssh.close()
def connect(self):
ssh = SSHClient()
config = SSHConfig()
with open(os.path.expanduser("~/.ssh/config")) as _file:
config.parse(_file)
host_config = config.lookup(self.host)
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.load_system_host_keys()
ssh.connect(
self.host,
username=self.user,
password=self.password,
key_filename=host_config["identityfile"][0],
allow_agent=False,
timeout=30,
)
transport = ssh.get_transport()
channel = transport.open_session()
channel.setblocking(1)
return ssh
def scp_pix_distrib_files(policyparams, agentparams):
ssh = SSHClient()
ssh.load_system_host_keys()
# ssh.connect(hostname='newton4', username='******')
ssh.connect(hostname='deepthought', username='******')
# SCPCLient takes a paramiko transport as its only argument
scp = SCPClient(ssh.get_transport())
print('scp pred.pkl')
scp.get(policyparams['current_dir'] + '/verbose/pred.pkl',
policyparams['current_dir'] + '/verbose/pred.pkl')
# for i in range(1,agentparams['T']):
# filename = '/verbose/gen_distrib_t{}.pkl'.format(i)
# scp.get(policyparams['current_dir']+filename, policyparams['current_dir']+filename)
#
# filename = '/verbose/gen_image_t{}.pkl'.format(i)
# scp.get(policyparams['current_dir'] + filename, policyparams['current_dir'] + filename)
scp.close()
