def put_file(self,mac_addr,function_name):
scp = None
file_path = self._lookup_function_files(function_name)
print file_path
if(os.path.exists(file_path)):
if(os.path.isdir(file_path)):
if(self.scp_clients.has_key(mac_addr)):
ssh = self.scp_clients[mac_addr]
scp = SCPClient(ssh.get_transport())
print scp
print file_path
try:
scp.put(file_path,recursive=True)
except:
print "SCPException"
#files = os.listdir(file_path)
## assumes no directory in the element dir
#for item in files:
# print item
# scp.put(file_path+'/'+item) #Not portable use os.path.join
if(os.path.isfile(file_path)):
if(self.scp_client.has_key(mac_addr)):
scp = self.scp_clients[mac_addr]
scp.put(file_path)
pass
return True
def remote_exec_file(script, host, port, user, password, test_name, result_query, test_detail, log_file):
start = time.asctime()
cl = paramiko.SSHClient()
cl.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
cc = cl.connect(host, port=port, username=user, password=password)
except paramiko.ssh_exception.AuthenticationException:
print "Auth Error"
except paramiko.ssh_exception.SSHException:
print "Protocol Error"
except paramiko.transport:
print "General Error"
except socket.error:
print "Socket Error"
scp = SCPClient(cl.get_transport())
scp.put(script,script)
cl.exec_command("chmod +x ./{0}".format(script))
stdin, stdout, stderr = cl.exec_command("./{0}".format(script))
a = stdout.readlines()
cmd = str(a)
if result_query not in cmd:
print "Failed {0}\n".format(test_name)
print "Test detail: \n" + test_detail + "\n"
print "Result: \n" + cmd + "\n"
stop = time.asctime()
test_logger(log_file, test_name, start, stop, "Fail")
if result_query in cmd:
print "Pass {0}".format(test_name)
print "Test detail: \n" + test_detail + "\n"
print "Result: \n" + cmd + "\n"
stop = time.asctime()
test_logger(log_file, test_name, start, stop, "Pass")
def put(self, files, remote_path=None, out_stream=sys.stdout, verbose=False):
"""
Copy a file from the local system to the remote system.
:param files:
:param remote_path:
:param out_stream:
:param verbose:
:return: :rtype:
"""
if remote_path is None:
remote_path = files
self.display("scp '{src}' '{dest}'".format(src=files, dest=remote_path),
out_stream=out_stream, verbose=verbose)
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(Project.address, Project.port, Project.user, Project.password)
scp = SCPClient(ssh.get_transport())
# scp = SCPClient(self.ssh.get_transport())
# noinspection PyBroadException
try:
info("\nfiles: %s" % repr(files))
info("remote_path: %s" % remote_path)
output = scp.put(files, '"{dest}"'.format(dest=remote_path), recursive=True) or ''
except Exception:
try:
output = scp.put(files, remote_path, recursive=True) or ''
except Exception as ex:
output = str(ex)
self.display("\n" + output, out_stream=out_stream, verbose=verbose)
return output
def scp_file(ip, uname, pswd, image): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.load_system_host_keys() ssh.connect(ip, username=uname, password=pswd) scp = SCPClient(ssh.get_transport()) scp.put(image, '/tmp/flashimg')
def get_file(remote_file, local_path, ip, username, password, logger=None):
# Получает с удаленной машины файл remote_file с помощью scp и сохраняет его в local_path.
if local_path[len(local_path) - 1] != '/': local_path += '/'
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.load_system_host_keys()
if logger is not None: logger.info("SCP GET: connecting to %s" % (ip))
try:
ssh.connect(ip, username=username, password=password)
except:
if logger is not None: logger.info("SCP GET: failed to connect to %s" % (ip))
return False
else:
if logger is not None: logger.info("SCP GET: connected to %s" % (ip))
try:
if logger is not None: logger.info("SCP GET: retrieving file %s" % (remote_file))
scp = SCPClient(ssh.get_transport())
scp.get(remote_file, local_path)
except:
if logger is not None: logger.error("SCP GET: error: failed to retrieve file %s" % (remote_file))
ssh.close()
return False
else:
if logger is not None: logger.info("SCP GET: file saved to %s folder" % (local_path))
ssh.close()
return True
def run_main(args):
if not os.path.exists(SCRIPT_FILENAME):
print 'ERROR: Please run "solo script pack" first to bundle your archive.'
return 1
print 'connecting to solo...'
solo = soloutils.connect_solo(await=True)
scp = SCPClient(solo.get_transport())
# Requires pip
print 'checking pip...'
if soloutils.install_pip.run(solo, scp) != 0:
print 'failed installing pip.'
sys.exit(1)
push(solo, scp, '--force' in args['<arg>'])
print 'running script...'
print ''
soloutils.command_stream(solo, '''
set -e
cd /log/solo-script
source ./env/bin/activate
exec python /log/solo-script/''' + args['<arg>'][1] + '''
''')
scp.close()
solo.close()
def upload_test(self, filenames, recursive, expected=[]):
destination = b'/tmp/upp\xC3\xA9' + next(unique_names)
chan = self.ssh.get_transport().open_session()
chan.exec_command(b'mkdir ' + destination)
assert chan.recv_exit_status() == 0
previous = os.getcwd()
try:
os.chdir(self._temp)
scp = SCPClient(self.ssh.get_transport())
scp.put(filenames, destination, recursive)
chan = self.ssh.get_transport().open_session()
chan.exec_command(
b'echo -ne "' +
destination.decode('iso-8859-1')
.encode('ascii', 'backslashreplace') +
b'" | xargs find')
out_list = b''
while True:
data = chan.recv(1024)
if not data:
break
out_list += data
prefix = len(destination) + 1
out_list = [l[prefix:] for l in out_list.splitlines()
if len(l) > prefix]
self.assertEqual(normalize_paths(out_list), set(expected))
finally:
os.chdir(previous)
chan = self.ssh.get_transport().open_session()
chan.exec_command(b'rm -Rf ' + destination)
assert chan.recv_exit_status() == 0
def download_test(self, filename, recursive, destination=None,
expected_win=[], expected_posix=[]):
# Make a temporary directory
temp = tempfile.mkdtemp(prefix='scp-py_test_')
# Add some unicode in the path
if WINDOWS:
if isinstance(temp, bytes):
temp = temp.decode(sys.getfilesystemencoding())
temp_in = os.path.join(temp, u'cl\xE9')
else:
if not isinstance(temp, bytes):
temp = temp.encode('utf-8')
temp_in = os.path.join(temp, b'cl\xC3\xA9')
previous = os.getcwd()
os.mkdir(temp_in)
os.chdir(temp_in)
try:
scp = SCPClient(self.ssh.get_transport())
scp.get(filename, destination if destination is not None else u'.',
preserve_times=True, recursive=recursive)
actual = []
def listdir(path, fpath):
for name in os.listdir(fpath):
fname = os.path.join(fpath, name)
actual.append(os.path.join(path, name))
if os.path.isdir(fname):
listdir(name, fname)
listdir(u'' if WINDOWS else b'',
u'.' if WINDOWS else b'.')
self.assertEqual(normalize_paths(actual),
set(expected_win if WINDOWS else expected_posix))
finally:
os.chdir(previous)
shutil.rmtree(temp)
def _queue_worker(rc):
# multithreaded file puller, takes tuples of remote, local, item, items_done
# pulls the files and then updates the progress meter
jenkins_host = composite['jenkins_host']
client = rc.ssh_client
client.connect(jenkins_host, username=credentials['jenkins-result']['username'],
password=credentials['jenkins-result']['password'],
timeout=10,
allow_agent=False,
look_for_keys=False,
gss_auth=False)
scp = None
while True:
source, destination, item, items_done = rc._queue.get()
destination = local(destination)
destination_dir = local(destination.dirname)
destination_dir.ensure(dir=True)
if not destination.check():
if scp is None:
scp = SCPClient(client.get_transport())
try:
scp.get(source, destination.strpath)
except SCPException:
# remote destination didn't exist
pass
except (SSHException, socket.timeout):
# SSH blew up :(
rc._queue.put((source, destination, item, items_done))
rc._queue.task_done()
continue
rc._progress_update(item, items_done)
rc._queue.task_done()
def scp2(Key, arg, wline, swline):
try:
ssh = paramiko.SSHClient()
key = paramiko.RSAKey.from_private_key_file(key_file)
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
for ip in ips:
ip = ip.strip()
Redis.lpush(Key, 'sync ' + line + ' to ' + ip + '\n')
ssh.connect(ip, 22, username, pkey=key, timeout=30)
scp = SCPClient(ssh.get_transport())
cmd = 'mkdir -p ' + wline
stdin, stdout, stderr = ssh.exec_command(cmd)
if os.path.isdir(swline):
scp.put(swline, wline, recursive=True)
else:
scp.put(swline, swline)
ssh.close()
if arg == 'publish':
if os.path.isdir(swline):
dirs = create_paths(swline)
for path in dirs:
Verify(Key, ip, path)
else:
Verify(Key, ip, swline)
except Exception as e:
Redis.lpush(Key, 'scp2:{0} fail'.format(e))
ssh.close()
def Update(sip, S_md5, App, path):
ssh = paramiko.SSHClient()
key = paramiko.RSAKey.from_private_key_file(key_file)
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
for ip in sip:
ip = ip.strip()
Redis.lpush(Key, '-' * 60)
Redis.lpush(Key, 'publish ' + path + ' to ' + ip + '\n')
try:
ssh.connect(ip, 22, username, pkey=key, timeout=15)
cmd = "mkdir -p {0}".format(web_path)
ssh.exec_command(cmd)
scp = SCPClient(ssh.get_transport())
scp.put('%s%s.zip' % (web_path, App), web_path)
cmd = '/usr/bin/md5sum %s/%s.zip' % (web_path, App)
stdin, stdout, stderr = ssh.exec_command(cmd)
R_md5 = stdout.read().split()[0]
if R_md5 == S_md5:
Redis.lpush(Key, '%s md5 verify --->pass!' % App)
Redis.lpush(Key, 'unzip start ......')
cmds = ['cd %s && /usr/bin/unzip -qo %s.zip && /bin/rm -f %s.zip' % (web_path, App, App),
'[ -e %s ] && echo ok' % tag_path]
for cmd in cmds:
stdin, stdout, stderr = ssh.exec_command(cmd)
result_zip = stdout.read().strip()
if result_zip == 'ok':
Redis.lpush(Key, '%s --->unzip success!' % App)
Redis.lpush(Key, 'backup start ......')
cmds = ('/bin/mkdir -p %s' % bak_path,
'[ -e %s/%s ] && /bin/rm -rf %s/%s' % (bak_path, App, bak_path, App),
'[ -e %s%s ] && /bin/mv %s%s %s/%s' % (web_path, App, web_path, App, bak_path, App))
for cmd in cmds:
stdin, stdout, stderr = ssh.exec_command(cmd)
result_backup = stderr.read()
if result_backup:
Redis.lpush(Key, result_backup)
Redis.lpush(Key, '%s --->backup fail !' % App)
else:
Redis.lpush(Key, '%s --->backup success!' % App)
Redis.lpush(Key, 'publish start ......')
cmd = '/bin/mv %s %s%s' % (tag_path, web_path, App)
stdin, stdout, stderr = ssh.exec_command(cmd)
result_rsync = stderr.read()
if result_rsync:
Redis.lpush(Key, result_rsync)
Redis.lpush(Key, '%s --->publish fail !' % App)
else:
Redis.lpush(Key, '%s --->publish success!' % App)
ssh.close()
Redis.lpush(Key, '-' * 60)
else:
Redis.lpush(Key, '%s --->unzip fail !' % App)
else:
Redis.lpush(Key, 'S_MD5:%r' % S_md5)
Redis.lpush(Key, 'R_MD5:%r' % R_md5)
Redis.lpush(Key, ' verify %s on %s fail !!!' % (App, ip))
except Exception as e:
Redis.lpush(Key, 'Update:{0}'.format(e))
ssh.close()
sys.exit()
class Target:
def __init__(self, hostname, uname, pword, num, port=22):
self.hostname = hostname
self.uname = uname
self.pword = pword
self.target_num = num
self.port = port
self.ssh = None
self.is_open = False
self.scp = None
def conn(self):
# print("Opening SSH connection to target...")
self.ssh = paramiko.SSHClient() # use ssh.exec_command("") to perform an action.
self.ssh.load_system_host_keys()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(self.hostname, port=self.port, username=self.uname, password=self.pword)
self.scp = SCPClient(self.ssh.get_transport()) # don't call this, but use the above function instead.
self.is_open = True
# TODO: fix rm -rf bug
def scpFiles(self, filename, a, recur=True): # call this with a filename and false if it is a single file
print "files: "
print (a)
self.ssh.exec_command("rm ", a)
self.scp.put(a, recursive=recur)
def close(self):
self.is_open = False
self.ssh.close()
def create(tpr, cluster, shell, job_name="workdir", duration="24:00:00", nodes=1, processes=16, script=None, partition=None, ntasks_per_node=16):
"""
- Argument validation
- Copy from cwd to locker (on local machine)
- Copy from local locker to remote locker
- Submit
"""
if not os.path.exists(configuration.config.lockers): os.mkdir(configuration.config.lockers)
# Create workdir, copy files over there
assert not os.path.exists("workdir")
id0 = str(uuid.uuid4())
workdir = "%s/%s" % (configuration.config.lockers, id0)
local_dir = os.getcwd()
ignore = shutil.ignore_patterns("\#*", "workdir*", "analysis*", "test*", "trash*")
print("local copy:", "src=", local_dir, "dst=", workdir)
shutil.copytree(local_dir, workdir, symlinks=False, ignore=ignore)
os.symlink(workdir, "workdir")
# Setup SSH client to copy files over via scp
dst = "%s/.lockers/" % (cluster.path)
print("remote copy:", "src=", workdir, "dst=", cluster.name,":",)
scp = SCPClient(shell.get_transport(), socket_timeout = 600)
try:
scp.put(workdir, dst, recursive=True)
except SCPException, e:
print "SCPException",e
shutil.rmtree(workdir)
os.remove("workdir")
return None
class Sender(object):
def __init__(self,server,username,password,dest_path,from_path):
self.dest_path = dest_path
self.from_path = from_path
self.recorder = Recorder.Recorder()
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
self.ssh.connect(server,username=username,password=password)
self.sftp = self.ssh.open_sftp()
self.scp = SCPClient(self.ssh.get_transport())
def send(self,file_path):
if op.exists(file_path):
file_modify_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(os.stat(file_path).st_mtime))
if not self.recorder.isSynced(file_path,file_modify_time):
new_file_path = os.path.join(self.dest_path+'/',file_path.split(self.from_path+os.sep)[1])
new_file_path = new_file_path.replace('\\','/')
new_file_dir,new_file = op.split(new_file_path)
if not rexists(self.sftp,new_file_dir):
rmkdir(self.sftp,new_file_dir)
print 'uploading %s .....' % (file_path)
self.scp.put(file_path,new_file_path)
self.recorder.record(file_path,file_modify_time)
else:
pass
def send_file(local_file, remote_path, ip, username, password, logger=None):
# Отсылает файл local_file в remote_path удаленной машины по scp
if remote_path[len(remote_path) - 1] != '/': remote_path += '/'
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.load_system_host_keys()
if logger is not None: logger.info("SCP SEND: connecting to %s" % (ip))
try:
ssh.connect(ip, username=username, password=password)
except:
if logger is not None: logger.info("SCP SEND: failed to connect to %s" % (ip))
return False
else:
if logger is not None: logger.info("SCP SEND: connected to %s" % (ip))
try:
if logger is not None: logger.info("SCP SEND: sending file %s" % (local_file))
scp = SCPClient(ssh.get_transport())
scp.put(local_file, remote_path)
except:
if logger is not None: logger.error("SCP SEND: error: failed to send file %s" % (local_file))
ssh.close()
return False
else:
if logger is not None: logger.info("SCP SEND: file sent to %s@%s:%s " % (username, ip, remote_path))
ssh.close()
return True
def get_rawdata(server, month, tmpfile):
serv = server.objects.get(name=server)
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(serv.ipaddress, username=serv.username, password=serv.password)
client = SCPClient(ssh.get_transport())
client.get(serv.path + '/volume.' + month+ '.csv', tmpfile)
def scp_send(client,filename):
try:
scp = SCPClient(client.get_transport())
stdin,stdout,stderr=scp.put(filename,"/usr/bin/rconf_client")
except:
print "|"
def main(file, key, encrypt, host, username, password):
"""
The main function. Takes multiple parameters which are prompted if not given on the command line.
:param file: The paht of the file to encrypt or decrypt and send.
:param key: The key to encrypt or decrypt.
:param encrypt: Tells if the operation is an encryption or a decryption.
:param host: The host where to send the file.
:param username: Username on the host.
:param password: Password if needed. If not needed '-' should be used to tell that there is no password needed.
"""
ssh = SSHClient()
ssh.load_system_host_keys()
if password != "-":
ssh.connect(host, username=username, password=password)
else:
ssh.connect(host, username=username)
scp = SCPClient(ssh.get_transport())
if encrypt:
print("Encrypting... ", end="")
to_send = encrypt_file(file, key)
print("Done.")
print("Sending to {}...".format(host), end="")
scp.put(to_send)
print("Done.")
else:
print(decrypt_file(file, key))
def sync_status():
"""
Fetches all statuses from a remote node and writes to local status cache
"""
# Run status update, if not, write status file for host and return
# TODO: make this run in it's own cron job (locally?)
host_up = status()
# Make local status folder if necessary
local('mkdir -p ./status' % env)
local('mkdir -p ./statusnew' % env)
# Write out status if failed to contact server
if not host_up:
status_file = './status/status_%(host_string)s.txt' % env
local('echo DOWN at `date` > %s' % status_file)
return
# Pull remote server's status files using scp
from scp import SCPClient
client = SCPClient(connections[env.host_string]._transport)
client.get('~/status/*', './statusnew/' % env, preserve_times=True)
# Use rsync to update older files
local('rsync -auv ./statusnew/* ./status/' % env)
# Write time into last_sync file
local('date > ./status/last_sync' % env)
# TODO: build html from statuses and save atomically?
make_html.dump()
def get_file(fname):
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(file_server)
scp = SCPClient(ssh.get_transport())
scp.get(fname)
scp.close()
def upload_iso(iso_path, storage_name):
host_ip = get_mgmt_ip_by_storagename(storage_name)
ssh = createSSHClient(host_ip, 22, "root", "arbre1234")
scp = SCPClient(ssh.get_transport())
if os.path.isfile(iso_path):
scp.put(iso_path)
else:
return False
def push_config(self, client, restart=False):
self.write_config()
scp = SCPClient(client.get_transport())
scp.put(str(self._local_config_path), str(self._remote_config_path))
if restart:
client.exec_command(self.RESTART_COMMAND)
time.sleep(10)
def ssh_scp(path):
ssh = _init_ssh()
ha_path = app.config.get('HAPROXY_PATH')
ha_cmd = "/usr/sbin/service haproxy restart"
scp = SCPClient(ssh.get_transport())
scp.put(path,ha_path)
stdin, stdout, stderr = ssh.exec_command(ha_cmd)
return stderr.read()
def addMacs(user, passwd, ip):
'''Connects to DD-WRT router with ip address ip over ssh. Then SCPs the
maclist into the DD-WRT /root/ directory. Feeds this file into a Shell-
Variable, then updates the mac filterlist and commits the config.
Input: user: username, string
passwd: password, string
ip: ip address to connect to, string
Output: void'''
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(AllowAllKeys())
try:
statusChangeOneIP(ip, bcolors.OKBLUE + 'connecting' + bcolors.ENDC)
client.connect(ip, username=user, password=passwd)
except BadHostKeyException:
statusChangeOneIP(ip, bcolors.FAIL + 'Host Key Exception' +\
bcolors.ENDC)
client.close()
return
except AuthenticationException:
user = raw_input('Authentication Exception, username: '******'Password: '******'SSH Exception' + bcolors.ENDC)
client.close()
return
except socket_error:
statusChangeOneIP(ip, bcolors.FAIL + 'cannot reach device' +\
bcolors.ENDC)
client.close()
return
statusChangeOneIP(ip, bcolors.OKBLUE + 'connected' + bcolors.ENDC)
try:
scp = SCPClient(client.get_transport())
'''Copy the parse mac addresses over to /root/, wait a second to make
sure transfer is complete, then set the _maclist variables on the router
and commit changes'''
scp.put(installpath + '/macsParsed.txt', 'macs.txt')
time.sleep(1)
client.exec_command('nvram set ath0_maclist="$(cat macs.txt)" && '\
'nvram set wl0_maclist="$(cat macs.txt)" && nvram commit')
statusChangeOneIP(ip, bcolors.OKGREEN + 'changes committed' +\
bcolors.ENDC)
#print('Changes commited')
except SSHException:
statusChangeOneIP(ip, bcolors.FAIL + 'SSH Exception' + bcolors.ENDC)
client.close()
return
client.close()
def run(self):
try:
# run a command and wait for it to finish
def run(command):
_, stdout, _ = ssh.exec_command(command)
stdout.channel.recv_exit_status()
# send the webapp a textual progress update
listener = Listener()
def log(text):
listener.publish(('project', self.project.name, 'deploy', 'status'), { 'text': text })
# prevent accidentally deploying on the server itself
# (this doesn't prevent deploying to the server's public
# IP address, so there's still a security problem here)
if self.host in ['127.0.0.1', '0.0.0.0']:
raise Exception('you\'re trying to deploy to the server!')
# create temporary build file
path = os.path.join(tempfile.mkdtemp(), 'deploy.launch')
xml = roslaunch_xml_for_file(self.project.project_file_path, self.remote_deploy_path)
open(path, 'w').write(xml)
log('Created file: deploy.launch')
# create a ssh session
log('Connecting to %s@%s...' % (self.user, self.host))
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(self.host, username=self.user, password=self.password)
log('Connected to remote machine')
# clean the deploy location
run('rm -fr %s' % self.remote_deploy_path)
run('mkdir -p %s' % self.remote_deploy_path)
log('Cleaned deploy location')
# copy the launch file over
scp = SCPClient(ssh.get_transport())
scp.put(path, remote_path=self.remote_deploy_path)
log('Copied file: deploy.launch')
# run the file
command = 'cd %s && roslaunch deploy.launch' % self.remote_deploy_path
channel = ssh.invoke_shell()
channel.send(command + '\nexit\n')
log('Ran command: roslaunch deploy.launch')
# wait for the command to finish
import time
while not self.die:
time.sleep(0.1)
except:
import traceback
log('Exception:\n' + traceback.format_exc())
ssh.close()
log('Connection was closed')
def scp(self, filename, remote_path):
config = SSHConfig()
config.parse(open(os.path.expanduser('~/.ssh/config')))
o = config.lookup('geodata')
ssh_client = SSHClient()
ssh_client.load_system_host_keys()
ssh_client.connect(o['hostname'], username=o['user'])
scp = SCPClient(ssh_client.get_transport())
scp.put(filename, remote_path=remote_path)
def http_config(hostname, username, password):
connect = createSSHClient(hostname, 22, username, password)
""" Installing Apache on the servers"""
print "\n Installing Apache on the Servers "
stdin, stdout, stderr = connect.exec_command('apt-get -y install apache2')
print 'This is output =',stdout.readlines()
print 'This is error =',stderr.readlines()
"""Copying the index.php file"""
print "\n Copying the Index.php file remotely"
# ssh = createSSHClient(hostname, 22, username, password)
scp = SCPClient(connect.get_transport())
scp.put(local_path, remote_path)
"""Customizing Apache on the servers"""
print "\n Customizing Apache "
stdin, stdout, stderr = connect.exec_command('mv /var/www/html/index.html /var/www/html/index.html_backup')
print 'This is output =',stdout.readlines()
print 'This is error =',stderr.readlines()
"""Changing Permissions on index.php file"""
print "\n Changing File permissions "
stdin, stdout, stderr = connect.exec_command('chown root:root /var/www/html/index.php')
print 'This is output =',stdout.readlines()
print 'This is error =',stderr.readlines()
"""Customizing Apache on the servers"""
print "\n Changing File permissions "
stdin, stdout, stderr = connect.exec_command('chmod 644 /var/www/html/index.php')
print 'This is output =',stdout.readlines()
print 'This is error =',stderr.readlines()
"""Restarting the Apache server"""
print "\n Restarting the Apache Server "
stdin, stdout, stderr = connect.exec_command('service apache2 restart')
print 'This is output =',stdout.readlines()
print 'This is error =',stderr.readlines()
"""Verify that http server is up and can spit out Hello World"""
print "\n Verifying http server"
stdin, stdout, stderr = connect.exec_command('curl -sv "http://localhost"| grep -i hello')
list_http = stdout.readlines()
print 'This is output =',list_http
print 'This is error =',stderr.readlines()
i = 0
for each in list_http:
if re.search(r'Hello', each) != None:
print "\n ***HTTP server is up and running with Hello World*** \n\n\n"
exit
else:
i =+1
if i > 0:
print "\n ***HTTP Server does not have Hello World*** \n\n\n"
connect.close()
def get_reports(self, options, conf_dict):
master = self.existing_nodes[0]
ssh_c = SSHClient(hostname = master, username = \
self.username, password = self.password)
coverage_xml = conf_dict['coverage']['coverage_xml']
scp = SCPClient(ssh_c.get_transport())
scp.get(coverage_xml)
def copy(hostname, username, filepath, remotefile):
ssh = SSHClient()
# ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname, username=username)
ssh.exec_command("mkdir -p " + remotefile[0:remotefile.rindex('/')])
# SCPCLient takes a paramiko get_transportt as its only argument
scp = SCPClient(ssh.get_transport())
scp.put(filepath, remotefile)
def put_file(self, source, destination):
assert os.path.isfile(source)
try:
scp = SCPClient(self.__ssh.get_transport())
scp.put(source, destination)
except Exception as err:
print err
return {'status': 'KO'}
return {'status': 'OK'}
from paramiko import SSHClient, AutoAddPolicy
from scp import SCPClient
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(hostname='ftp.bewater.com.pt',
port='999',
username='******',
password='******')
scp = SCPClient(ssh.get_transport())
scp.put('/var/www/apps/eWater/storage/app/interruptions/comunicados.xls', '/')
def deploy(path: str = './', hostname: str = "ev3dev", username: str = "robot", password: str = "maker",
execute_file: Optional[str] = None, executable: List[str] = ('*.py',),
exclude_path: str = "./.ignore", print_console: bool = True,
redirect_stdout: bool = True, redirect_stderr: bool = True, redirect_stdin: bool = False) -> None:
"""
Send code to Ev3
:param path: The Directory to send (default is current directory).
:param hostname: The ssh hostname (default is 'ev3dev')
:param username: The ssh username (default is 'robot')
:param password: The ssh password (default is 'maker')
:param execute_file: A file to run on the ev3 when finished. 'None' to disable.
Note: this file must be marked as executable.
:param executable: A list of patterns of files that should be marked as executable (default is ['*.py']).
:param exclude_path: The file containing the list of files to ignore (default is '.ignore').
:param print_console: Should we print info to the console?
:param redirect_stdout: Should we redirect stdout form ev3 to console?
:param redirect_stderr: Should we redirect stderr form ev3 to console?
:param redirect_stdin: Should we redirect console input to ev3 stdin?
This is disabled by default as it cannot terminate without reading from stdin.
"""
# Get / Set working directory
if print_console:
print("CD", path)
os.chdir(path)
working_dir = os.getcwd()
dir_name = os.path.basename(working_dir)
exclude = read_exclude(exclude_path)
# Set up ssh
if print_console:
print("Starting ssh ...")
ssh = SSHClient()
ssh.load_system_host_keys()
if print_console:
print("Connecting to", F"{username}@{hostname} ...")
ssh.connect(hostname=hostname, username=username, password=password)
with SCPClient(ssh.get_transport()) as scp:
for subdir, dirs, files in os.walk('.'): # for every file in current working directory:
for filename in files:
filepath = subdir + '/' + filename # get full file path (relative to working directory)
if not match(filepath, exclude): # if the file path does not match any of the excluded patterns:
if print_console:
print("Sending", Path(filepath), "... ", end='')
# create the directory if it does not exist
ssh.exec_command('mkdir -p ' + path_join('~', dir_name, subdir).as_posix())
# copy files using scp
scp.put(str(path_join(working_dir, filepath)), path_join('~', dir_name, filepath).as_posix())
if print_console:
print("Sent")
if match(filepath, executable): # if file path matches any of the executable patterns:
# mark as executable
if print_console:
print(path_join('~', dir_name, filepath).as_posix(), "marked as executable.")
ssh.exec_command('chmod u+x ' + path_join('~', dir_name, filepath).as_posix())
else:
if print_console:
print('Excluding', Path(filepath), '.')
if execute_file:
if print_console:
print(F'\nExecuting {execute_file} ...\n')
# execute the file.
stdin, stdout, stderr = ssh.exec_command(path_join('~', dir_name, execute_file).as_posix(), get_pty=True)
# create the redirecting threads
if redirect_stdout:
out = threading.Thread(target=redirect_stdout_handler, args=(stdout,))
if redirect_stderr:
err = threading.Thread(target=redirect_stderr_handler, args=(stderr,))
if redirect_stdin:
child_pid = os.fork()
if child_pid == 0:
redirect_stdin_handler(stdin)
os.kill(os.getpid(), SIGTERM)
# sin = threading.Thread(target=redirect_stdin_handler, args=(stdin,))
# start them
if redirect_stdout:
out.start()
if redirect_stderr:
err.start()
# if redirect_stdin:
# sin.start()
# wait for them to terminate
if redirect_stdout:
out.join()
if redirect_stderr:
err.join()
if redirect_stdin:
global run_stdin
# tell redirect_stdin_handler to exit without sending data to stdin
run_stdin = False
# sys.stdin.close()
# sin.join()
os.kill(child_pid, SIGTERM)
if print_console:
print('\nFinished.')
def get(self, remote_path, local_path='/tmp/', recursive=True):
client = self._get_client()
with SCPClient(client.get_transport()) as scp:
scp.get(remote_path, local_path, recursive)
cur_time = int(time.time())
while cur_time < timeout:
if IsOnline(sync_ip):
break
cur_time = int(time.time())
if not IsOnline(sync_ip):
print("Failed to wake %s" % sync_ip)
time.sleep(60)
continue
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(sync_ip, username=username)
scp = SCPClient(ssh.get_transport(), progress=progress)
sync(monitor_dir, sync_dir, scp)
scp.close()
ssh.close()
# if there are, pause main queue and sync them to NAS
# resume when sync complete
time.sleep(60)
def Update(sip, S_md5, App, path):
ssh = paramiko.SSHClient()
key = paramiko.RSAKey.from_private_key_file(key_file)
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
for ip in sip:
ip = ip.strip()
Redis.lpush(Key, '-' * 60)
Redis.lpush(Key, 'publish ' + path + ' to ' + ip + '\n')
try:
ssh.connect(ip, 22, username, pkey=key, timeout=15)
cmd = "mkdir -p {0}".format(web_path)
ssh.exec_command(cmd)
scp = SCPClient(ssh.get_transport())
scp.put('%s%s.zip' % (web_path, App), web_path)
cmd = '/usr/bin/md5sum %s/%s.zip' % (web_path, App)
stdin, stdout, stderr = ssh.exec_command(cmd)
R_md5 = stdout.read().split()[0]
if R_md5 == S_md5:
Redis.lpush(Key, '%s md5 verify --->pass!' % App)
Redis.lpush(Key, 'unzip start ......')
cmds = [
'cd %s && /usr/bin/unzip -qo %s.zip && /bin/rm -f %s.zip' %
(web_path, App, App),
'[ -e %s ] && echo ok' % tag_path
]
for cmd in cmds:
stdin, stdout, stderr = ssh.exec_command(cmd)
result_zip = stdout.read().strip()
if result_zip == 'ok':
Redis.lpush(Key, '%s --->unzip success!' % App)
Redis.lpush(Key, 'backup start ......')
cmds = ('/bin/mkdir -p %s' % bak_path,
'[ -e %s/%s ] && /bin/rm -rf %s/%s' %
(bak_path, App, bak_path, App),
'[ -e %s%s ] && /bin/mv %s%s %s/%s' %
(web_path, App, web_path, App, bak_path, App))
for cmd in cmds:
stdin, stdout, stderr = ssh.exec_command(cmd)
result_backup = stderr.read()
if result_backup:
Redis.lpush(Key, result_backup)
Redis.lpush(Key, '%s --->backup fail!' % App)
else:
Redis.lpush(Key,
'%s --->backup success!' % App)
Redis.lpush(Key, 'publish start ......')
cmd = '/bin/mv %s %s%s' % (tag_path, web_path, App)
stdin, stdout, stderr = ssh.exec_command(cmd)
result_rsync = stderr.read()
if result_rsync:
Redis.lpush(Key, result_rsync)
Redis.lpush(Key,
'%s --->publish fail!' % App)
else:
Redis.lpush(
Key, '%s --->publish success!' % App)
ssh.close()
Redis.lpush(Key, '-' * 60)
else:
Redis.lpush(Key, '%s --->unzip fail!' % App)
else:
Redis.lpush(Key, 'S_MD5:%r' % S_md5)
Redis.lpush(Key, 'R_MD5:%r' % R_md5)
Redis.lpush(Key, ' verify %s on %s fail!!!' % (App, ip))
except Exception as e:
Redis.lpush(Key, 'Update:{0}'.format(e))
ssh.close()
sys.exit()
class Jaide():
""" Purpose: An object for manipulating a Junos device.
Methods include copying files, running show commands,
shell commands, commit configuration changes, finding
interface errors, and getting device status/information.
All of the methods listed below that touch Junos are wrapped by a
decorator function @check_instance, which handles ensuring the correct
connection is used to perform the requested operation.
"""
def __init__(self, host, username, password, connect_timeout=5,
session_timeout=300, connect="paramiko", port=22):
""" Initialize the Jaide object.
Purpose: This is the initialization function for the Jaide class,
| which creates a connection to a junos device. It will
| return a Jaide object, which can then be used to actually
| send commands to the device. This function establishes the
| connection to the device via a NCClient manager object.
| > **NOTE:** The connect parameter should be ignored under most
| > circumstances. Changing it only affects how Jaide first
| > connects to the device. The decorator function
| > @check_instance will handle moving between session
| > types for you.
@param host: The IP or hostname of the device to connect to.
@type host: str
@param username: The username for the connection
@type username: str
@param password: The password for the connection
@type password: str
@param connect_timeout: The timeout value, in seconds, for attempting
| to connect to the device.
@type connect_timeout: int
@param session_timeout: The timeout value, in seconds, for the
| session. If a command is sent and nothing
| is heard back from the device in this
| timeframe, the session is declared dead,
| and times out.
@type session_timeout: int
@param connect: **NOTE: We default to 'paramiko', but this
| parameter can be set to False to prevent connecting
| on object instantiation. The @check_instance
| decorator function will handle sliding between
| session types depending on what function is being
| called, meaning generally self.conn_type and this
| connect parameter should be ignored.**
|
| The connection type that should be made. Several
| options are available: 'ncclient', 'scp', and
| 'paramiko', 'shell' and 'root'.
|
| 'paramiko' : is used for operational commands
| (couldn't use ncclient because of lack of pipes `|`
| support.
|
| 'scp' : is used for copying files to/from
| the device, and uses an SCP connection.
|
| 'shell' : is for sending shell commands.
|
| 'root' : is when the user is doing operational
| commands, but is logged in as root, (requires
| handling separately, since this puts the session
| into a shell prompt)
|
| 'ncclient' : is used for all other commands.
@type connect: str
@param port: The destination port on the device to attempt the
| connection.
@type port: int
@returns: an instance of the Jaide class
@rtype: jaide.Jaide object
"""
# store object properties and set initial values.
self.host = host.strip()
self.port = port
self.username = username
self.password = password
self.session_timeout = session_timeout
self.connect_timeout = connect_timeout
self._shell = ""
self._scp = ""
self.conn_type = connect
self._in_cli = False
self._filename = None
# make the connection to the device
if connect:
self.connect()
def check_instance(function):
""" Wrapper that tests the type of _session.
Purpose: This decorator function is used by all functions within
| the Jaide class that interact with a device to ensure the
| proper session type is in use. If it is not, it will
| attempt to migrate _session to that type before moving
| to the originally requested function.
| > **NOTE:** This function is a decorator, and should not be
| > used directly. All other methods in this class that touch
| > the Junos device are wrapped by this function to ensure the
| > proper connection type is used.
@param function: the function that is being wrapped around
@type function: function
@returns: the originally requested function
@rtype: function
"""
def wrapper(self, *args, **kwargs):
func_trans = {
"commit": manager.Manager,
"compare_config": manager.Manager,
"commit_check": manager.Manager,
"device_info": manager.Manager,
"diff_config": manager.Manager,
"health_check": manager.Manager,
"interface_errors": manager.Manager,
"op_cmd": paramiko.client.SSHClient,
"shell_cmd": paramiko.client.SSHClient,
"scp_pull": paramiko.client.SSHClient,
"scp_push": paramiko.client.SSHClient
}
# when doing an operational command, logging in as root
# brings you to shell, so we need to enter the device as a shell
# connection, and move to cli to perform the command
# this is a one-off because the isinstance() check will be bypassed
if self.username == "root" and function.__name__ == "op_cmd":
if not self._session:
self.conn_type = "paramiko"
self.connect()
if not self._shell:
self.conn_type = "root"
self.connect()
self.shell_to_cli() # check if we're in the cli
# Have to call shell command separately, since we are using _shell
# for comparison, not _session.
elif function.__name__ == 'shell_cmd':
if not self._shell:
self.conn_type = "shell"
self.connect()
self.cli_to_shell() # check if we're in shell.
if isinstance(self._session, func_trans[function.__name__]):
# If they're doing SCP, we have to check for both _session and
# _scp
if function.__name__ in ['scp_pull', 'scp_push']:
if not isinstance(self._scp, SCPClient):
self.conn_type = "scp"
self.connect()
else:
self.disconnect()
if function.__name__ == "op_cmd":
self.conn_type = "paramiko"
elif function.__name__ in ["scp_pull", "scp_push"]:
self.conn_type = "scp"
else:
self.conn_type = "ncclient"
self.connect()
return function(self, *args, **kwargs)
return wrapper
def cli_to_shell(self):
""" Move _shell to the shell from the command line interface (CLI). """
if self._in_cli:
self._shell.send("start shell\n")
time.sleep(2)
self._shell.recv(9999)
self._in_cli = False
return True
return False
@check_instance
def commit(self, commands="", confirmed=None, comment=None,
at_time=None, synchronize=False, req_format='text'):
""" Perform a commit operation.
Purpose: Executes a commit operation. All parameters are optional.
| commit confirm and commit at are mutually exclusive. All
| the others can be used with each other and commit confirm/at.
@param commands: A string or list of multiple commands
| that the device will compare with.
| If a string, it can be a single command,
| multiple commands separated by commas, or
| a filepath location of a file with multiple
| commands, each on its own line.
@type commands: str or list
@param confirmed: integer value of the number of **seconds** to
| confirm the commit for, if requested.
@type confirmed: int
@param comment: string that the user wants to comment the commit
| with. Will show up in the 'show system commit' log.
@type comment: str
@param at_time: string designating the time at which the commit
| should happen. Can be in one of two Junos approved
| formats.
@type comment: str
@param synchronize: boolean set to true if desiring a commit
| synchronize operation.
@type synchronize: bool
@param req_format: string to specify the response format. Accepts
| either 'text' or 'xml'
@type req_format: str
@returns: The reply from the device.
@rtype: str
"""
# ncclient doesn't support a truly blank commit, so if nothing is
# passed, use 'annotate system' to make a blank commit
if not commands:
commands = 'annotate system ""'
clean_cmds = []
for cmd in clean_lines(commands):
clean_cmds.append(cmd)
# try to lock the candidate config so we can make changes.
self.lock()
self._session.load_configuration(action='set', config=commands)
results = ""
# confirmed and commit at are mutually exclusive. commit confirm
# takes precedence.
if confirmed:
results = self._session.commit(confirmed=True,
timeout=str(confirmed),
comment=comment,
synchronize=synchronize)
else:
results = self._session.commit(comment=comment, at_time=at_time,
synchronize=synchronize)
self.unlock()
if results:
if req_format == 'xml':
return results
# commit() DOES NOT return a parse-able xml tree, so we
# convert it to an ElementTree xml tree.
results = ET.fromstring(results.tostring)
out = ''
for i in results.iter():
# the success message is just a tag, so we need to get it
# specifically.
if i.tag == 'commit-check-success':
out += 'configuration check succeeds\n'
elif i.tag == 'commit-success':
out += 'commit complete\n'
elif i.tag == 'ok':
out += 'commit complete\n'
# this is for normal output with a tag and inner text, it will
# strip the inner text and add it to the output.
elif i.text is not None:
if i.text.strip() + '\n' != '\n':
out += i.text.strip() + '\n'
# this is for elements that don't have inner text,
# it will add the tag to the output.
elif i.text is None:
if i.tag + '\n' != '\n':
out += i.tag + '\n'
return out
return False
@check_instance
def commit_check(self, commands="", req_format="text"):
""" Execute a commit check operation.
Purpose: This method will take in string of multiple commands,
| and perform and 'commit check' on the device to ensure
| the commands are syntactically correct. The response can
| be formatted as text or as xml.
@param commands: A string, filepath, or list of multiple commands
| that the device will compare with.
@type commands: str or list
@param req_format: The desired format of the response, defaults to
| 'text', but also accepts 'xml'
@type req_format: str
@returns: The reply from the device.
@rtype: str
"""
if not commands:
raise InvalidCommandError('No commands specified')
clean_cmds = []
for cmd in clean_lines(commands):
clean_cmds.append(cmd)
self.lock()
self._session.load_configuration(action='set', config=clean_cmds)
# conn.validate() DOES NOT return a parse-able xml tree, so we
# convert it to an ElementTree xml tree.
results = ET.fromstring(self._session.validate(
source='candidate').tostring)
# release the candidate configuration
self.unlock()
if req_format == "xml":
return ET.tostring(results)
out = ""
# we have to parse the elementTree object, and get the text
# from the xml.
for i in results.iter():
# the success message is just a tag, so we need to get it
# specifically.
if i.tag == 'commit-check-success':
out += 'configuration check succeeds\n'
# this is for normal output with a tag and inner text, it will
# strip the inner text and add it to the output.
elif i.text is not None:
if i.text.strip() + '\n' != '\n':
out += i.text.strip() + '\n'
# this is for elements that don't have inner text, it will add the
# tag to the output.
elif i.text is None:
if i.tag + '\n' != '\n':
out += i.tag + '\n'
return out
@check_instance
def compare_config(self, commands="", req_format="text"):
""" Execute a 'show | compare' against the specified commands.
Purpose: This method will take in string of multiple commands,
| and perform and 'show | compare' on the device to show the
| differences between the active running configuration and
| the changes proposed by the passed commands parameter.
@param commands: A string, filepath, or list of multiple commands
| that the device will compare with.
@type commands: str or list
@param req_format: The desired format of the response, defaults to
| 'text', but also accepts 'xml'
@type req_format: str
@returns: The reply from the device.
@rtype: str
"""
if not commands:
raise InvalidCommandError('No commands specified')
clean_cmds = [cmd for cmd in clean_lines(commands)]
self.lock()
self._session.load_configuration(action='set', config=clean_cmds)
out = self._session.compare_configuration()
self.unlock()
if req_format.lower() == "xml":
return out
return out.xpath(
'configuration-information/configuration-output')[0].text
def connect(self):
""" Establish a connection to the device.
Purpose: This method is used to make a connection to the junos
| device. The internal property conn_type is what
| determines the type of connection we make to the device.
| - 'paramiko' is used for operational commands (to allow
| pipes in commands)
| - 'scp' is used for copying files
| - 'shell' is used for to send shell commands
| - 'root' is used when logging into the device as root, and
| wanting to send operational commands
| - 'ncclient' is used for the rest (commit, compare_config,
| commit_check)
@returns: None
@rtype: None
"""
if self.conn_type == 'paramiko':
self._session = paramiko.SSHClient()
# These two lines set the paramiko logging to Critical to
# remove extra messages from being sent to the user output.
logger = logging.Logger.manager.getLogger('paramiko.transport')
logger.setLevel(logging.CRITICAL)
self._session.set_missing_host_key_policy(
paramiko.AutoAddPolicy())
self._session.connect(hostname=self.host,
username=self.username,
password=self.password,
port=self.port,
timeout=self.connect_timeout)
if self.conn_type == 'scp':
self._scp_session = paramiko.SSHClient()
logger = logging.Logger.manager.getLogger('paramiko.transport')
logger.setLevel(logging.CRITICAL)
self._scp_session.set_missing_host_key_policy(
paramiko.AutoAddPolicy())
self._scp_session.connect(hostname=self.host,
username=self.username,
password=self.password,
port=self.port,
timeout=self.connect_timeout)
self._scp = SCPClient(self._scp_session.get_transport())
elif self.conn_type == "ncclient":
self._session = manager.connect(
host=self.host,
port=self.port,
username=self.username,
password=self.password,
timeout=self.connect_timeout,
device_params={'name': 'junos'},
hostkey_verify=False
)
elif self.conn_type == 'shell':
if not self._session:
self.conn_type = 'paramiko'
self.connect()
self.conn_type = 'shell'
if not self._shell:
self._shell = self._session.invoke_shell()
time.sleep(2)
if self.username != 'root' and not self._in_cli:
self._in_cli = True
if not self.cli_to_shell():
self._shell.recv(9999)
elif self.conn_type == 'root':
# open the shell if necessary, and move into CLI
if not self._shell:
self._shell = self._session.invoke_shell()
time.sleep(2)
if not self.shell_to_cli():
self._shell.recv(9999)
self._update_timeout(self.session_timeout)
def _copy_status(self, filename, size, sent):
""" Echo status of an SCP operation.
Purpose: Callback function for an SCP operation. Used to show
| the progress of an actively running copy. This directly
| prints to stdout, one line for each file as it's copied.
| The parameters received by this function are those received
| from the scp.put or scp.get function, as explained in the
| python scp module docs.
@param filename: The filename of file being copied.
@type filename: str
@param size: The total size of the current file being copied.
@type size: str or float
@param sent: The amount of data sent for the current file being copied.
@type sent: str or float
@returns: None
"""
output = "Transferred %.0f%% of the file %s" % (
(float(sent) / float(size) * 100), path.normpath(filename))
output += (' ' * (120 - len(output)))
if filename != self._filename:
if self._filename is not None:
print('')
self._filename = filename
print(output, end='\r')
@check_instance
def device_info(self):
""" Pull basic device information.
Purpose: This function grabs the hostname, model, running version, and
| serial number of the device.
@returns: The output that should be shown to the user.
@rtype: str
"""
# get hostname, model, and version from 'show version'
resp = self._session.get_software_information(format='xml')
hostname = resp.xpath('//software-information/host-name')[0].text
model = resp.xpath('//software-information/product-model')[0].text
version = (resp.xpath('//software-information/package-information/'
'comment')[0].text.split('[')[1].split(']')[0])
# get uptime from 'show system uptime'
resp = self._session.get_system_uptime_information(format='xml')
current_time = resp.xpath('//current-time/date-time')[0].text
uptime = resp.xpath('//uptime-information/up-time')[0].text
# get serial number from 'show chassis hardware'
show_hardware = self._session.get_chassis_inventory(format='xml')
# If we're hitting an EX, grab each Routing Engine Serial number
# to get all RE SNs in a VC
if (('EX' or 'ex' or 'Ex') in
show_hardware.xpath('//chassis-inventory/chassis/chassis-module'
'/description')[0].text):
serial_num = ""
for eng in show_hardware.xpath('//chassis-inventory/chassis/chassis-module'):
if 'Routing Engine' in eng.xpath('name')[0].text:
serial_num += (eng.xpath('name')[0].text + ' Serial #: ' +
eng.xpath('serial-number')[0].text)
else: # Any other device type, just grab chassis SN
serial_num = ('Chassis Serial Number: ' +
show_hardware.xpath('//chassis-inventory/chassis/'
'serial-number')[0].text)
return ('Hostname: %s\nModel: %s\nJunos Version: %s\n%s\nCurrent Time:'
' %s\nUptime: %s\n' %
(hostname, model, version, serial_num, current_time, uptime))
# TODO: [2.1] @rfe optional different username/password.
@check_instance
def diff_config(self, second_host, mode='stanza'):
""" Generate configuration differences with a second device.
Purpose: Open a second ncclient.manager.Manager with second_host, and
| and pull the configuration from it. We then use difflib to
| get the delta between the two, and yield the results.
@param second_host: the IP or hostname of the second device to
| compare against.
@type second_host: str
@param mode: string to signify 'set' mode or 'stanza' mode.
@type mode: str
@returns: iterable of strings
@rtype: str
"""
second_conn = manager.connect(
host=second_host,
port=self.port,
username=self.username,
password=self.password,
timeout=self.connect_timeout,
device_params={'name': 'junos'},
hostkey_verify=False
)
command = 'show configuration'
if mode == 'set':
command += ' | display set'
# get the raw xml config
config1 = self._session.command(command, format='text')
# for each /configuration-output snippet, turn it to text and join them
config1 = ''.join([snippet.text.lstrip('\n') for snippet in
config1.xpath('//configuration-output')])
config2 = second_conn.command(command, format='text')
config2 = ''.join([snippet.text.lstrip('\n') for snippet in
config2.xpath('//configuration-output')])
return difflib.unified_diff(config1.splitlines(), config2.splitlines(),
self.host, second_host)
def disconnect(self):
""" Close the connection(s) to the device.
Purpose: Closes the current connection(s) to the device, no matter
| what types exist.
@returns: None
@rtype: None
"""
if self._shell:
self._shell.close()
self._shell = ""
if isinstance(self._session, manager.Manager):
self._session.close_session()
elif isinstance(self._session, paramiko.client.SSHClient):
self._session.close()
self._session = ""
elif isinstance(self._session, SCPClient):
self._session.close()
self._session = ""
self._scp = ""
def _error_parse(self, interface, face):
""" Parse the extensive xml output of an interface and yield errors.
Purpose: Takes the xml output of 'show interfaces extensive' for a
| given interface and yields the error types that have a
| significant number of errors.
@param interface: The xml output of the 'sh int ext' command for
| the desired interface.
@type interface: lxml.etree._Element object
@param face: The direction of the errors we're wanting. Either 'input'
| or 'output' is accepted.
@type face: str
@returns: Yields each error that has a significant number
@rtype: iterable of strings.
"""
try:
error_list = interface.xpath(face + '-error-list')[0].getchildren()
except IndexError: # no error list on this interface
pass
else:
for x in range(len(error_list)):
if error_list[x].tag == "carrier-transitions":
if int(error_list[x].text.strip()) > 50:
yield " has greater than 50 flaps."
elif int(error_list[x].text.strip()) > 0:
yield " has %s of %s." % (error_list[x].text.strip(),
error_list[x].tag.strip())
@check_instance
def health_check(self):
""" Pull health and alarm information from the device.
Purpose: Grab the cpu/mem usage, system/chassis alarms, top 5
| processes, and states if the primary/backup partitions are on
| different versions.
@returns: The output that should be shown to the user.
@rtype: str
"""
output = 'Chassis Alarms:\n\t'
# Grab chassis alarms, system alarms, show chassis routing-engine,
# 'show system processes extensive', and also xpath to the
# relevant nodes on each.
chassis_alarms = self._session.command("show chassis alarms")
chassis_alarms = chassis_alarms.xpath('//alarm-detail')
system_alarms = self._session.command("show system alarms")
system_alarms = system_alarms.xpath('//alarm-detail')
chass = self._session.command(command="show chassis routing-engine",
format='text').xpath('//output')[0].text
proc = self._session.command("show system processes extensive")
proc = proc.xpath('output')[0].text.split('\n')
if chassis_alarms == []: # Chassis Alarms
output += 'No chassis alarms active.\n'
else:
for i in chassis_alarms:
output += (i.xpath('alarm-class')[0].text.strip() + ' Alarm \t'
'\t' + i.xpath('alarm-time')[0].text.strip() +
'\n\t' +
i.xpath('alarm-description')[0].text.strip() + '\n')
output += '\nSystem Alarms: \n\t'
if system_alarms == []: # System Alarms
output += 'No system alarms active.\n'
else:
for i in system_alarms:
output += (i.xpath('alarm-class')[0].text.strip() + ' Alarm '
'\t\t' + i.xpath('alarm-time')[0].text.strip() +
'\n\t' +
i.xpath('alarm-description')[0].text.strip() + '\n')
# add the output of the show chassis routing-engine to the command.
output += '\n' + chass
# Grabs the top 5 processes and the header line.
output += ('\n\nTop 5 busiest processes (high mgd values likely from '
'script execution):\n')
for line_number in range(8, 14):
output += proc[line_number] + '\n'
return output
@check_instance
def interface_errors(self):
""" Parse 'show interfaces extensive' and return interfaces with errors.
Purpose: This function is called for the -e flag. It will let the user
| know if there are any interfaces with errors, and what those
| interfaces are.
@returns: The output that should be shown to the user.
@rtype: str
"""
output = [] # used to store the list of interfaces with errors.
# get a string of each physical and logical interface element
dev_response = self._session.command('sh interfaces extensive')
ints = dev_response.xpath('//physical-interface')
ints += dev_response.xpath('//logical-interface')
for i in ints:
# Grab the interface name for user output.
int_name = i.xpath('name')[0].text.strip()
# Only check certain interface types.
if (('ge' or 'fe' or 'ae' or 'xe' or 'so' or 'et' or 'vlan' or
'lo0' or 'irb') in int_name):
try:
status = (i.xpath('admin-status')[0].text.strip() +
'/' + i.xpath('oper-status')[0].text.strip())
except IndexError:
pass
else:
for error in self._error_parse(i, "input"):
output.append("%s (%s)%s" % (int_name, status,
error))
for error in self._error_parse(i, "output"):
output.append("%s (%s)%s" % (int_name, status,
error))
if output == []:
output.append('No interface errors were detected on this device.')
return '\n'.join(output) + '\n'
def lock(self):
""" Lock the candidate config. Requires ncclient.manager.Manager. """
if isinstance(self._session, manager.Manager):
self._session.lock()
@check_instance
def op_cmd(self, command, req_format='text', xpath_expr=""):
""" Execute an operational mode command.
Purpose: Used to send an operational mode command to the connected
| device. This requires and uses a paramiko.SSHClient() as
| the handler so that we can easily pass and allow all pipe
| commands to be used.
|
| We indiscriminately attach ' | no-more' on the end of
| every command so the device doesn't hold output. The
| req_format parameter can be set to 'xml' to force raw
| xml output in the reply.
@param command: The single command that to retrieve output from the
| device. Any pipes will be taken into account.
@type command: str
@param req_format: The desired format of the response, defaults to
| 'text', but also accepts 'xml'. **NOTE**: 'xml'
| will still return a string, not a libxml ElementTree
@type req_format: str
@returns: The reply from the device.
@rtype: str
"""
if not command:
raise InvalidCommandError("Parameter 'command' cannot be empty")
if req_format.lower() == 'xml' or xpath_expr:
command = command.strip() + ' | display xml'
command = command.strip() + ' | no-more\n'
out = ''
# when logging in as root, we use _shell to get the response.
if self.username == 'root':
self._shell.send(command)
time.sleep(3)
while self._shell.recv_ready():
out += self._shell.recv(999999)
time.sleep(.75)
# take off the command being sent and the prompt at the end.
out = '\n'.join(out.split('\n')[1:-2])
# not logging in as root, and can grab the output as normal.
else:
stdin, stdout, stderr = self._session.exec_command(command=command,
timeout=float(self.session_timeout))
stdin.close()
# read normal output
while not stdout.channel.exit_status_ready():
out += stdout.read()
stdout.close()
# read errors
while not stderr.channel.exit_status_ready():
out += stderr.read()
stderr.close()
return out if not xpath_expr else xpath(out, xpath_expr)
@check_instance
def scp_pull(self, src, dest, progress=False, preserve_times=True):
""" Makes an SCP pull request for the specified file(s)/dir.
Purpose: By leveraging the _scp private variable, we make an scp pull
| request to retrieve file(s) from a Junos device.
@param src: string containing the source file or directory
@type src: str
@param dest: destination string of where to put the file(s)/dir
@type dest: str
@param progress: set to `True` to have the progress callback be
| printed as the operation is copying. Can also pass
| a function pointer to handoff the progress callback
| elsewhere.
@type progress: bool or function pointer
@param preserve_times: Set to false to have the times of the copied
| files set at the time of copy.
@type preserve_times: bool
@returns: `True` if the copy succeeds.
@rtype: bool
"""
# set up the progress callback if they want to see the process
if progress is True:
self._scp._progress = self._copy_status
# redirect to another function
elif hasattr(progress, '__call__'):
self._scp._progress = progress
else: # no progress callback
self._scp._progress = None
# retrieve the file(s)
self._scp.get(src, dest, recursive=True, preserve_times=preserve_times)
self._filename = None
return False
@check_instance
def scp_push(self, src, dest, progress=False, preserve_times=True):
""" Purpose: Makes an SCP push request for the specified file(s)/dir.
@param src: string containing the source file or directory
@type src: str
@param dest: destination string of where to put the file(s)/dir
@type dest: str
@param progress: set to `True` to have the progress callback be
| printed as the operation is copying. Can also pass
| a function pointer to handoff the progress callback
| elsewhere.
@type progress: bool or function pointer
@param preserve_times: Set to false to have the times of the copied
| files set at the time of copy.
@type preserve_times: bool
@returns: `True` if the copy succeeds.
@rtype: bool
"""
# set up the progress callback if they want to see the process
if progress is True:
self._scp._progress = self._copy_status
# redirect to another function
elif hasattr(progress, '__call__'):
self._scp._progress = progress
else: # no progress callback
self._scp._progress = None
# push the file(s)
self._scp.put(src, dest, recursive=True, preserve_times=preserve_times)
self._filename = None
return False
@check_instance
def shell_cmd(self, command=""):
""" Execute a shell command.
Purpose: Used to send a shell command to the connected device.
| This uses the self._shell instance, which should be a
| paramiko.Channel object, instead of a SSHClient.
| This is because we cannot send shell commands to the
| device using a SSHClient.
@param command: The single command that to retrieve output from the
| device.
@type command: str
@returns: The reply from the device.
@rtype: str
"""
if not command:
raise InvalidCommandError("Parameter 'command' must not be empty.")
command = command.strip() + '\n'
self._shell.send(command)
time.sleep(2)
out = ''
while self._shell.recv_ready():
out += self._shell.recv(999999)
time.sleep(.75)
# take off the command being sent and the prompt at the end.
return '\n'.join(out.split('\n')[1:-1])
def shell_to_cli(self):
""" Move _shell to the command line interface (CLI). """
if not self._in_cli:
self._shell.send("cli\n")
time.sleep(4)
self._shell.recv(9999)
self._in_cli = True
return True
return False
def unlock(self):
""" Unlock the candidate config.
Purpose: Unlocks the candidate configuration, so that other people can
| edit the device. Requires the _session private variable to be
| a type of a ncclient.manager.Manager.
"""
if isinstance(self._session, manager.Manager):
self._session.unlock()
def _update_timeout(self, value):
if isinstance(self._session, manager.Manager):
self._session.timeout = value
if self._shell:
self._shell.settimeout(value)
# SSHClient not here because timeout is sent with each command.
@property
def host(self):
return self.host
@host.setter
def host(self, value):
self.host = value
@property
def conn_type(self):
return self.conn_type
@conn_type.setter
def conn_type(self, value):
self.conn_type = value
@property
def username(self):
return self.username
@username.setter
def username(self, value):
self.username = value
@property
def password(self):
return self.password
@password.setter
def password(self, value):
self.password = value
@property
def port(self):
return self.port
@port.setter
def port(self, value):
self.port = value
@property
def connect_timeout(self):
return self.connect_timeout
@connect_timeout.setter
def connect_timeout(self, value):
self.connect_timeout = value
@property
def session_timeout(self):
return self.session_timeout
@session_timeout.setter
def session_timeout(self, value):
self.session_timeout = value
self._update_timeout(value)
def deploy_orchestrator(self):
"""
Deploy Cloudify Manager.
network, security group, fip, VM creation
"""
# network creation
start_time = time.time()
self.__logger.info("Creating keypair ...")
kp_file = os.path.join(self.data_dir, "cloudify_ims.pem")
keypair_settings = KeypairConfig(name='cloudify_ims_kp',
private_filepath=kp_file)
keypair_creator = OpenStackKeypair(self.snaps_creds, keypair_settings)
keypair_creator.create()
self.created_object.append(keypair_creator)
self.__logger.info("Creating full network ...")
subnet_settings = SubnetConfig(name='cloudify_ims_subnet',
cidr='10.67.79.0/24')
network_settings = NetworkConfig(name='cloudify_ims_network',
subnet_settings=[subnet_settings])
network_creator = OpenStackNetwork(self.snaps_creds, network_settings)
network_creator.create()
self.created_object.append(network_creator)
ext_net_name = snaps_utils.get_ext_net_name(self.snaps_creds)
router_creator = OpenStackRouter(
self.snaps_creds,
RouterConfig(name='cloudify_ims_router',
external_gateway=ext_net_name,
internal_subnets=[subnet_settings.name]))
router_creator.create()
self.created_object.append(router_creator)
# security group creation
self.__logger.info("Creating security group for cloudify manager vm")
sg_rules = list()
sg_rules.append(
SecurityGroupRuleConfig(sec_grp_name="sg-cloudify-manager",
direction=Direction.ingress,
protocol=Protocol.tcp,
port_range_min=1,
port_range_max=65535))
sg_rules.append(
SecurityGroupRuleConfig(sec_grp_name="sg-cloudify-manager",
direction=Direction.ingress,
protocol=Protocol.udp,
port_range_min=1,
port_range_max=65535))
securit_group_creator = OpenStackSecurityGroup(
self.snaps_creds,
SecurityGroupConfig(name="sg-cloudify-manager",
rule_settings=sg_rules))
securit_group_creator.create()
self.created_object.append(securit_group_creator)
# orchestrator VM flavor
self.__logger.info("Get or create flavor for cloudify manager vm ...")
flavor_settings = FlavorConfig(
name=self.orchestrator['requirements']['flavor']['name'],
ram=self.orchestrator['requirements']['flavor']['ram_min'],
disk=50,
vcpus=2)
flavor_creator = OpenStackFlavor(self.snaps_creds, flavor_settings)
flavor_creator.create()
self.created_object.append(flavor_creator)
image_settings = ImageConfig(
name=self.orchestrator['requirements']['os_image'],
image_user='******',
exists=True)
port_settings = PortConfig(name='cloudify_manager_port',
network_name=network_settings.name)
manager_settings = VmInstanceConfig(
name='cloudify_manager',
flavor=flavor_settings.name,
port_settings=[port_settings],
security_group_names=[securit_group_creator.sec_grp_settings.name],
floating_ip_settings=[
FloatingIpConfig(
name='cloudify_manager_fip',
port_name=port_settings.name,
router_name=router_creator.router_settings.name)
])
manager_creator = OpenStackVmInstance(self.snaps_creds,
manager_settings, image_settings,
keypair_settings)
self.__logger.info("Creating cloudify manager VM")
manager_creator.create()
self.created_object.append(manager_creator)
public_auth_url = keystone_utils.get_endpoint(self.snaps_creds,
'identity')
self.__logger.info("Set creds for cloudify manager")
cfy_creds = dict(keystone_username=self.snaps_creds.username,
keystone_password=self.snaps_creds.password,
keystone_tenant_name=self.snaps_creds.project_name,
keystone_url=public_auth_url)
cfy_client = CloudifyClient(host=manager_creator.get_floating_ip().ip,
username='******',
password='******',
tenant='default_tenant')
self.orchestrator['object'] = cfy_client
self.__logger.info("Attemps running status of the Manager")
cfy_status = None
retry = 10
while str(cfy_status) != 'running' and retry:
try:
cfy_status = cfy_client.manager.get_status()['status']
self.__logger.debug("The current manager status is %s",
cfy_status)
except Exception: # pylint: disable=broad-except
self.__logger.warning("Cloudify Manager isn't " +
"up and running. Retrying ...")
retry = retry - 1
time.sleep(30)
if str(cfy_status) == 'running':
self.__logger.info("Cloudify Manager is up and running")
else:
raise Exception("Cloudify Manager isn't up and running")
self.__logger.info("Put OpenStack creds in manager")
secrets_list = cfy_client.secrets.list()
for k, val in cfy_creds.iteritems():
if not any(d.get('key', None) == k for d in secrets_list):
cfy_client.secrets.create(k, val)
else:
cfy_client.secrets.update(k, val)
duration = time.time() - start_time
self.__logger.info("Put private keypair in manager")
if manager_creator.vm_ssh_active(block=True):
ssh = manager_creator.ssh_client()
scp = SCPClient(ssh.get_transport(), socket_timeout=15.0)
scp.put(kp_file, '~/')
cmd = "sudo cp ~/cloudify_ims.pem /etc/cloudify/"
run_blocking_ssh_command(ssh, cmd)
cmd = "sudo chmod 444 /etc/cloudify/cloudify_ims.pem"
run_blocking_ssh_command(ssh, cmd)
cmd = "sudo yum install -y gcc python-devel"
run_blocking_ssh_command(
ssh, cmd, "Unable to install packages \
on manager")
self.details['orchestrator'].update(status='PASS', duration=duration)
self.vnf['inputs'].update(
dict(external_network_name=ext_net_name,
network_name=network_settings.name))
self.result = 1 / 3 * 100
return True
def __init__(self, transport):
SCPClient.__init__(self, transport)
import botocore
import paramiko
import glob
from scp import SCPClient
KEY = paramiko.RSAKey.from_private_key_file(
r"C:\Users\user\Desktop\MalKey.pem")
URL = "ec2-13-235-128-119.ap-south-1.compute.amazonaws.com"
DEST = r"/home/ec2-user/apps/"
USER = "******"
DIR = r"apps\*.apk"
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(hostname=URL, username=USER, pkey=KEY, port=22)
scp = SCPClient(client.get_transport())
files = glob.glob(DIR)
for file in files:
name = file.split("apps\\")[1]
scp.put(file, remote_path=DEST)
stdin, stdout, stder = client.exec_command(
"docker cp /home/ec2-user/apps/{} malDetCont:/deploy/").format(name)
dat = {'file_name': name}
resp = requests.post("http://{}:80/predict".format(URL), json=dat)
if resp.status_code == 200:
val = resp.json()
print("File name\t", val['fileName'])
print("App name\t", val['appName'])
print("Number of requested permissions\t", val['permCount'])
print("Making file system writable ... ", end="")
stdout, stdin, stderr = ssh.exec_command(
"sudo mount -o remount,rw / ; sudo mount -o remount,rw /boot"
)
for line in stderr:
print(line)
exit_status = stdout.channel.recv_exit_status()
if exit_status != 0:
print(f"Something's gone wrong! Error exit status: {exit_status}")
quit()
else:
print("Done")
print("Uploading files ... ", end="")
scp = SCPClient(ssh.get_transport())
if args.initial:
scp.put("runCamera")
ssh.exec_command("chmod 755 runCamera")
scp.put(files, recursive=True)
scp.put(main_file, remote_path="~/uploaded.py")
print("Done")
print("Making file system read-only ... ", end="")
ssh.exec_command("sudo mount -o remount,ro / ; sudo mount -o remount,ro /boot")
print("Done")
print("Turning on vision ... ", end="")
ssh.exec_command("sudo svc -u /service/camera")
print("Done")
import paramiko
from scp import SCPClient
def createSSHClient(server, port, user, password):
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(server, port, user, password)
return client
client = createSSHClient('34.199.132.33', '22', 'alumno', 'maipu2021')
scp = SCPClient(client.get_transport())
scp.get('ventasMes.csv')
def main(username, password, test=False, force=False):
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.load_system_host_keys()
ssh.connect('nova.cs.tau.ac.il', username=username, password=password)
scp = SCPClient(ssh.get_transport(), progress = progress)
# copy benchmark unbound receptor and ligand to nova:
all_complex_codes = next(os.walk('../data'))[1]
if test:
all_complex_codes = all_complex_codes[:1]
with open('./copied_successfully.json', 'r') as f:
copied_successfully = json.load(f)
if force:
copied_successfully = []
for complex_code in all_complex_codes:
if len(complex_code) != 4:
continue
print("Copying complex: %s" % complex_code)
try:
ssh.exec_command('mkdir ~/data/' + complex_code + '/')
ssh.exec_command('mkdir ~/data/' + complex_code + '/benchmark/')
ssh.exec_command('mkdir ~/data/' + complex_code + '/patch_dock/')
scp.put('../data/' + complex_code + '/benchmark/' + complex_code + '_l_u.pdb', '~/data/' + complex_code + '/benchmark/')
scp.put('../data/' + complex_code + '/benchmark/' + complex_code + '_r_u.pdb', '~/data/' + complex_code + '/benchmark/')
scp.put("../data/{0}/patch_dock/{0}.patch_dock_output".format(complex_code), '~/data/' + complex_code + '/patch_dock/')
copied_successfully.append(complex_code)
with open('./copied_successfully.json', 'w') as f:
json.dump(copied_successfully, f)
except:
print("ERROR with: %s" % complex_code)
# run patch dock on each benchmark receptor and ligand which doesn't have an ouput file yet
with open('./patch_dock_ran_successfully.json', 'r') as f:
patch_dock_ran_successfully = json.load(f)
if force:
patch_dock_ran_successfully = []
for complex_code in all_complex_codes:
if len(complex_code) != 4 or complex_code in patch_dock_ran_successfully:
continue
print("Running PatchDock on: %s" % complex_code)
try:
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command('cd ~/PatchDock; sh run_patch_dock_on_one_complex.sh ' + complex_code)
print(ssh_stdout.read(), ssh_stderr.read())
if ssh_stderr.read() == b'':
patch_dock_ran_successfully.append(complex_code)
with open('./patch_dock_ran_successfully.json', 'w') as f:
json.dump(patch_dock_ran_successfully, f)
except Exception as e:
print(e.message)
print("ERROR with: %s" % complex_code)
print("DONE")
scp.close()
import paramiko
from scp import SCPClient
ssh_client = paramiko.SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect('10.1.10.200',
port=22,
username='******',
password='******',
look_for_keys=False,
allow_agent=False)
scp = SCPClient(ssh_client.get_transport())
#scp.put('10.1.10.200-2019-10-9.txt', '/var/tmp/newfile.txt')
#Copy a directory
#scp.put('directory1', recursive=True, remote_path='/var/tmp')
scp.get('/var/tmp/directory1/10.1.10.200-2019-10-9.txt', 'copiedfile.txt')
scp.close()
class SSHConnect(object):
"""
Deals with SSHClient and SCPClient functionality
"""
# pylint: disable=line-too-long, too-many-arguments
def __init__(self,
dns_prefix,
location,
user_name,
password=None,
ssh_private_key=None,
port=22):
self._ssh = self.__get_ssh_client(dns_prefix, location, port,
user_name, password, ssh_private_key)
# SCPCLient takes a paramiko transport as its only argument
self._scp = SCPClient(self._ssh.get_transport())
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def __get_ssh_client(self, dns_prefix, location, port, user_name, password,
ssh_private_key):
"""
Gets SSHClient
"""
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
if password and ssh_private_key:
raise NotImplementedError(
'Protected SSH keys are not supported yet')
if password:
ssh.connect(utils.get_remote_host(dns_prefix, location),
port,
username=user_name,
password=password)
if ssh_private_key:
pkey = paramiko.RSAKey.from_private_key_file(ssh_private_key)
ssh.connect(utils.get_remote_host(dns_prefix, location),
port,
username=user_name,
pkey=pkey)
return ssh
def get(self, remote_file, local_file):
"""
Get remote file using SCPClient
"""
self._scp.get(remote_file, local_file)
def put(self, local_file, remote_file):
"""
Put local file to remote using SCPClient
"""
self._scp.put(local_file, remote_file)
def run_command(self, command, async_call=False):
"""
Runs a command on remote machine
"""
if async_call:
transport = self._ssh.get_transport()
channel = transport.open_session()
channel.exec_command('{} > /dev/null 2>&1 &'.format(command))
else:
_, stdout, stderr = self._ssh.exec_command(command)
for line in stdout:
logger.info(line)
for line in stderr:
logger.debug(line)
return stdout, stderr
def close(self):
"""
Close SSHClient and SCPClient
"""
if self._ssh:
self._ssh.close()
if self._scp:
self._scp.close()
def connect(self):
""" Establish a connection to the device.
Purpose: This method is used to make a connection to the junos
| device. The internal property conn_type is what
| determines the type of connection we make to the device.
| - 'paramiko' is used for operational commands (to allow
| pipes in commands)
| - 'scp' is used for copying files
| - 'shell' is used for to send shell commands
| - 'root' is used when logging into the device as root, and
| wanting to send operational commands
| - 'ncclient' is used for the rest (commit, compare_config,
| commit_check)
@returns: None
@rtype: None
"""
if self.conn_type == 'paramiko':
self._session = paramiko.SSHClient()
# These two lines set the paramiko logging to Critical to
# remove extra messages from being sent to the user output.
logger = logging.Logger.manager.getLogger('paramiko.transport')
logger.setLevel(logging.CRITICAL)
self._session.set_missing_host_key_policy(
paramiko.AutoAddPolicy())
self._session.connect(hostname=self.host,
username=self.username,
password=self.password,
port=self.port,
timeout=self.connect_timeout)
if self.conn_type == 'scp':
self._scp_session = paramiko.SSHClient()
logger = logging.Logger.manager.getLogger('paramiko.transport')
logger.setLevel(logging.CRITICAL)
self._scp_session.set_missing_host_key_policy(
paramiko.AutoAddPolicy())
self._scp_session.connect(hostname=self.host,
username=self.username,
password=self.password,
port=self.port,
timeout=self.connect_timeout)
self._scp = SCPClient(self._scp_session.get_transport())
elif self.conn_type == "ncclient":
self._session = manager.connect(
host=self.host,
port=self.port,
username=self.username,
password=self.password,
timeout=self.connect_timeout,
device_params={'name': 'junos'},
hostkey_verify=False
)
elif self.conn_type == 'shell':
if not self._session:
self.conn_type = 'paramiko'
self.connect()
self.conn_type = 'shell'
if not self._shell:
self._shell = self._session.invoke_shell()
time.sleep(2)
if self.username != 'root' and not self._in_cli:
self._in_cli = True
if not self.cli_to_shell():
self._shell.recv(9999)
elif self.conn_type == 'root':
# open the shell if necessary, and move into CLI
if not self._shell:
self._shell = self._session.invoke_shell()
time.sleep(2)
if not self.shell_to_cli():
self._shell.recv(9999)
self._update_timeout(self.session_timeout)
def __init__(self,
file_path,
timer,
onehot=False,
reward_scaler=1e6,
op_age=min,
save_ll=False,
remote=False,
with_age=True,
ip_address=None,
h_size=2):
super(LLVMEnv, self).__init__()
self.file_path = file_path
self.reward_scaler = reward_scaler
self.timer = timer
self.extremum = op_age
self.onehot = onehot
self.MAX_AGE = 10
self.save_ll = save_ll
self.with_age = with_age
self.ssh_con = False
self.scp_con = False
self.ip_address = None
self.h_size = h_size # size of the history
# configure ssh connection to raspberryPi
if remote:
self.ip_address = ip_address
self.ssh_con = paramiko.SSHClient()
self.ssh_con.load_system_host_keys()
self.ssh_con.connect(self.ip_address,
username="******",
password="******")
self.scp_con = SCPClient(self.ssh_con.get_transport())
# instructions handle by the environment
self.instr_to_opcode = {
"call": 1,
"tail": 1,
"store": 2,
"load": 3,
"fadd": 4,
"fsub": 5,
"fmul": 6,
"fdiv": 7,
"bitcast": 8,
"fpext": 9,
"sitofp": 10,
"fptrunc": 11,
"other": 12
}
# self.instr_to_opcode = {
# "alloca": 1,
# "alloca": 1,
# "store": 2,
# "load": 3,
# "fpext": 4,
# "fmul": 5,
# "fdiv": 6,
# "sitofp": 7,
# "other": 8
# }
self.max_step = 50000
self.curr_step = 0
# Define action and state spaces
# 2 actions: choose instruction or not.
self.action_space = spaces.Discrete(2)
if not self.onehot:
if self.with_age:
self.observation_space = spaces.Box(
low=np.array([0, 0, 0, 0, 1]),
high=np.array([12, 12, 12, self.MAX_AGE, 4]),
dtype=int)
else:
self.observation_space = spaces.Box(low=np.array([0, 0, 0, 0]),
high=np.array(
[12, 12, 12, 4]),
dtype=int)
else:
if self.with_age:
low = np.concatenate(
(np.array([0] * (self.h_size + 1) *
len(self.instr_to_opcode)),
np.array([0] * (self.MAX_AGE + 1)), np.array([0] * 4)))
high = np.concatenate(
(np.array([1] * (self.h_size + 1) *
len(self.instr_to_opcode)),
np.array([1] * (self.MAX_AGE + 1)), np.array([1] * 4)))
else:
low = np.concatenate(
(np.array([0] * (self.h_size + 1) *
(len(self.instr_to_opcode))), np.array([0] * 4)))
high = np.concatenate(
(np.array([1] * (self.h_size + 1) *
(len(self.instr_to_opcode))), np.array([1] * 4)))
self.observation_space = spaces.Box(low=low, high=high, dtype=int)
self.selected_instruction = None # current selected instruction
self.schedulable_instructions = None
# last scheduled instructions's opcode
self.history = collections.deque([0] * (self.h_size), self.h_size)
self.llvm = LLVMController(self.file_path)
print("timing original file")
self.original_time = self.llvm.run(self.timer,
self.curr_step,
False,
self.ssh_con,
self.scp_con,
self.ip_address,
time_original=True)
print(f"original program time is: {self.original_time}s")
class Connection:
def __init__(self,
username: Union[str, Path] = None,
host: Union[str, Path] = None,
protocol: str = "sftp",
*args,
**kwargs):
self.user = username or input("Enter username: "******"Enter host URL: ")
self._args = list(*args)
self._kwargs = {**kwargs}
self.__c = None
if protocol.lower() in ["sftp", "scp"]:
self.protocol = protocol.lower()
else:
raise ValueError('Protocol must be "sftp" or "scp".')
def update(self, **kwargs):
self._kwargs = kwargs
def __call__(self, **kwargs):
self.update(**kwargs)
return self
def __str__(self):
return "Connection to {} as {}".format(self.host, self.user)
def __repr__(self):
return "<{}.{} object at {}>".format(self.__class__.__module__,
self.__class__.__name__,
hex(id(self)))
def connect(self, **kwargs):
try:
keywords = (dict(**kwargs) or dict(**self._kwargs)
or dict(password=getpass("Enter password: "******"sftp":
c = fabric.Connection(host=self.host,
user=self.user,
connect_kwargs=keywords)
self.__c = c
else:
c = SSHClient()
c.connect(
self.host,
username=self.user,
password=self._kwargs["password"]
or getpass("Enter password: "),
)
self.__c = SCPClient(c.get_transport())
return self.__c
except Exception as e:
raise e
def __enter__(self, **kwargs):
return self.connect()
def __exit__(self, exc_type, exc_val, exc_tb):
self.__c.close()
class TestProgressCallback(unittest.TestCase):
def setUp(self):
self.peer = ('localhost', 42345)
class T(object):
def getpeername(s):
return self.peer
self.scp = SCPClient(T())
def call_progress(self, progress):
self.scp._progress_tracker(progress, 'name', 100, 42, self.peer)
def test_functions(self):
called = [0]
def progress(name, size, pos):
called[0] += 1
self.call_progress(progress)
self.assertEqual(called, [1])
def progress(name, size, pos, peername):
called[0] += 1
self.call_progress(progress)
self.assertEqual(called, [2])
def test_methods(self):
called = [0]
class C(object):
def method3(self, name, size, pos):
called[0] += 1
def method4(self, name, size, pos, peername):
called[0] += 1
@classmethod
def c_method3(cls, name, size, pos):
called[0] += 1
@classmethod
def c_method4(cls, name, size, pos, peername):
called[0] += 1
@staticmethod
def s_method3(name, size, pos):
called[0] += 1
@staticmethod
def s_method4(name, size, pos, peername):
called[0] += 1
self.call_progress(C().method3)
self.assertEqual(called, [1])
self.call_progress(C().method4)
self.assertEqual(called, [2])
self.call_progress(C().c_method3)
self.assertEqual(called, [3])
self.call_progress(C().c_method4)
self.assertEqual(called, [4])
self.call_progress(C().s_method3)
self.assertEqual(called, [5])
self.call_progress(C().s_method4)
self.assertEqual(called, [6])
class SearchVideo():
def __init__(self, master):
self.parent = master
self.ADDRESS = tk.StringVar()
self.ADDRESS.set('')
self.USER = tk.StringVar()
self.USER.set('')
self.PASSWD = tk.StringVar()
self.PASSWD.set('')
self.Path = tk.StringVar()
self.Path.set('/')
self.SearchVideoPanel = tk.LabelFrame(self.parent,
text="Search Video",
font=('Courier', 10))
self.SearchVideoPanel.pack(side=tk.LEFT, expand=tk.YES, fill=tk.BOTH)
self.init_ctrlPanel()
self.init_Stream_tab()
def init_ctrlPanel(self):
self.ctrlPanel = tk.LabelFrame(self.SearchVideoPanel,
text="Control Panel",
font=('Courier', 10))
self.ctrlPanel.pack(side=tk.TOP, expand=tk.NO, fill=tk.X)
tk.Label(self.ctrlPanel,
text='IP Address',
font=('Courier', 10),
width=10,
height=2).grid(row=0, column=0, sticky=tk.E + tk.W)
self.Address = tk.Entry(self.ctrlPanel,
textvariable=self.ADDRESS,
font=('Courier', 10))
self.Address.grid(row=0, column=1, sticky=tk.E + tk.W)
tk.Label(self.ctrlPanel,
text='User',
font=('Courier', 10),
width=6,
height=2).grid(row=0, column=2, sticky=tk.E + tk.W)
self.User = tk.Entry(self.ctrlPanel,
textvariable=self.USER,
font=('Courier', 10))
self.User.grid(row=0, column=3, sticky=tk.E + tk.W)
tk.Label(self.ctrlPanel,
text='Password',
font=('Courier', 10),
width=8,
height=2).grid(row=0, column=4, sticky=tk.E + tk.W)
self.Passwd = tk.Entry(self.ctrlPanel,
textvariable=self.PASSWD,
show="*",
width=12,
font=('Courier', 10))
self.Passwd.grid(row=0, column=5, sticky=tk.E + tk.W)
self.ConfButton = tk.Button(self.ctrlPanel,
text="Config",
font=('Courier', 10),
command=self.Config)
self.ConfButton.grid(row=1, column=2, sticky=tk.E + tk.W)
#43)
self.ssh_loginButton = tk.Button(self.ctrlPanel,
text="ssh_login",
font=('Courier', 10),
command=self.ssh_login)
self.ssh_loginButton.grid(row=1, column=3, sticky=tk.E + tk.W)
#44)
self.ssh_closeButton = tk.Button(self.ctrlPanel,
text="ssh_close",
font=('Courier', 10),
command=self.ssh_close)
self.ssh_closeButton.grid(row=1, column=4, sticky=tk.E + tk.W)
def ssh_login(self, event=None):
self.ssh = SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.load_system_host_keys()
echo = self.ssh.connect(hostname=self.ADDRESS,
username=self.USER,
password=self.PASSWD)
#print(echo)
# SCPCLient takes a paramiko transport as an argument
self.scp = SCPClient(self.ssh.get_transport())
tkmsg.showinfo(
"Information",
"ssh " + self.USER + "@" + self.ADDRESS + " Connection is setup")
def ssh_close(self, event=None):
echo = self.ssh.close()
print(echo)
tkmsg.showinfo(
"Information",
"ssh " + self.USER + "@" + self.ADDRESS + " Connection is closed")
def Config(self, event=None):
self.ADDRESS = self.Address.get()
self.USER = self.User.get()
self.PASSWD = self.Passwd.get()
tkmsg.showinfo(
"Information", "IP Address:" + self.ADDRESS + " User:"******" Pass Word:" + self.PASSWD)
def DisplayVideoInfo_Clear(self, event=None):
#print("Doc_Clear")
self.DisplaySceneMarkInfo.delete('1.0', tk.END)
tkmsg.showinfo("Information", "CLEAR")
def Search_DeviceVideo(self, event=None):
rawcommand = 'find {path} -name {pattern}'
command = rawcommand.format(path=self.Path.get(), pattern='*.mp4')
stdin, stdout, stderr = self.ssh.exec_command(command)
#filelist = stdout.read()#.splitlines()
filelist = stdout.readlines()
#if not os.path.exists(os.getcwd()+"/DeviceVideo"):
# os.mkdir("DeviceVideo")
for afile in filelist:
(head, filename) = os.path.split(afile)
#print(filename)
#print(afile)
self.Table_of_DiscoverVideo.insert("",
index='end',
text=filename,
values=(afile))
#####self.scp.get(afile.rstrip('\n'), "DeviceVideo/"+filename)
#print("DeviceVideo/"+filename)
def SelectDevVideo(self, event=None):
for item in self.Table_of_DiscoverVideo.selection():
self.item_text = self.Table_of_DiscoverVideo.item(item, "values")
tkmsg.showinfo("Information", self.item_text[0])
def Download_DeviceVideo(self, event=None):
#print("Download_DeviceVideo")
if not os.path.exists(os.getcwd() + "/DeviceVideo"):
os.mkdir("DeviceVideo")
(head, filename) = os.path.split(self.item_text[0])
self.scp.get(self.item_text[0].rstrip('\n'), "DeviceVideo/" + filename)
tkmsg.showinfo("Information", self.item_text[0] + "Download Finished")
def init_Stream_tab(self):
self.Stream_tab = tk.Frame(self.SearchVideoPanel)
#self.notebook.add(self.Stream_tab, text="Stream")
self.Stream_tab.pack(side=tk.TOP, expand=tk.YES, fill=tk.BOTH)
self.StreamCtrlPanel = tk.LabelFrame(self.Stream_tab,
text="Stream Control Panel",
font=('Courier', 10))
self.StreamCtrlPanel.pack(side=tk.TOP, expand=tk.NO, fill=tk.X)
tk.Label(self.StreamCtrlPanel,
text='Path',
font=('Courier', 10),
width=8).pack(side=tk.LEFT, expand=tk.NO, fill=tk.BOTH)
self.UpPath = tk.Entry(self.StreamCtrlPanel,
textvariable=self.Path,
font=('Courier', 10))
self.UpPath.pack(side=tk.LEFT, expand=tk.NO, fill=tk.BOTH)
SearDevVideobutton = tk.Button(self.StreamCtrlPanel,
font=('Courier', 10),
text="Search Video",
command=self.Search_DeviceVideo)
SearDevVideobutton.pack(side=tk.LEFT, expand=tk.NO, fill=tk.BOTH)
DowlDevVideobutton = tk.Button(self.StreamCtrlPanel,
font=('Courier', 10),
text="Download",
command=self.Download_DeviceVideo)
DowlDevVideobutton.pack(side=tk.LEFT, expand=tk.NO, fill=tk.BOTH)
self.Table_of_DiscoverVideo = ttk.Treeview(self.Stream_tab,
columns=["#1"],
height=30)
self.Table_of_DiscoverVideo.heading(
"#0",
text="Search of Videos",
) #icon column
self.Table_of_DiscoverVideo.heading("#1", text="Path")
self.Table_of_DiscoverVideo.column("#0", width=320) #icon column
self.Table_of_DiscoverVideo.column("#1", width=820)
self.Table_of_DiscoverVideo.tag_configure('T', font='Courier,4')
self.Table_of_DiscoverVideo.bind("<Double-1>", self.SelectDevVideo)
self.Table_of_DiscoverVideo.pack(side=tk.TOP, expand=tk.NO, fill=tk.Y)
'sudo pip install --upgrade google-api-python-client',
'sudo pip install beaker', 'sudo pip install peewee',
'sudo apt-get install unzip'
]
for n, com in enumerate(commands):
stdin, stdout, stderr = client.exec_command(com)
if n == 1:
# time.sleep(3)
stdin.write('y\n')
time.sleep(100)
else:
time.sleep(20)
print "executed '" + com + "'"
if __name__ == "__main__":
scp = SCPClient(client.get_transport())
print "Uploading Webserver"
scp.put(file_name)
time.sleep(5)
stdin, stdout, stderr = client.exec_command("unzip " + file_name)
print "Setting up server"
time.sleep(10)
stdin, stdout, stderr = client.exec_command("cd " + file_name[:-4])
time.sleep(1)
print "Executing"
stdin, stdout, stderr = client.exec_command(
"screen -d -m sudo python server.py")
time.sleep(2)
# print stdout.next()
print "connection %s" % ("ok" if connection else "failed")
ourCrop = ourImage.crop((ourX, (chunk - 1) * 10, ourX + 1, chunk * 10))
ourMath = ImageStat.Stat(ourCrop)
pprint(ourMath.mean)
color = (round(ourMath.mean[0]), round(ourMath.mean[1]),
round(ourMath.mean[2]), round(ourMath.mean[3]))
if (color[0] > 0 or color[1] > 0 or color[2] > 0):
ourOffset = (chunk - 1) * 10
break
s = sched.scheduler(time.time, time.sleep)
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect("raspbmc.local", username="******")
scp = SCPClient(ssh.get_transport())
iterator = 99
enabled = 1
lastMillis = millis()
def doLoop(sc):
global iterator, enabled, ourScreenWidth, lastMillis, ourOffset
iterator += 1
sc.enter(0.2, 1, doLoop, (sc, ))
if (iterator == 100):
class RemoteClient:
"""Client to interact with a remote host via SSH & SCP."""
def __init__(self, host, user, ssh_key_filepath, remote_path):
self.host = host
self.user = user
self.ssh_key_filepath = ssh_key_filepath
self.remote_path = remote_path
self.client = None
self.scp = None
self.conn = None
self._upload_ssh_key()
@logger.catch
def _get_ssh_key(self):
"""
Fetch locally stored SSH key.
"""
try:
self.ssh_key = RSAKey.from_private_key_file(self.ssh_key_filepath)
logger.info(f'Found SSH key at self {self.ssh_key_filepath}')
except SSHException as error:
logger.error(error)
return self.ssh_key
@logger.catch
def _upload_ssh_key(self):
try:
system(
f'ssh-copy-id -i {self.ssh_key_filepath} {self.user}@{self.host}>/dev/null 2>&1'
)
system(
f'ssh-copy-id -i {self.ssh_key_filepath}.pub {self.user}@{self.host}>/dev/null 2>&1'
)
logger.info(f'{self.ssh_key_filepath} uploaded to {self.host}')
except FileNotFoundError as error:
logger.error(error)
@logger.catch
def _connect(self):
"""Open connection to remote host."""
if self.conn is None:
logger.info(f'host: {self.host}, user: {self.user}')
try:
self.client = SSHClient()
self.client.load_system_host_keys()
self.client.set_missing_host_key_policy(AutoAddPolicy())
self.client.connect(self.host,
username=self.user,
key_filename=self.ssh_key_filepath,
look_for_keys=True,
timeout=5000)
self.scp = SCPClient(self.client.get_transport())
except AuthenticationException as error:
logger.error(f'Authentication failed: \
did you remember to create an SSH key? {error}')
raise error
return self.client
def disconnect(self):
"""Close ssh connection."""
if self.client:
self.client.close()
if self.scp:
self.scp.close()
@logger.catch
def bulk_upload(self, files):
"""
Upload multiple files to a remote directory.
:param files: List of paths to local files.
:type files: List[str]
"""
self.conn = self._connect()
uploads = [self._upload_single_file(file) for file in files]
logger.info(
f'Finished uploading {len(uploads)} files to {self.remote_path} on {self.host}'
)
def _upload_single_file(self, file):
"""Upload a single file to a remote directory."""
upload = None
try:
self.scp.put(file, recursive=True, remote_path=self.remote_path)
upload = file
except SCPException as error:
logger.error(error)
raise error
finally:
logger.info(f'Uploaded {file} to {self.remote_path}')
return upload
def download_file(self, file):
"""Download file from remote host."""
self.conn = self._connect()
self.scp.get(file)
def put(self, files, remote_path=b'.', recursive=False):
client = self._get_client()
with SCPClient(client.get_transport()) as scp:
scp.put(files, remote_path, recursive)
def cpfiles(self):
with SCPClient(ssh.get_transport()) as scp:
scp.put('a.csv', 'b.csv')
scp.get('23.txt')
from scp import SCPClient
#default paramiko config
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
#testing linux VM
ssh.connect("192.168.198.142",
port=22,
username="******",
password="******",
look_for_keys=False,
allow_agent=False)
#create 'scp' variable that calls SCPClient function over paramiko transport established by 'ssh'
scp = SCPClient(ssh.get_transport())
# to UPLOAD FILE from this PC to remote device
# ("from where" "to where"):
# on Windows it is '\\' used when on linux is '/' for paths
scp.put("C:\\Users\\test.txt", "/home/pfne/scp_test/test.txt")
# to UPLOAD Directory from this PC to remote device
# 'recursive' means copy all the content of this directory
# WARNING! it can overwrite files inside of directory
scp.put("C:\\Users\\venv", recursive=True, remote_path="/home/pfne/")
# to DOWNLOAD FILE from remote device to this PC
scp.get("/home/pfne/scp_test.txt", "C:\\Users\\")
# always close connection at end
class scpData:
"""
ssh and scp combination toolkit for file transfers
Parameters
----------
password : str
password for ssh server
address : str
address for ssh server
port : int
port number for ssh server
username : str
username for ssh server
Attributes
----------
ssh : 'paramiko.SSHClient'
scp : 'scp.SCPClient'
Notes
-----
References
----------
https://gist.github.com/stonefury/06ab3531a1c30c3b998a
https://github.com/jbardin/scp.py
Examples
--------
"""
# def __init__(self,password=getPwd(),username=_pref._HBT_SERVER_USERNAME,
# address=_pref._HBT_SERVER_ADDRESS,port=22):
def __init__(self,
password,
username=_HBT_SERVER_USERNAME,
address=_pref._HBT_SERVER_ADDRESS,
port=22):
from paramiko import SSHClient
from scp import SCPClient
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
self.ssh.connect(address,
port=port,
username=username,
password=password)
self.scp = SCPClient(self.ssh.get_transport())
def downloadFile(self, remoteFilePath, localFilePath=''):
# try:
self.scp.get(remoteFilePath, localFilePath)
# except:
# print("%s not present. skipping..." % remoteFilePath)
# pass
def uploadFile(self, localFilePath, remoteFilePath):
self.scp.put(localFilePath, remoteFilePath)
def uploadFolder(self, localDirPath, remoteDirPath):
""" upload folder and all contents """
self.scp.put(localDirPath, remoteDirPath, recursive=True)
def closeConnection(self):
self.scp.close()
self.ssh.close()
def __del__(self):
""" upon deletion of object, close ssh/scp connections """
self.closeConnection()
ssh.load_system_host_keys()
ssh.connect('sundog.uchicago.edu',username='******')
loading_string = ''
while True:
files = []
for ext in extensions:
files.extend(glob.glob(input_dir + ext))
for f in files:
f = f.replace('\\','/') # To handle windows paths.
t = os.path.getmtime(f)
if t >= call_time:
loading_string = '.'
# SCPCLient takes a paramiko transport and progress callback as its arguments.
# Might want to look into a keep alive if I only want to ssh once and not every call.
with SCPClient(ssh.get_transport(), progress=progress) as scp:
#ready file
f_new = output_dir + f.split('/')[-1]
try:
print('Removing %s'%f)
os.remove(f)
except Exception as e:
print('Failed deleting %s.'%f)
print(e)
#met file
try:
f = f.replace('.ready','.met')
f_new = f_new.replace('.ready','.met')
print('Transferring met file modified since last call')
scp.put(f, f_new)
print('\n')
class RemoteClient:
""" Client to interact with remote host via SSH & SCP """
def __init__(self, host, user, ssh_key_filepath, password=None ,remote_path='/'):
self.host = host
self.user = user
self.password = password
self.ssh_key_filepath = ssh_key_filepath
self.remote_path = remote_path
self.client = None
self.scp = None
self.__upload_ssh_key()
def __get_ssh_key(self):
"""Fetch locally stored SSH key."""
try:
self.ssh_key = RSAKey.from_private_key_file(self.ssh_key_filepath)
logger.info('Found SSH key at self {self.ssh_key_filepath}')
except SSHException as error:
logger.error(error)
return self.ssh_key
def __upload_ssh_key(self):
try:
system(f'ssh-copy-id -i {self.ssh_key_filepath} {self.user}@{self.host}>/dev/null 2>&1')
system(f'ssh-copy-id -i {self.ssh_key_filepath}.pub {self.user}@{self.host}>/dev/null 2>&1')
logger.info(f'{self.ssh_key_filepath} uploaded to {self.host}')
except IOError as error:
logger.exception(error)
def __connect(self):
"""Open connection to remote host.
@:raises PasswordRequiredException if private key file is encrypted. This should be handled by CLI by asking the
password from the user at the command line.
@:raises AuthenticationException if the user failed to specify proper ssh keys. The CLI should print the exception
and exit.
"""
try:
self.client = SSHClient()
self.client.load_system_host_keys()
self.client.set_missing_host_key_policy(AutoAddPolicy())
if self.password is not None:
self.client.connect(self.host,
username=self.user,
password = self.password,
key_filename=self.ssh_key_filepath,
look_for_keys=True,
timeout=5000)
else:
self.client.connect(self.host,
username=self.user,
key_filename=self.ssh_key_filepath,
look_for_keys=True,
timeout=5000)
self.scp = SCPClient(self.client.get_transport())
except PasswordRequiredException as error:
logger.exception(error)
raise error
except AuthenticationException as error:
logger.info('Authentication failed: did you remember to create an SSH key?')
logger.error(error)
raise error
return self.client
def disconnect(self):
"""Close ssh connection."""
self.client.close()
self.scp.close()
def execute_commands(self, commands):
"""Execute multiple commands in succession."""
if self.client is None:
self.client = self.__connect()
for cmd in commands:
stdin, stdout, stderr = self.client.exec_command(cmd)
stdout.channel.recv_exit_status()
response = stdout.readlines()
for line in response:
logger.info(f'INPUT: {cmd} | OUTPUT: {line}')
def bulk_upload(self, files):
"""Upload multiple files to a remote directory."""
if self.client is None:
self.client = self.__connect()
uploads = [self.__upload_single_file(file) for file in files]
logger.info(f'Uploaded {len(uploads)} files to {self.remote_path} on {self.host}')
def __upload_single_file(self, file):
"""Upload a single file to a remote directory."""
try:
self.scp.put(file,
recursive=True,
remote_path=self.remote_path)
except SCPException as error:
logger.error(error)
raise error
finally:
return file
def download_file(self, file):
"""Download file from remote host."""
if self.conn is None:
self.conn = self.connect()
self.scp.get(file)
def _publish_code(package_type, d_files, package_name, ssh, ip, ssh_port, app_port, package_path, package_md5,execute):
#上线操作前进行数据备份
_console_out(channel, Msg_Key,"ip:%s ssh_port:%s app_port:%s --->start deploy %s......" % (ip, ssh_port, app_port,project))
if execute == 'publish':
try:
cmd = "[ -e %s%s ] && echo ok" % (web_path, package_name)
stdin, stdout, stderr = ssh.exec_command(cmd)
result = str(stdout.read().strip(),encoding='utf8')
if result == 'ok':
if package_name.endswith('.war'):
cmd = "\cp -rf %s%s %s%s" % (web_path, package_name, bak_path,package_name)
else:
cmd = "/usr/bin/rsync -av --delete %s%s/ %s%s/" % (web_path, package_name, bak_path, package_name)
stdin, stdout, stderr = ssh.exec_command(cmd)
result = stderr.read()
if result:
_flow_log(flow_number,"Error:%s"%result)
return "ip:%s ssh_port:%s app_port:%s --->backup Fail !" % (ip, ssh_port, app_port)
except Exception as e:
_flow_log(flow_number,'Error:%s'%str(e))
return "ip:%s ssh_port:%s app_port:%s --->backup Fail !" % (ip, ssh_port, app_port)
scp = SCPClient(ssh.get_transport())
#增量包部署
if package_type == 'part':
try:
scp.put("%s/"%d_files, "%s%s/" % (web_path, package_name), recursive=True)
except Exception as e:
# 传输错误重试3次
for i in range(3):
time.sleep(3)
try:
scp.put("%s/" % d_files, "%s%s/" % (web_path, package_name), recursive=True)
except:
if i >=2:
break
continue
else:
cmd = "chown %s:%s -R %s%s/" % (service_user, service_user, web_path, package_name)
stdin, stdout, stderr = ssh.exec_command(cmd)
result = stderr.read()
if result:
_flow_log(flow_number, "Error:%s" % result)
return "ip:%s ssh_port:%s web_path:%s%s --->chown Fail !" % (ip, ssh_port, web_path, package_name)
_flow_log(flow_number,'Error:%s'%str(e))
return "ip:%s ssh_port:%s app_port:%s --->deploy Fail !" % (ip, ssh_port, app_port)
else:
cmd = "chown %s:%s -R %s%s/" %(service_user,service_user,web_path, package_name)
stdin, stdout, stderr = ssh.exec_command(cmd)
result = stderr.read()
if result:
_flow_log(flow_number, "Error:%s" % result)
return "ip:%s ssh_port:%s web_path:%s%s --->chown Fail !" % (ip, ssh_port,web_path, package_name)
#整包部署
if package_type == 'full':
try:
d_zip = "%s%s" % (web_path, package_path.split('/')[-1])
try:
scp.put(package_path, d_zip)
except Exception as e:
# 传输错误重试3次
for i in range(3):
time.sleep(3)
try:
scp.put(package_path, d_zip)
except:
if i >= 2:
break
continue
_flow_log(flow_number,'Error:%s'%str(e))
return "ip:%s ssh_port:%s app_port:%s --->transfers Fail !" % (ip, ssh_port, app_port)
cmd = '/usr/bin/md5sum %s' % d_zip
stdin, stdout, stderr = ssh.exec_command(cmd)
R_md5 = str(stdout.read().split()[0],encoding='utf8')
if R_md5 == package_md5:
package_zip = package_path.split('/')[-1]
cmd = "cd %s && /usr/bin/unzip -qo %s && /bin/rm -f %s && [ -e %s ] && echo ok" % (web_path,package_zip,package_zip,package_zip.replace('.zip', ''))
stdin, stdout, stderr = ssh.exec_command(cmd)
result_zip = str(stdout.read().strip(),encoding='utf8')
result = stderr.read()
if result_zip == 'ok':
cmd = "cd %s && /bin/rm -rf %s{,.war} &&/bin/mv %s %s" % (web_path,project,package_zip.replace('.zip', ''), package_name)
stdin, stdout, stderr = ssh.exec_command(cmd)
result = stderr.read()
if result:
_flow_log(flow_number,"Error:%s"%result)
return "ip:%s ssh_port:%s app_port:%s --->deploy Fail !" % (ip, ssh_port, app_port)
else:
cmd = "chown %s:%s %s%s" % (service_user, service_user, web_path, package_name)
stdin, stdout, stderr = ssh.exec_command(cmd)
result = stderr.read()
if result:
_flow_log(flow_number, "Error:%s" % result)
return "ip:%s ssh_port:%s web_path:%s%s --->chown Fail !" % (ip, ssh_port, web_path, package_name)
else:
_flow_log(flow_number, "Error:%s" % result)
return "ip:%s ssh_port:%s app_port:%s --->unzip Fail !" % (ip, ssh_port, app_port)
else:
return "ip:%s ssh_port:%s app_port:%s --->md5 Fail !" % (ip, ssh_port, app_port)
except Exception as e:
if 'old-style' not in str(e):
_flow_log(flow_number,'Error:%s'%str(e))
return "ip:%s ssh_port:%s app_port:%s --->deploy Fail !" % (ip, ssh_port, app_port)
def transfer_file(self,
hostname=None,
username=None,
password=None,
allow_agent=False,
look_for_keys=False,
ssh_config=None):
"""Transfer the file to the remote device over SCP.
Note:
If any arguments are omitted, the corresponding attributes
of the ``self.device`` will be used.
Args:
hostname (str): OPTIONAL - The name or
IP address of the remote device.
proto (str): OPTIONAL - Protocol to be used
for transfer - 'scp' or 'sftp'
username (str): OPTIONAL - The SSH username
for the remote device.
password (str): OPTIONAL - The SSH password
for the remote device.
Raises:
FileTransferError: if an error occurs during the file transfer.
FileHashMismatchError: if the source and
destination hashes don't match.
FileNotReadableError: if the local file doesn't exist or isn't readable.
FileNotEnoughSpaceError: if there isn't enough space on the device.
FileRemoteDirDoesNotExist: if the remote directory doesn't exist.
"""
hostname = hostname or self.device.host
username = username or self.device.username
password = password or self.device.password
ssh_config = ssh_config or self.device.ssh_config
key_filename = None
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
if ssh_config is not None:
sshconfig = paramiko.SSHConfig()
sshconfig.parse(open(ssh_config))
cfg = sshconfig.lookup('dev')
key_filename = cfg['identityfile'][0]
ssh.connect(hostname=hostname,
username=username,
password=password,
port=self.port,
allow_agent=allow_agent,
look_for_keys=look_for_keys,
key_filename=key_filename)
if self.proto == "scp":
scp = SCPClient(ssh.get_transport())
try:
scp.get(self.src, self.dst)
except:
raise FileTransferError
scp.close()
elif self.proto == "sftp":
sftp = ssh.open_sftp()
sftp.sshclient = ssh
try:
sftp.get(self.src.strip('flash:'), self.dst)
except:
raise FileTransferError
sftp.close()
else:
raise FileTransferUnsupportedProtocol(self.proto)
src_hash = self._get_local_md5()
dst_hash = self._get_remote_md5()
if src_hash != dst_hash:
raise FileHashMismatchError(self.src, self.dst, src_hash, dst_hash)
