def copy_file(local_file_path, dest_path, host=cfg.HOST_CONF['HOST'], port=cfg.HOST_CONF['PORT'], user=cfg.HOST_CONF['USER'], password=cfg.HOST_CONF['PASSWORD'], mode = 'PUT'):
transport = None
if not path.exists(local_file_path):
if mode is 'GET':
mkdir(local_file_path)
else:
print "Local file path doesnt exist %s" %(local_file_path)
paramiko.util.log_to_file(cfg.PARAMIKO_LOG)
transport = paramiko.Transport((host, port))
try:
transport.connect(username=user, password=password)
transport.get_exception()
scp = SCPClient(transport)
except:
print "Error while connecting to host %s:%s" %(host, port)
#scp.put('test_eth.sh', 'test_eth_1.sh')
#scp.get('test_eth_1.sh')
if mode is 'PUT':
#print "copying file from %s to %s on host %s " %(local_file_path, dest_path, host)
scp.put(local_file_path, dest_path)
else:
#print scp.get(dest_path,local_file_path, recursive=True)
scp.get(dest_path, recursive=True)
#print scp.get(dest_path, local_path="path_to_directory where store this directory", recursive= True)
scp.close()
def run_main(args):
if not os.path.exists(SCRIPT_FILENAME):
print 'ERROR: Please run "solo script pack" first to bundle your archive.'
return 1
print 'connecting to solo...'
solo = soloutils.connect_solo(await=True)
scp = SCPClient(solo.get_transport())
# Requires pip
print 'checking pip...'
if soloutils.install_pip.run(solo, scp) != 0:
print 'failed installing pip.'
sys.exit(1)
push(solo, scp, '--force' in args['<arg>'])
print 'running script...'
print ''
soloutils.command_stream(solo, '''
set -e
cd /log/solo-script
source ./env/bin/activate
exec python /log/solo-script/''' + args['<arg>'][1] + '''
''')
scp.close()
solo.close()
def transfer_file(self, pull=False):
if pull is False:
if not self.local_file_exists():
raise FileTransferError(
'Could not transfer file. Local file doesn\'t exist.')
if not self.enough_remote_space():
raise FileTransferError(
'Could not transfer file. Not enough space on device.')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
hostname=self.device.host,
username=self.device.username,
password=self.device.password,
port=self.port,
allow_agent=False,
look_for_keys=False)
scp = SCPClient(ssh.get_transport())
try:
if pull:
scp.get(self.remote, self.local)
else:
scp.put(self.local, self.remote)
except Exception as e:
print(e)
raise FileTransferError
finally:
scp.close()
return True
def get_file(fname):
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(file_server)
scp = SCPClient(ssh.get_transport())
scp.get(fname)
scp.close()
def transfer_file(self, hostname=None, username=None, password=None, pull=False):
"""Transfer the file to the remote device over SCP.
Note:
If any arguments are omitted, the corresponding attributes
of ``self.device`` will be used.
Args:
hostname (str): OPTIONAL - The name or
IP address of the remote device.
username (str): OPTIONAL - The SSH username
for the remote device.
password (str): OPTIONAL - The SSH password
for the remote device.
Returns:
True if successful.
Raises:
FileTransferError: if the transfer isn't successful.
"""
if pull is False:
if not self.local_file_exists():
raise FileTransferError(
'Could not transfer file. Local file doesn\'t exist.')
if not self.enough_space():
raise FileTransferError(
'Could not transfer file. Not enough space on device.')
hostname = hostname or self.device.host
username = username or self.device.username
password = password or self.device.password
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
hostname=hostname,
username=username,
password=password,
port=self.port,
allow_agent=False,
look_for_keys=False)
full_remote_path = '{}{}'.format(self.file_system, self.dst)
scp = SCPClient(ssh.get_transport())
try:
if pull:
scp.get(full_remote_path, self.src)
else:
scp.put(self.src, full_remote_path)
except:
raise FileTransferError(
'Could not transfer file. There was an error during transfer. Please make sure remote permissions are set.')
finally:
scp.close()
return True
def scp(file, path, ip, user, password):
"""Uses paramiko and scp module built on top of it to SCP vEOS to the VM.
Inputs are the .swi filename from arguments, and a path."""
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(hostname=ip, username=user, password=password)
scp = SCPClient(ssh.get_transport())
print "\nWriting " + file + " to " + path
scp.put(file, remote_path=path)
scp.close()
def secure_copy(user, host, src, dest, key_filename=None, allow_agent=True):
keys = _load_keys(key_filename, allow_agent)
pkey = keys[0]
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host, username=user, pkey=pkey)
scp = SCPClient(ssh.get_transport())
scp.get(src, dest)
scp.close()
def transport_assembly_run( seed_name, tfilename, workdir='./build' ):
# Push package: to-be processed packages are placed in 'put' directory.
# Pulled package: finished work is in the 'get' directory.
retry = 1
# Credentials wide out in the open:
ssh = createSSHClient(CL_SERVER_NAME, 22, CL_SERVER_USER, CL_SERVER_PW)
while (retry < 10):
try:
scp = SCPClient(ssh.get_transport())
scp.put(tfilename, 'run')
# If it got this far, it succeeded
break;
except Exception as err:
print '=============='
print err[0]
print ' Retry attemp #'+retry
print '=============='
sleep(5)
#
# Server Processing Phase
#
waittime = 0
while (waittime < 600):
try:
scp = SCPClient(ssh.get_transport())
# Check the log file
scp.get('get/'+seed_name+'_run.log', local_path=workdir)
line = tail("-1", os.path.join(workdir,seed_name+'_run.log') ).strip()
print line
if 'Assembly Complete' in line:
break
if 'ERROR' in line:
exit(1)
except Exception as err:
print '=============='
print err[0]
print '=============='
if 'No such file or directory' in err[0]:
print "---- No status yet ----"
sleep(2)
waittime = waittime + 1
scp.close()
# Compile complete. Download results.
try:
scp.get('get/'+seed_name+'_assembly.dat', local_path=workdir)
os.rename( os.path.join(workdir, seed_name+'_assembly.dat'), os.path.join(workdir,'assembly.dat'))
except Exception as err:
print '=============='
print err[0]
print '=============='
exit(1)
return True
def SecureCopy(user, host, src, dest,
key_filename=os.path.join(os.path.expanduser("~"), '.ssh', 'id_rsa')):
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host, username=user, key_filename=key_filename)
scp = SCPClient(ssh.get_transport())
scp.get(src, dest)
scp.close()
def fetch(fileDirectory):
print('Fetching ' + fileDirectory)
client = connect()
try:
scp = SCPClient(client.get_transport())
scp.get(fileDirectory)
scp.close()
client.close()
return True
except:
client.close()
return False
def transfer_file(self, hostname=None, username=None, password=None):
"""Transfer the file to the remote device over SCP.
Note:
If any arguments are omitted, the corresponding attributes
of the ``self.device`` will be used.
Args:
hostname (str): OPTIONAL - The name or
IP address of the remote device.
username (str): OPTIONAL - The SSH username
for the remote device.
password (str): OPTIONAL - The SSH password
for the remote device.
Raises:
FileTransferError: if an error occurs during the file transfer.
FileHashMismatchError: if the source and
destination hashes don't match.
FileNotReadableError: if the local file doesn't exist or isn't readable.
FileNotEnoughSpaceError: if there isn't enough space on the device.
FileRemoteDirDoesNotExist: if the remote directory doesn't exist.
"""
self._safety_checks()
hostname = hostname or self.device.host
username = username or self.device.username
password = password or self.device.password
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
hostname=hostname,
username=username,
password=password,
port=self.port,
allow_agent=False,
look_for_keys=False)
scp = SCPClient(ssh.get_transport())
try:
scp.put(self.src, self.dst)
except:
raise FileTransferError
scp.close()
src_hash = self._get_local_md5()
dst_hash = self._get_remote_md5()
if src_hash != dst_hash:
raise FileHashMismatchError(self.src, self.dst, src_hash, dst_hash)
def main(args):
print 'connecting to solo...'
solo = soloutils.connect_solo(await=True)
scp = SCPClient(solo.get_transport())
code = run(solo, scp)
if code == 0:
print 'pip is ready to use.'
scp.close()
solo.close()
sys.exit(code)
def file_upload() -> str:
for host in host_list :
if ipaddress.ip_address(host).is_multicast == False:
node.load_system_host_keys()
node.connect(hostname=host,
username=user,
password=passwd)
scp = SCPClient(node.get_transport())
scp.put('/home/klukonin/TEST/TEST','/tmp/TEST')
print('Файл записан на ноду ' + host)
scp.close()
sleep(2)
print('Все файлы записаны')
def flash(target, firmware_file, firmware_md5, args):
# Connect to controller...
if target == 'controller':
errprinter('connecting to Controller...')
client = soloutils.connect_controller(await=True)
else:
errprinter('connecting to Solo...')
client = soloutils.connect_solo(await=True)
# Prepare the update.
# older versions don't have sololink_config and ssh returns 127, so do it manually
code = soloutils.command_stream(client, 'sololink_config --update-prepare sololink')
if code != 0:
soloutils.command_stream(client, 'rm -rf /log/updates && mkdir -p /log/updates')
# Upload the files.
errprinter('uploading files...')
scp = SCPClient(client.get_transport())
scp.put(firmware_file, posixpath.join('/log/updates/', posixpath.basename(firmware_file)))
scp.put(firmware_md5, posixpath.join('/log/updates/', posixpath.basename(firmware_md5)))
scp.close()
if target == 'controller':
errprinter("starting update on the Controller...")
else:
errprinter("starting update on Solo...")
if args['--clean']:
errprinter('marking all user-local changes to be reset...')
code = soloutils.command_stream(client, 'sololink_config --update-apply sololink --reset')
else:
code = soloutils.command_stream(client, 'sololink_config --update-apply sololink')
# Fallback to earlier versions.
if code != 0:
if args['--clean']:
code = soloutils.command_stream(client, 'touch /log/updates/UPDATE && touch /log/updates/RESETSETTINGS && shutdown -r now')
else:
code = soloutils.command_stream(client, 'touch /log/updates/UPDATE && shutdown -r now')
if target == 'controller':
errprinter('the Controller will update once it reboots!')
else:
errprinter('Solo will update once it reboots!')
dt = datetime.today() + timedelta(minutes=4)
errprinter('please wait up to four minutes longer for the installation to complete (by {}).'.format(dt.strftime('%-I:%M')))
# Complete!
client.close()
return code
def scp_connect(server, username, password, col_log, col_type):
print ('Connecting to remote host ' + server)
ssh = SSHClient()
#ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(server, 22, username, password)
scp = SCPClient(ssh.get_transport())
except:
print ('Unable to connect, please check your credentials and hostname')
sys.exit()
# Copy files to localdir
if col_log == 'None' and col_type == 'vcd':
col_log = default_vcd_debug
col_log_info = default_vcd_info
# Create local directory to hold log files
try:
os.stat(tmp_dir)
print ('Local temp folder exists')
except:
os.mkdir(tmp_dir)
print ('Creating local temp folder')
#attempting to grab debug log
print ('Connected to remote host\n')
file_path = '/tmp/vmw_eps_calc/'
try:
print ('Attempting to copy DEBUG logs to your machine. This (~11M) can take a a bit depending on your connection....\n')
scp.get(col_log, file_path)
# Pass file location to file_list() after parsing it a bit to extract the actual file name
file_name_index = col_log.rfind('/')
file_name = col_log[file_name_index:]
local_file_path = tmp_dir + file_name
print ('File copied successfully')
file_list(local_file_path, col_type)
except:
print ('Error retrieving log file ' + col_log)
try:
print ('Now doing the info level logs...\n')
scp.get(col_log_info, file_path)
# Pass file location to file_list() after parsing it a bit to extract the actual file name
file_name_index = col_log_info.rfind('/')
file_name = col_log_info[file_name_index:]
local_file_path = tmp_dir + file_name
print ('File copied successfully')
file_list(local_file_path, col_type)
except:
print ('Error retrieving log file ' + col_log_info)
scp.close()
def SecureCopy(user, host, src, dest, # pylint: disable=too-many-arguments
key_filename=os.path.join(os.path.expanduser("~"), '.ssh', 'id_rsa')):
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
pkey = _load_key(key_filename)
ssh.connect(host, username=user, pkey=pkey)
scp = SCPClient(ssh.get_transport())
scp.get(src, dest)
scp.close()
def scp(self, local_path, remote_path):
"""Copy files from local_path to remote_path.
connect() method has to be called first!
"""
logger.trace('SCP {0} to {1}:{2}'.format(
local_path, self._hostname, remote_path))
# SCPCLient takes a paramiko transport as its only argument
scp = SCPClient(self._ssh.get_transport())
start = time()
scp.put(local_path, remote_path)
scp.close()
end = time()
logger.trace('SCP took {0} seconds'.format(end-start))
def send_file(self, src, dest, prog=False):
if self._ssh is None:
print('SSH not connected ...')
return
def progress(filename, size, sent):
sys.stdout.write("%s progress: %.2f%% \r" % (str(filename), float(sent)/float(size)*100) )
if prog:
scp = SCPClient(self._ssh.get_transport(), progress=progress)
else:
scp = SCPClient(self._ssh.get_transport())
print('Sending', src, ' to', self._host, '...')
scp.put(os.path.expanduser(src), dest)
scp.close()
def secure_copy(user, host, src, dest, key_filename=None, allow_agent=True):
keys = _load_keys(key_filename, allow_agent)
pkey = keys[0]
ssh = paramiko.SSHClient()
conf = paramiko.SSHConfig()
ssh_config_file = os.path.expanduser("~/.ssh/config")
conf.parse(open(ssh_config_file))
host_config = conf.lookup(host)
proxy = paramiko.ProxyCommand(host_config['proxycommand'])
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host, username=user, pkey=pkey, sock=proxy)
scp = SCPClient(ssh.get_transport())
scp.get(src, dest)
scp.close()
def copy_tmp_to_ssh(self, uri):
ssh = SSHClient()
ssh.load_system_host_keys()
print uri.username
if not uri.username:
ssh.connect(uri.hostname, allow_agent=True)
elif uri.username and not uri.password:
ssh.connect(uri.hostname, username=uri.username, allow_agent=True)
elif uri.username and uri.password:
ssh.connect(uri.hostname, username=uri.username, password=uri.password, allow_agent=True)
# SCPCLient takes a paramiko transport as its only argument
scp = SCPClient(ssh.get_transport())
scp.put(self.tmp_file.name, uri.path)
scp.close()
def broadcast(file, hosts):
client = SSHClient()
util.log_to_file("this.log")
client.load_system_host_keys()
client.set_missing_host_key_policy(AutoAddPolicy())
for ip in hosts:
try:
print("Logging into: " + ip)
client.connect(hostname=ip, username="******", password="******")
scp = SCPClient(client.get_transport())
scp.put(file)
scp.close()
client.close()
except:
print(ip + " is being stubborn")
def __init__(self, race_num):
############################################
# building the path to the directory #
# User input race_unm #
# Absolute path is: '/mnt/bugs/RACE-XXXX' #
############################################
self.racePath = HOST_PATH + str(race_num)
##########################################
# Creating secure connection to IBM host #
# with SSH protocol #
##########################################
print "Creating connection......."
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
ssh.connect(HOST, username=USER_NAME, password=PASS)
print "Connected successfully to %s!!!" % HOST
except paramiko.AuthenticationException:
print "Authentication failed when connecting to %s !!!" % HOST
print "=" * 30
print "path is: %s" % self.racePath
print "=" * 30
##################################################
# Copy the required directory from Remote host #
# to local host with SCP (secured Copy) protocol #
##################################################
try:
scp = SCPClient(ssh.get_transport())
print "Copying Directory..."
except Exception as e:
print "Other exception: " + e.message + str(e.__class__) #+ "SCP=" + str( scp)
scp.get(self.racePath, '/home/support/Desktop', True, False)
###########################################
# Close SCP and SSH concetion when done #
###########################################
scp.close()
print "SCP Connection closed %s" % scp
ssh.close()
print "SSH Connection closed %s" % ssh
def push_main(args):
if not os.path.exists(SCRIPT_FILENAME):
print 'ERROR: Please run "solo script pack" first to bundle your archive.'
return 1
print 'connecting to solo...'
solo = soloutils.connect_solo(await=True)
scp = SCPClient(solo.get_transport())
# Requires pip
print 'checking pip...'
if soloutils.install_pip.run(solo, scp) != 0:
print 'failed installing pip.'
sys.exit(1)
# TODO check args['<arg>'] for --force
push(solo, scp, '--force' in args['<arg>'])
scp.close()
solo.close()
def transfer_file(module, dest):
file_size = os.path.getsize(module.params['local_file'])
if not enough_space(module):
module.fail_json(msg='Could not transfer file. Not enough space on device.')
provider = module.params['provider']
hostname = module.params.get('host') or provider.get('host')
username = module.params.get('username') or provider.get('username')
password = module.params.get('password') or provider.get('password')
port = module.params.get('connect_ssh_port')
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
hostname=hostname,
username=username,
password=password,
port=port)
full_remote_path = '{0}{1}'.format(module.params['file_system'], dest)
scp = SCPClient(ssh.get_transport())
try:
scp.put(module.params['local_file'], full_remote_path)
except:
time.sleep(10)
temp_size = verify_remote_file_exists(
module, dest, file_system=module.params['file_system'])
if int(temp_size) == int(file_size):
pass
else:
module.fail_json(msg='Could not transfer file. There was an error '
'during transfer. Please make sure remote '
'permissions are set.', temp_size=temp_size,
file_size=file_size)
scp.close()
ssh.close()
return True
def secure_copy(user, host, src, dest, key_filename=None, allow_agent=True):
keys = _load_keys(key_filename, allow_agent)
pkey = keys[0]
ssh = paramiko.SSHClient()
proxy = None
ssh_config_file = os.path.expanduser("~/.ssh/config")
if os.path.isfile(ssh_config_file):
conf = paramiko.SSHConfig()
with open(ssh_config_file) as f:
conf.parse(f)
host_config = conf.lookup(host)
if 'proxycommand' in host_config:
proxy = paramiko.ProxyCommand(host_config['proxycommand'])
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(host, username=user, pkey=pkey, sock=proxy)
scp = SCPClient(ssh.get_transport())
try:
scp.get(src, dest)
scp.close()
except paramiko.ssh_exception.SSHException:
subprocess.call(["scp", "{}@{}:{}".format(user, host, src), dest])
def download_and_restore_syncserver(db_name):
s = 'clean.sql'
try:
os.unlink(s)
except OSError as e:
pass
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(URL_BACKUPS, username=LOGIN_BACKUPS, password=PASSWORD_BACKUPS)
path = "/home/unifield_backups/syncsdv/dump_msfsync-slave/"
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command('ls -1rth ' + path + ' | tail -2')
filename = ssh_stdout.read()
filename = filename.strip().split()[0]
new_path = os.path.join(path, filename)
scp = SCPClient(ssh.get_transport())
scp.get(new_path)
scp.close()
filename_sql = '.'.join(filename.split('.')[:-1])
os.system("unlzma %s" % filename)
run_script("postgres", 'DROP DATABASE IF EXISTS "%s";' % db_name)
run_script("postgres", 'CREATE DATABASE "%s";' % db_name)
os.environ['PGPASSWORD'] = POSTGRESQL_PASSWORD
os.system('psql -p %d -h %s -U %s %s < %s' % (POSTGRESQL_PORT, POSTGRESQL_SERVER, POSTGRESQL_USERNAME, db_name, filename_sql))
try:
os.unlink(filename_sql)
os.unlink(s)
except OSError as e:
pass
def transfer_file(self, dest):
"""Begin to transfer file by scp"""
if not self.local_file_exists():
self.module.fail_json(
msg='Could not transfer file. Local file doesn\'t exist.')
if not self.enough_space():
self.module.fail_json(
msg='Could not transfer file. Not enough space on device.')
hostname = self.module.params['provider']['host']
username = self.module.params['provider']['username']
password = self.module.params['provider']['password']
port = self.module.params['provider']['port']
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=hostname, username=username, password=password, port=port)
full_remote_path = '{}{}'.format(self.file_system, dest)
scp = SCPClient(ssh.get_transport())
try:
scp.put(self.local_file, full_remote_path)
except:
time.sleep(10)
file_exists, temp_size = self.remote_file_exists(
dest, self.file_system)
file_size = os.path.getsize(self.local_file)
if file_exists and int(temp_size) == int(file_size):
pass
else:
scp.close()
self.module.fail_json(msg='Could not transfer file. There was an error '
'during transfer. Please make sure the format of '
'input parameters is right.')
scp.close()
return True
def flash_px4(firmware_file):
errprinter('connecting to Solo...')
client = soloutils.connect_solo(await=True)
soloutils.command_stream(client, 'rm -rf /firmware/loaded')
# Upload the files.
errprinter('uploading files...')
scp = SCPClient(client.get_transport())
scp.put(firmware_file, '/firmware')
scp.close()
# shutdown solo for firmware reflash
#code = soloutils.command_stream(client, 'shutdown -r now')
code = soloutils.command_stream(client, 'loadPixhawk.py')
#errprinter('Solo will update once it reboots!')
errprinter('Pixhawk has been updated to new firmware')
#dt = datetime.today() + timedelta(minutes=4)
#errprinter('please wait up to four minutes longer for the installation to complete (by {}).'.format(dt.strftime('%-I:%M')))
# Complete!
client.close()
sys.exit(0)
def transfer_file(self, dest):
"""Begin to transfer file by scp"""
if not self.local_file_exists():
self.module.fail_json(
msg='Could not transfer file. Local file doesn\'t exist.')
if not self.enough_space():
self.module.fail_json(
msg='Could not transfer file. Not enough space on device.')
hostname = self.module.params['provider']['host']
username = self.module.params['provider']['username']
password = self.module.params['provider']['password']
port = self.module.params['provider']['port']
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=hostname, username=username, password=password, port=port)
full_remote_path = '{0}{1}'.format(self.file_system, dest)
scp = SCPClient(ssh.get_transport())
try:
scp.put(self.local_file, full_remote_path)
except Exception:
time.sleep(10)
file_exists, temp_size = self.remote_file_exists(
dest, self.file_system)
file_size = os.path.getsize(self.local_file)
if file_exists and int(temp_size) == int(file_size):
pass
else:
scp.close()
self.module.fail_json(msg='Could not transfer file. There was an error '
'during transfer. Please make sure the format of '
'input parameters is right.')
scp.close()
return True
def upload_picture(self, filepath):
"""
Upload the file on the given path to the server
via scp
Uses class configuration to determine the server and
user to connect to. Relies on user public key installed
on the target server for authentication
"""
from paramiko import SSHClient
from scp import SCPClient
# skip if improperly configured
if not self.upload_host or not self.upload_dest_filename:
return
try:
# configure ssh transport
ssh = SSHClient()
ssh.load_system_host_keys()
connect_kwargs = {}
if self.upload_host_username:
connect_kwargs['username'] = self.upload_host_username
ssh.connect(self.upload_host, **connect_kwargs)
# Configure scp and perform upload
scp = SCPClient(ssh.get_transport())
scp.put(filepath, self.upload_dest_filename)
# clean up
scp.close()
ssh.close()
# log success
logger.info("Snapshot uploaded")
except Exception:
logger.exception("Error uploading snapshot")
def transfer_file(module, dest):
file_size = os.path.getsize(module.params['local_file'])
if not local_file_exists(module):
module.fail_json(
msg='Could not transfer file. Local file doesn\'t exist.')
if not enough_space(module):
module.fail_json(
msg='Could not transfer file. Not enough space on device.')
hostname = module.params['host']
username = module.params['username']
password = module.params['password']
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=hostname, username=username, password=password)
full_remote_path = '{}{}'.format(module.params['file_system'], dest)
scp = SCPClient(ssh.get_transport())
try:
scp.put(module.params['local_file'], full_remote_path)
except:
time.sleep(10)
temp_size = verify_remote_file_exists(
module, dest, file_system=module.params['file_system'])
if int(temp_size) == int(file_size):
pass
else:
module.fail_json(msg='Could not transfer file. There was an error '
'during transfer. Please make sure remote '
'permissions are set.',
temp_size=temp_size,
file_size=file_size)
scp.close()
return True
def install_monet():
print "in scp client"
print "copying Monet installation file"
Username = "******"
Password = "******"
Server = "192.168.56.20"
ssh = SSHClient()
ssh.set_missing_host_key_policy(client.AutoAddPolicy())
ssh.connect(Server, username=Username, password=Password)
# SCPCLient takes a paramiko transport as its only argument
scp = SCPClient(ssh.get_transport())
scp.put('6.125/Monet.6.125.solaris11.amd64.run', '/tmp/Monet.6.125.solaris11.amd64.run')
#scp.get('/var/tmp/tdir','c:/temp',recursive=True)
print "scp done"
scp.close()
print "executing Monet Install file"
command="(chmod +x /tmp/Monet*;/tmp/Monet.6.125.solaris11.amd64.run)"
output,error=connection.run_Cmd_stderr(command)
#command="/tmp/Monet.6.125.solaris11.amd64.run)"
#output,error=connection.run_Cmd_stderr(command)
#if output.strip()=='0' and ' cannot open' in error:
## print 'file sd.conf on existing system does not exist'
#return False
#else:
# return output.strip()
#install_monet()
def from_HPC2Local(remote,remote_path,file,**kwargs):
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
uname, host = remote.split('@')
try:
os.system("stty -echo")
ssh.connect(host,username=uname)
os.system("stty echo")
termios.tcflush(sys.stdin, termios.TCIOFLUSH)
except paramiko.PasswordRequiredException as e:
import getpass
password = getpass.getpass("%s, enter password/phrase: "%e)
try:
ssh.connect(host,username=uname,password=password)
except paramiko.PasswordRequiredException as e:
passphrase = getpass.getpass("%s, enter password/phrase: "%e)
ssh.connect(host,username=uname,password=password,passphrase=passphrase)
scp = SCPClient(ssh.get_transport(),**kwargs)
file_path = os.path.join(remote_path, file)
scp.get(file_path)
scp.close()
def scp(ip, port, username, password, mode, remotePath, localPath):
ssh = SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(ip, int(port), username, password, look_for_keys=False)
def progress4(filename, size, sent, peername):
sys.stdout.write("(%s:%s) %s\'s progress: %.2f%% \r" %
(peername[0], peername[1], filename,
float(sent) / float(size) * 100))
scp = SCPClient(ssh.get_transport(), progress4=progress4)
try:
if mode == "upload":
scp.put(localPath, remotePath, recursive=True)
elif mode == "download":
scp.get(localPath, remotePath, recursive=True)
except Exception as e:
print(e)
scp.close()
ssh.close()
def copy_file(local_file_path, dest_path, host, port, user, password, mode = 'PUT'):
transport = None
if not path.exists(local_file_path):
if mode is 'GET':
mkdir(local_file_path)
else:
print "Local file path doesnt exist %s" %(local_file_path)
paramiko.util.log_to_file(ser_info['paramiko_log'])
try:
transport = paramiko.Transport((host, port))
transport.connect(username=user, password=password)
transport.get_exception()
scp = SCPClient(transport)
except:
print "FAILED: Error while connecting to host %s:%s and copy file" %(host, port)
return 0
if mode is 'PUT':
scp.put(local_file_path, dest_path)
else:
scp.get(dest_path, recursive=True)
scp.close()
return 1
def pre_test(self):
'''Settings before test '''
if self.host:
# test on specified host.
if self.script:
# cp or scp script to test directory.
ssh_client = ssh(self.user, self.host, self.password)
scp_client = SCPClient(ssh_client.get_transport())
scp_client.put(self.script, self.tst_dir, recursive=True)
ssh_client.close()
scp_client.close()
else:
if self.script:
pass
print('####################### TEST INFO #########################')
print('# name: %-49s #' % self.name)
print('# doc: %-50s #' % self.__doc__)
print('# start: %-48s #' % self.tst_time)
print('# user: %-49s #' % self.user)
print('# host: %-49s #' % self.host)
print('# script: %-47s #' % self.script)
print('# iteration: %-44s #' % self.iteration)
print('###########################################################')
print('')
def copy_script(self, ssh_client, script_string, script_name):
source_filename = self.tmp_dir + os.sep + script_name
target_filename = os.path.basename(self.tmp_dir) + '/' + script_name
with open(source_filename, 'w') as f:
f.write(script_string)
scp_client = SCPClient(ssh_client.get_transport())
try:
scp_client.put(source_filename, target_filename)
except Exception as e:
print(e)
scp_client.close()
# try:
# transport = paramiko.Transport(ssh_client.get_transport())
# transport.connect(username=self.username, password=self.password)
# sftp = paramiko.SFTPClient.from_transport(transport)
# sftp.put(script_name, script_name)
# except Exception as e:
# print(e)
stdin, stdout, stderr = ssh_client.exec_command("chmod 755 " +
target_filename)
check_exec_result(stdout, stderr)
stdin, stdout, stderr = ssh_client.exec_command("dos2unix " +
target_filename)
check_exec_result(stdout, stderr)
def upload_script_and_execute(self, ssh, parameter):
Platform = self.get_platform(ssh)
if parameter.strip().startswith("script:"):
parameter = parameter[7:]
try:
shell, filepath = parameter.strip().split(":")
except ValueError:
shell = None
filepath = parameter.strip()
STREAM.info(" -> Executing script: %s" % filepath)
if Platform == "win-like":
STREAM.debug(" -> Remote system probably is windows type")
temppath = os.path.join("C:\Windows\Temp", os.path.basename(filepath))
default_shell = r"C:\Windows\System32\cmd.exe /c start"
else:
STREAM.debug(" -> Remote system probably is unix type")
temppath = os.path.join("/tmp", os.path.basename(filepath))
default_shell = "bash"
if shell is None:
STREAM.debug(" -> Shell is not specified, using default: %s" % default_shell)
shell = default_shell
scp = SCPClient(ssh.get_transport())
scp.put(filepath, temppath)
scp.close()
ssh_stdin, ssh_stdout, ssh_stderr = ssh.exec_command("%s %s" % (shell, temppath))
stdout = ssh_stdout.read()
stderr = ssh_stderr.read()
STREAM.debug(self.get_decoded(stdout))
if len(stderr) > 0:
STREAM.debug(self.get_decoded(stderr))
exit_code = ssh_stdout.channel.recv_exit_status()
STREAM.debug(" -> Script exitcode: %s" % exit_code)
if exit_code == 0:
STREAM.success(" -> Script executed successfully")
else:
raise Exception("Executed script exit status not 0")
def sshupload(file, remote_directory):
global default_vps
"""Upload a single file to a remote directory"""
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.WarningPolicy())
#f = open(sshkey, 'r')
#s = f.read()
#keyfile = StringIO(s)
#pkey = paramiko.RSAKey.from_private_key(keyfile)
ssh.connect('bitclouds.link',
username='******',
port=default_vps['ssh_port'],
key_filename=sshkey,
look_for_keys=False)
scp = SCPClient(ssh.get_transport(), progress=progress)
try:
scp.put(file, recursive=True, remote_path=remote_directory)
except SCPException:
print('scp error')
finally:
scp.close()
ssh.close()
def copy_bundle(vxrail_ip):
vxrm_location = '/tmp'
global bundle_filename
try:
# Define progress callback that prints the current percentage completed for the file
def progress(filename, size, sent):
sys.stdout.buffer.write(
b"%s\'s progress: %.2f%% \r" %
(filename, float(sent) / float(size) * 100))
bundle = input(
'Type the location of the Upgrade Bundle on your local machine')
bundle_filename = os.path.basename(bundle)
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.load_system_host_keys()
ssh.connect(vxrail_ip,
username='******',
password='******')
# SCPCLient takes a paramiko transport as an argument
scp = SCPClient(ssh.get_transport(), progress=progress)
# Uploading
scp.put(bundle, recursive=False, remote_path=vxrm_location)
stdin, stdout, stderr = ssh.exec_command('chmod 777 ' + vxrm_location +
'/' + bundle_filename)
scp.close()
except Exception as err:
print('copy_bundle(): ', err)
return None
def main():
env.check()
parser = argparse.ArgumentParser()
parser.add_argument("cluster_name")
subparsers = parser.add_subparsers(dest='op')
parser_scp = subparsers.add_parser('scp')
parser_scp.add_argument('key_pair_path')
parser_scp.add_argument('from_path')
parser_scp.add_argument('to_path')
parser_launch = subparsers.add_parser('launch')
parser_launch.add_argument('cluster_conf_path')
parser_start = subparsers.add_parser('start')
parser_stop = subparsers.add_parser('stop')
parser_cleanup = subparsers.add_parser('cleanup')
parser_status = subparsers.add_parser('status')
parser_dns = subparsers.add_parser('dns')
parser_ip = subparsers.add_parser('ip')
subparsers.add_parser('test')
subparsers.add_parser('terminate')
args = parser.parse_args()
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
region = os.getenv("AWS_DEFAULT_REGION")
cluster = Cluster(args.cluster_name)
if args.op == 'launch':
try:
conf = read_conf(args.cluster_name, args.cluster_conf_path)
ClusterLauncher().launch(conf)
except botocore.exceptions.WaiterError:
logging.error("--x Failed to launch instances, Please check your AWS console, some machines may be already running!")
cluster.terminate()
cluster.cleanup()
elif args.op == "status":
print(cluster.status)
elif args.op == "test":
groups = list(cluster.instances)[0].security_groups
for g in groups:
try:
aws.resource('ec2').SecurityGroup(g['GroupId']).delete()
except botocore.exceptions.ClientError as e:
print(g['GroupName'])
elif args.op == "dns":
print([i.public_dns_name for i in cluster.instances])
elif args.op == "ip":
print([i.public_ip_address for i in cluster.instances])
elif args.op == "stop":
cluster.stop()
elif args.op == "start":
cluster.start()
elif args.op == 'scp':
dns_name = list(cluster.instances)[0].public_dns_name
key = paramiko.RSAKey.from_private_key_file(args.key_pair_path)
ssh = SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname = dns_name, username = '******', pkey = key)
scp = SCPClient(ssh.get_transport())
scp.put(args.from_path, args.to_path)
scp.close()
elif args.op == "terminate":
utils.confirm("You are about to terminate the whole cluster.")
cluster.terminate()
elif args.op == "cleanup":
utils.confirm("You are about to terminate and remove the whole cluster.")
cluster.cleanup()
def enterkdfumode(kloader, kloader10, ibss):
ip = input(
"Please enter your devices IP address (Find it in WiFi settings):\n")
try:
socket.inet_aton(ip)
except:
print("ERROR: Invalid IP address...")
a = ip
while not a.startswith("b'"):
ip = input("Please enter a valid IP Address...")
try:
socket.inet_aton(ip)
except:
print("")
else:
print("")
if ip.count(".") != 3:
print("ERROR: Invalid IP address...")
b = ip.count(".")
while b != 3:
ip = input("Please enter a valid IP Address...")
b = ip.count(".")
print(b)
else:
print("")
devicepassword = getpass.getpass(
"Please enter the root password to your device (Default is 'alpine'):\n"
)
print("Connecting to device via SSH...")
try:
client = paramiko.SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(paramiko.WarningPolicy)
client.connect(hostname=ip, password=devicepassword, username="******")
command = "uname -r"
stdin, stdout, stderr = client.exec_command(command)
result = stdout.read(),
scp = SCPClient(client.get_transport())
finalresult = str(result)
with silence_stdout():
print(finalresult)
scp.put(ibss, '/')
if finalresult.startswith("(b'16"):
print("Device is running iOS 10.x, using HGSP4 kloader...")
scp.put(kloader10, '/')
command2 = "/kloader10 /ibss > /dev/null 2>&1 &"
else:
print(
'Device is not running iOS 10.x, using normal TFP0 kloader...')
scp.put(kloader, '/')
command2 = "/kloader /ibss > /dev/null 2>&1 &"
stdin, stdout, stderr = client.exec_command(command2)
result2 = stdout.read(),
with silence_stdout():
print(result2)
scp.close()
client.close()
print(
"Please press the home button on your device or unplug and replug it back in.\nWaiting 10 seconds for you to do this."
)
time.sleep(13)
except:
print(
"ERROR: SSH/SCP failed, make sure that you are jailbroken with OpenSSH installed, try rebooting the device\nExiting..."
)
exit(222)
class SSH:
def __init__(self, servidor, usuario):
self.usuario = usuario
self.servidor = servidor
self.ssh = paramiko.SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.scp = None
self.senha_servidor = None
self.arquivo_chave = None
def senha(self, senha_servidor):
self.senha_servidor = senha_servidor
def chave_acesso(self, arquivo_chave):
self.arquivo_chave = arquivo_chave
def abrir_conexao(self):
try:
if self.senha_servidor is not None:
self.ssh.connect(self.servidor,
username=self.usuario,
password=self.senha_servidor)
else:
self.ssh.load_system_host_keys(self.arquivo_chave)
self.ssh.connect(self.servidor, username=self.usuario)
except:
Log.imprime("FALHA\n" + format_exc(),
"FALHA -- ERRO NA CONEXAO SSH",
classe=SSH)
exit(1)
try:
self.scp = SCPClient(self.ssh.get_transport())
except:
Log.imprime("ERRO NA CONEXAO SSH\n" + format_exc(),
"ERRO NA CONEXAO SCP",
classe=SSH)
exit(1)
Log.imprime(
"CONEXAO ABERTA COM SUCESSO.\nUSUARIO " + self.usuario +
"\nSERVIDOR " + self.servidor, "CONEXAO SSH ABERTA COM SUCESSO")
def executar_comando(self, comando, timeout=None, bufsize=-1):
if comando:
Log.imprime("COMANDO A SER EXECUTADO VIA SSH: " + comando,
"EXECUTANDO COMANDO VIA SSH")
chan = self.ssh.get_transport().open_session(timeout=timeout)
chan.get_pty()
chan.settimeout(timeout)
chan.exec_command(comando)
saida = ""
output_instantaneo = True
while not chan.exit_status_ready():
if chan.recv_ready():
output_instantaneo = False
retorno = chan.recv(1024)
saida += retorno.decode("utf-8")
print(retorno.decode("utf-8"), end="")
# stdin = chan.makefile('wb', bufsize)
if output_instantaneo:
stdout = chan.makefile('r', bufsize)
for linha in stdout:
saida += str(linha)
if saida:
print(saida, end="")
stderr = chan.makefile_stderr('r', bufsize)
saida_erro = ""
for linha in stderr:
saida_erro += str(linha)
if saida_erro:
print("ERRO:", saida_erro)
retorno = chan.recv_exit_status()
if retorno != 0:
print("ExitCode:", retorno)
self.fechar_conexao()
exit(retorno)
return saida
def fechar_conexao(self):
self.ssh.close()
self.scp.close()
Log.imprime("CONEXAO SSH ENCERRADA", "CONEXAO SSH ENCERRADA")
def obter_arquivo(self, path_origem_arquivo, path_destino_arquivo=''):
Log.imprime(
"OBTENDO ARQUIVO " + path_origem_arquivo + " QUE SERA SALVO EM " +
path_destino_arquivo, "OBTENDO ARQUIVO " + path_origem_arquivo)
try:
self.scp.get(path_origem_arquivo, path_destino_arquivo, False,
True)
Log.imprime("ARQUIVO " + path_origem_arquivo + " SALVO EM " +
path_destino_arquivo)
except:
Log.imprime("OCORREU UM ERRO AO TENTAR OBTER O ARQUIVO " +
path_origem_arquivo + format_exc(),
"OCORREU UM ERRO AO TENTAR OBTER O ARQUIVO " +
path_origem_arquivo,
classe=SSH)
exit(1)
def enviar_arquivo(self, path_origem_arquivo, path_destino_arquivo):
Log.imprime(
"ENVIANDO ARQUIVO " + path_origem_arquivo + " PARA " +
path_destino_arquivo, "ENVIANDO ARQUIVO " + path_origem_arquivo)
try:
self.scp.put(path_origem_arquivo, path_destino_arquivo)
Log.imprime("ARQUIVO " + path_origem_arquivo + " ENVIADO PARA " +
path_destino_arquivo)
except:
Log.imprime("OCORREU UM ERRO NO ENVIO PARA " +
path_destino_arquivo + format_exc(),
"OCORREU UM ERRO NO ENVIO PARA " +
path_destino_arquivo,
classe=SSH)
exit(1)
def test_apply_template_playbook(self):
"""
Tests application of an Ansible playbook that applies a template to a
file:
1. Have a ~/.ansible.cfg (or alternate means) to set
host_key_checking = False
2. Set the following environment variable in your executing shell:
ANSIBLE_HOST_KEY_CHECKING=False
Should this not be performed, the creation of the host ssh key will
cause your ansible calls to fail.
"""
self.inst_creator.create(block=True)
priv_ip = self.inst_creator.get_port_ip(self.port_1_name)
self.assertTrue(check_dhcp_lease(self.inst_creator, priv_ip))
# Apply Security Group
self.inst_creator.add_security_group(
self.sec_grp_creator.get_security_group())
# Block until VM's ssh port has been opened
self.assertTrue(self.inst_creator.vm_ssh_active(block=True))
# Block until cloud-init has completed
self.assertTrue(self.inst_creator.cloud_init_complete(block=True))
# Apply Security Group
self.inst_creator.add_security_group(
self.sec_grp_creator.get_security_group())
# Need to use the first floating IP as subsequent ones are currently
# broken with Apex CO
ip = self.inst_creator.get_floating_ip().ip
user = self.inst_creator.get_image_user()
priv_key = self.inst_creator.keypair_settings.private_filepath
relative_pb_path = pkg_resources.resource_filename(
'snaps.provisioning.tests.playbooks', 'template_playbook.yml')
self.inst_creator.apply_ansible_playbook(relative_pb_path,
variables={'name': 'Foo'})
ssh = ansible_utils.ssh_client(
ip,
user,
private_key_filepath=priv_key,
proxy_settings=self.os_creds.proxy_settings)
self.assertIsNotNone(ssh)
scp = None
try:
scp = SCPClient(ssh.get_transport())
scp.get('/tmp/hello.txt', self.test_file_local_path)
finally:
if scp:
scp.close()
ssh.close()
self.assertTrue(os.path.isfile(self.test_file_local_path))
test_file = None
try:
with open(self.test_file_local_path) as test_file:
file_contents = test_file.readline()
self.assertEqual('Hello Foo!', file_contents)
finally:
if test_file:
test_file.close()
class SCPLibrary(object):
"""Robot Framework test library for Secure Copy (SCP).
This library can be used to copy files to and from a remote machine using Secure Copy (SCP). It uses the
Paramiko SSH Python library (just like robotframework-sshlibrary) and an SCP wrapper ('scp' by James Bardin).
The library does not currently support Jython or IronPython at this time.
== Table of contents ==
- `Connection`
- `File transfer`
= Connection =
Before files can be transferred a connection to remote machine must first be made. A connection can be made with the
`Open Connection` keyword. Both normal username/password authentication and asymmetric key-pair authentication may
be used.
Connections should be closed using the `Close Connection` when they are no longer in use.
= File transfer =
Files and directories can be uploaded to the remote machine using the `Put File` or `Put Directory` keywords or
downloaded to the local machine using the `Get File` keyword.
A connection must be made using the `Open Connection` keyword before file transfers may be made.
"""
ROBOT_LIBRARY_SCOPE = 'GLOBAL'
ROBOT_LIBRARY_VERSION = __version__
def __init__(self):
self.ssh = None
self.scp_client = None
def open_connection(self, hostname, port='22', username=None, password=None, key_filename=None):
"""Opens a new SCP connection to the given host.
The default port used is `22`:
| Open Connection | host.tylercrumpton.com |
A different port may be optionally given by using the `port` argument:
| Open Connection | host.tylercrumpton.com | port=4242 |
Authentication may be done using a username and password:
| Open Connection | host.tylercrumpton.com | username=tyler | password=iamateapot |
Or by using a private keyfile:
| Open Connection | host.tylercrumpton.com | username=tyler | key_filename=myprivatekey |
"""
try:
port = int(port)
except:
raise ValueError('Port must be a valid number.')
self.ssh = SSHClient()
self.ssh.set_missing_host_key_policy(AutoAddPolicy())
self.ssh.connect(hostname, port=port, username=username, password=password, key_filename=key_filename)
self.scp_client = SCPClient(self.ssh.get_transport())
def close_connection(self):
"""Closes the SCP connection.
Example:
| Open Connection | host.tylercrumpton.com | username=tyler | password=iamateapot |
| Get File | tea.txt | /mytea/newtea.txt |
| Close Connection |
"""
if self.scp_client is not None:
self.scp_client.close()
self.scp_client = None
if self.ssh is not None:
self.ssh.close()
self.ssh = None
def put_file(self, local_filepath, remote_filepath):
"""Uploads a file to the remote machine from the local machine.
Note: A connection to the remote machine must be made first using the `Open Connection` keyword.
Examples:
| Put File | mytea.txt | /home/tyler/tea.txt |
| Put File | mytea.txt | /home/tyler/ |
"""
if self.scp_client is None:
raise SCPNotConnectedError("An SCPLibrary connection must be created first using the 'Open Connection' keyword.")
self.scp_client.put(local_filepath, remote_filepath, recursive=False)
def put_directory(self, local_directory, remote_filepath):
"""Uploads a directory to the remote machine from the local machine.
Note: A connection to the remote machine must be made first using the `Open Connection` keyword.
Examples:
| Put File | mytea_dir | /home/tyler/newtea_dir |
| Put File | mytea.txt | /home/tyler/ |
"""
if self.scp_client is None:
raise SCPNotConnectedError("An SCPLibrary connection must be created first using the 'Open Connection' keyword.")
self.scp_client.put(local_directory, remote_filepath, recursive=True)
def get_file(self, remote_filepath, local_filepath, recursive=False):
"""Downloads a file from the remote machine to the local machine.
`remote_filepath` determines the path to retrieve from remote host. Shell wildcards and environment variables
on the remote machine may be used.
Setting `recursive` to True will transfer files and directories recursively.
Note: A connection to the remote machine must be made first using the `Open Connection` keyword.
Example:
| Get File | /home/tyler/tea.txt | sametea.txt | |
| Get File | /home/tyler/*.txt | myteas/ | |
| Get File | /home/tyler/ | mytylerdir/ | recursive=True |
"""
if self.scp_client is None:
raise SCPNotConnectedError("An SCPLibrary connection must be created first using the 'Open Connection' keyword.")
self.scp_client.get(remote_filepath, local_filepath, recursive=recursive)
def flash(target, firmware_file, firmware_md5, args):
# Connect to controller...
if target == 'controller':
errprinter('connecting to Controller...')
client = soloutils.connect_controller(await=True)
else:
errprinter('connecting to Solo...')
client = soloutils.connect_solo(await=True)
# Prepare the update.
# older versions don't have sololink_config and ssh returns 127, so do it manually
code = soloutils.command_stream(
client, 'sololink_config --update-prepare sololink')
if code != 0:
soloutils.command_stream(
client, 'rm -rf /log/updates && mkdir -p /log/updates')
# Upload the files.
errprinter('uploading files...')
scp = SCPClient(client.get_transport())
scp.put(firmware_file,
posixpath.join('/log/updates/', posixpath.basename(firmware_file)))
scp.put(firmware_md5,
posixpath.join('/log/updates/', posixpath.basename(firmware_md5)))
scp.close()
if target == 'controller':
errprinter("starting update on the Controller...")
else:
errprinter("starting update on Solo...")
if args['--clean']:
errprinter('marking all user-local changes to be reset...')
code = soloutils.command_stream(
client, 'sololink_config --update-apply sololink --reset')
else:
code = soloutils.command_stream(
client, 'sololink_config --update-apply sololink')
# Fallback to earlier versions.
if code != 0:
if args['--clean']:
code = soloutils.command_stream(
client,
'touch /log/updates/UPDATE && touch /log/updates/RESETSETTINGS && shutdown -r now'
)
else:
code = soloutils.command_stream(
client, 'touch /log/updates/UPDATE && shutdown -r now')
if target == 'controller':
errprinter('the Controller will update once it reboots!')
else:
errprinter('Solo will update once it reboots!')
dt = datetime.today() + timedelta(minutes=4)
errprinter(
'please wait up to four minutes longer for the installation to complete (by {}).'
.format(dt.strftime('%-I:%M')))
# Complete!
client.close()
return code
class pypitomirror(object):
def extractpypi(
self,
packagelist=[],
sourceIndex='/mnt/DATA/projects/bandersnatch/mirror/web/simple/',
sourcePackages='/mnt/DATA/projects/bandersnatch/mirror/web/packages/',
dest='/mnt/DATA/temptemp/pypi6',
ssh="jax/Dh123/jax79sg.hopto.org/10022"):
if (ssh is not None):
user = ssh.split("/")[0]
password = ssh.split("/")[1]
host = ssh.split("/")[2]
port = ssh.split("/")[3]
self.sshclient = paramiko.SSHClient()
self.sshclient.load_system_host_keys()
self.sshclient.connect(hostname=host,
port=port,
username=user,
password=password)
self.scp = SCPClient(self.sshclient.get_transport())
with tqdm.tqdm(total=len(packagelist)) as pbar:
for package in packagelist:
listOfPackages = self.processIndexHtmls(package,
sourceIndex,
dest,
ssh=ssh)
self.processPackages(listOfPackages,
sourcePackages,
dest,
ssh=ssh)
pbar.update()
self.scp.close()
self.sshclient.close()
def processIndexHtmls(self, package, sourceIndex, dest, ssh):
#Copy it to dest
indexpath = sourceIndex + package
indexhtml = sourceIndex + package + '/index.html'
if (ssh is not None):
self.scp.put(indexpath,
recursive=True,
remote_path=dest + '/simple/' + package)
else:
try:
result = shutil.copytree(indexpath,
dest + '/simple/' + package)
# print("Copy completed in {}".format(result))
except FileExistsError:
print("File {} exists, skip!".format(dest + '/simple/' +
package))
#Get the index.html
from bs4 import BeautifulSoup
htmlfile = open(indexhtml, 'r')
soup = BeautifulSoup(htmlfile, 'html.parser')
#Get all the packages links
packagesLinks = []
for a in soup.find_all('a', href=True):
# print("Found the URL:", a['href'].split("#")[0].split("packages/")[1])
packagesLinks.append(a['href'].split("#")[0].split("packages/")[1])
return packagesLinks
def processPackages(self, listOfPackages, sourcePackages, dest, ssh=None):
#Copy the entire path of packges into dest
for package in listOfPackages:
subDir = package.split("/")
subDir = subDir[:-1]
currentDir = dest + '/packages'
if (ssh is not None):
self.sshclient.exec_command("mkdir -p " + currentDir)
for dir in subDir:
currentDir = currentDir + "/" + dir
self.sshclient.exec_command("mkdir -p " + currentDir)
self.scp.put(sourcePackages + package,
dest + '/packages/' + package)
else:
try:
os.mkdir(currentDir)
except FileExistsError:
pass
for dir in subDir:
currentDir = currentDir + "/" + dir
try:
os.mkdir(currentDir)
except FileExistsError:
pass
result = shutil.copy(sourcePackages + package,
dest + '/packages/' + package)
print("Copy completed in {}".format(result))
pass
class RemoteClient:
"""Client to interact with a remote host via SSH & SCP."""
def __init__(
self,
host=os.environ["REMOTE_HOST"],
user=os.environ["REMOTE_USER"],
password=os.environ["REMOTE_PASS"],
ssh_key_filepath=None,
remote_path=os.environ["KOR_PATH"],
):
self.host = host
self.user = user
self.ssh_key_filepath = ssh_key_filepath
self.remote_path = remote_path
self.client = self.connect(host, user, password)
print("SSH Connected")
self.scp = SCPClient(self.client.get_transport())
print("SCP Connected")
def __del__(self):
if self.client:
print("Disconnecting...")
self.disconnect()
def disconnect(self):
"""Close ssh connection."""
if self.client:
self.client.close()
if self.scp:
self.scp.close()
def connect(self, host, user, password, timeout=10):
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(AutoAddPolicy())
client.connect(host, username=user, password=password, timeout=timeout)
return client
def execute_commands(self, commands):
"""
Execute multiple commands in succession.
:param commands: List of unix commands as strings.
:type commands: List[str]
"""
for cmd in commands:
stdin, stdout, stderr = self.client.exec_command(cmd)
stdout.channel.recv_exit_status()
response = stdout.readlines()
for line in response:
print(f"INPUT: {cmd} | OUTPUT: {line}")
return response
def bulk_upload(self, files):
"""
Upload multiple files to a remote directory.
:param files: List of paths to local files.
:type files: List[str]
"""
total_files = len(files)
file_count = 0
for f in files:
self.upload_file(f)
file_count += 1
print(f"Uploaded {f}\n{total_files - file_count} to go...")
print(
f"Finished uploading {total_files} files to {self.remote_path} on {self.host}"
)
def upload_file(self, file):
"""Upload file to remote host"""
self.scp.put(file, remote_path=self.remote_path)
def download_file(self, file):
"""Download file from remote host."""
self.scp.get(file)
class SSH():
def __init__(self, ip, port = 22, username = '******', password = '******'):
self.ip = ip
self.port = port
self.username = username
self.password = password
self.ssh = None
self.scp = None
self.response = None
super().__init__()
def info(self):
print(self.ip, self.port, self.username, self.password)
return
def connect(self):
sshClient = paramiko.SSHClient()
try:
sshClient.load_system_host_keys()
sshClient.set_missing_host_key_policy(paramiko.AutoAddPolicy())
sshClient.connect(self.ip, self.port, self.username, self.password)
self.ssh = sshClient
except:
pass
return
def isAlive(self):
if self.ssh.get_transport() is not None:
return self.ssh.get_transport().is_active()
def close(self):
self.ssh.close()
def fileTransfer(self, srcFile, destFolder):
#transefer file form source to dest folder via scp
try:
self.scp = SCPClient(self.ssh.get_transport())
self.scp.put(srcFile,remote_path=destFolder)
print('transfer success, file {} \'s location is {}/{}'.format(srcFile, destFolder,srcFile))
self.scp.close()
except:
pass
return
def fileDownload(self, srcFile, destFolder):
#ransefer file form source to dest folder via scp
try:
self.scp = SCPClient(self.ssh.get_transport())
self.scp.get(srcFile,local_path=destFolder)
print('download success, file {} \'s location is {}'.format(srcFile, destFolder))
self.scp.close()
except:
traceback.print_exc()
return
def cmd(self, command):
#execute command on server via ssh
response = self.ssh.exec_command(command, get_pty=True)
self.response = response
return response
def stdout(self):
#return stdout of cmd
return self.response[1].readlines()
class Remote():
def __init__(self, name="", workdir=None, hostname=None,\
port=22, user=None, ssh_key_file=None, shell="bash",\
lookup_keys=False, allow_agent=True, resource_manager=None):
self.name = name
self.ssh_key_file = ssh_key_file
self.lookup_keys = lookup_keys
self.allow_agent = allow_agent
self.hostname = hostname
self.port = port
self.user = user
self.workdir = workdir
self.shell = shell
self.resource_manager = resource_manager
self.ssh = SSHClient()
try:
self.ssh.load_system_host_keys()
except:
pass
self.command_status = None
self.scp = None
self._progress_callback = None
self.cmd = None
self.auth_type = "password"
self.remote_linux_commands = [
"mkdir", "rm", "cd", "tar", "which", "qstat", "qdel", "qsub"
]
def __del__(self):
self.ssh.close()
def passphrase_required(self):
if self.ssh_key_file is None:
return False
for key_type in (DSSKey, ECDSAKey, Ed25519Key, RSAKey):
try:
pkey = key_type.from_private_key_file(self.ssh_key_file,
password=None)
except paramiko.ssh_exception.PasswordRequiredException:
return True
except paramiko.ssh_exception.SSHException:
pass
return False
def configure(self, remote_name, yaml_remote):
# Mandatory fields
self.name = remote_name
self.workdir = os.path.normpath(yaml_remote["remote-workdir"])
self.resource_manager = yaml_remote["resource-manager"]
if "config-host" in yaml_remote:
host = yaml_remote["config-host"]
host_data = {}
with open("/home/eduardo/.ssh/config", 'r') as config_file:
sshconfig = paramiko.config.SSHConfig()
sshconfig.parse(config_file)
if host in sshconfig.get_hostnames():
host_data = sshconfig.lookup(host)
else:
raise Exception(
"Host '{}' not found in '$HOME/.ssh/config'.".format(
host))
# Mandatory
try:
self.user = host_data["user"]
self.hostname = host_data["hostname"]
except KeyError as e:
raise Exception(
"Field %s not found in host '{}' at '$HOME/.ssh/config'.".
format(str(e)))
# Optional
if "port" in host_data.keys():
self.port = host_data["port"]
if "identityfile" in host_data.keys():
self.auth_type = "key"
self.ssh_key_file = host_data["identityfile"][0]
if "identitiesonly" in host_data.keys():
self.lookup_keys = not host_data["identitiesonly"]
else:
# Mandatory
self.hostname = yaml_remote["hostname"]
self.user = yaml_remote["user"]
# Optional
if "port" in yaml_remote.keys():
self.port = yaml_remote["port"]
if "ssh-key" in yaml_remote.keys():
self.auth_type = "key"
self.ssh_key_file = yaml_remote["ssh-key"]["file"]
if "lookup-keys" in yaml_remote["ssh-key"].keys():
self.lookup_keys = yaml_remote["ssh-key"]["lookup-keys"]
if "allow-agent" in yaml_remote["ssh-key"].keys():
self.allow_agent = yaml_remote["ssh-key"]["allow-agent"]
# Mandatory
self.workdir = os.path.normpath(yaml_remote["remote-workdir"])
self.resource_manager = yaml_remote["resource-manager"]
self.shell = yaml_remote["shell"]
# Optional
if "shell" in yaml_remote.keys():
self.shell = yaml_remote["shell"]
if "jobs-commands" in yaml_remote.keys():
self.jobs_commands = yaml_remote["jobs-commands"]
def available(self, timeout=60):
try:
self.connect(passwd="", timeout=timeout)
except paramiko.AuthenticationException:
pass
except socket.timeout:
raise ConnectionTimeout("Connection time-out after 60 seconds.")
finally:
self.close()
#TODO: Check python version (>2.7.* ???)
def init_remote(self):
if not self.remote_dir_exists(self.workdir):
time.sleep(1)
self.command("mkdir -p %s" % self.workdir)
_printer.print_msg("Remote workdir created.")
else:
raise Exception("Directory %s already exists in remote '%s'." %
(self.workdir, self.name))
cmd_not_available = False
_printer.print_msg("Checking remote dependencies...")
if not self.cmd_avail("qsub"):
_printer.print_msg(
"Warning: Command 'qsub' not available in '%s'." % self.name,
ignore_quiet=True)
cmd_not_available = True
if not self.cmd_avail("qstat"):
_printer.print_msg(
"Warning: Command 'qstat' not available in '%s'." % self.name,
ignore_quiet=True)
cmd_not_available = True
if not self.cmd_avail("qdel"):
_printer.print_msg(
"Warning: Command 'qdel' not available in '%s'." % self.name,
ignore_quiet=True)
cmd_not_available = True
if cmd_not_available:
_printer.print_msg("Info: Sometimes it is necessary to add the path where the\n" +\
" binaries qsub/qstat/qdel are located in the remote to the\n" +\
" ~/.bashrc or ~/.cshrc files.", ignore_quiet=True)
def connect(self, passwd=None, timeout=None, progress_callback=None):
self._progress_callback = progress_callback
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.connect(self.hostname, port=self.port, password=passwd, timeout=timeout, username=self.user,\
key_filename=self.ssh_key_file, look_for_keys=self.lookup_keys)
self.scp = SCPClient(self.ssh.get_transport(),
socket_timeout=60.0,
progress=self._progress_callback)
self.cmd = CommandExecuter(self.ssh)
# Unalias all the commands to avoid unexpected behaviour
for remote_cmd in self.remote_linux_commands:
cmd = "unalias {}".format(remote_cmd)
self.command(cmd, timeout=60, fail_on_error=False)
# def command(self, cmd, timeout=None, fail_on_error=True):
# stdin, stdout, stderr = self.ssh.exec_command(cmd, timeout=timeout)
# self.command_status = stdout.channel.recv_exit_status()
# self.command_status = stdout.channel.recv_exit_status()
# if fail_on_error:
# if self.command_status != 0:
# error = stderr.readlines()
# raise CmdExecutionError("".join([l for l in error if l]))
# return stdout.readlines()
def command(self, cmd, timeout=None, fail_on_error=True):
stdin, stdout, stderr, exit_status = self.cmd.exec_command(cmd)
self.command_status = exit_status
if fail_on_error:
if self.command_status != 0:
error = stderr
raise CmdExecutionError("".join([l for l in error if l]))
return stdout
def cmd_avail(self, cmd_name):
try:
output = self.command("which %s" % cmd_name, timeout=60)
except CmdExecutionError:
return False
return True
def upload(self, path_orig, path_dest):
self.scp.put(path_orig, path_dest)
def download(self, path_orig, path_dest):
self.scp.get(path_orig, path_dest)
def remote_file_exists(self, f):
try:
out = self.command("[ -f %s ]" % f)
except CmdExecutionError:
return False
return True
def remote_dir_exists(self, d):
try:
out = self.command("[ -d %s ]" % d)
except CmdExecutionError:
return False
return True
#TODO: Add debug flag GLOBALLY so remote can access it
def close(self):
if self.scp is not None:
self.scp.close()
self.ssh.close()
class SSH:
def __init__(self, key, user, host):
self.filepath = key
self.user = user
self.host = host
self.max_tries = 100
self.scp = None
self.client = None
self.ssh_key = None
self._load_ssh()
for i in range(self.max_tries):
try:
self._connect()
except NoValidConnectionsError as e:
if i == self.max_tries - 1:
print("Reached max number of tries for ssh connection. ")
raise e
continue
break
def _load_ssh(self):
self.ssh_key = RSAKey.from_private_key_file(self.filepath)
system(
f'ssh-copy-id -i {self.filepath} {self.user}@{self.host}>/dev/null 2>&1'
)
system(
f'ssh-copy-id -i {self.filepath}.pub {self.user}@{self.host}>/dev/null 2>&1'
)
def _connect(self):
""" Conect to remote host """
self.client = SSHClient()
self.client.load_system_host_keys()
self.client.set_missing_host_key_policy(AutoAddPolicy())
self.client.connect(self.host,
username=self.user,
key_filename=self.filepath,
look_for_keys=True,
timeout=5000)
self.scp = SCPClient(self.client.get_transport())
def disconnect(self):
""" Close ssh connection """
self.client.close()
self.scp.close()
def execute_commands(self, commands):
""" Execute commands """
for cmd in commands:
stdin, stdout, stderr = self.client.exec_command(cmd)
if stdout.channel.recv_exit_status() != 0:
for line in stderr.readlines():
print(line)
raise Exception
else:
for line in stdout.readlines():
print(line)
def upload_files(self, files):
""" Upload files to remote directory.
__________
files : [str], list of file paths to upload
"""
for f in files:
self.scp.put(f, recursive=True)
class RemoteClient:
"""Client to interact with a remote host via SSH & SCP."""
def __init__(self, host, user, ssh_key_filepath, remote_path):
self.host = host
self.user = user
self.ssh_key_filepath = ssh_key_filepath
self.remote_path = remote_path
self.client = None
self.scp = None
self.conn = None
def __connect(self):
"""
Open connection to remote host.
"""
try:
self.client = SSHClient()
self.client.load_system_host_keys()
self.client.set_missing_host_key_policy(AutoAddPolicy())
self.client.connect(self.host,
username=self.user,
key_filename=self.ssh_key_filepath,
look_for_keys=True,
timeout=5000)
self.scp = SCPClient(self.client.get_transport())
except AuthenticationException as error:
logger.info('Authentication failed: did you remember to create an SSH key?')
logger.error(error)
raise error
finally:
return self.client
def __disconnect(self):
self.client.close()
self.scp.close()
self.client = None
def upload_file(self, file, remote_path=None):
"""Upload a single file to a remote directory."""
if self.client is None:
self.client = self.__connect()
if not remote_path:
remote_path = self.remote_path
try:
self.scp.put(file,
recursive=True,
remote_path=remote_path)
except SCPException as error:
logger.error(error)
raise error
finally:
logger.info(f'Uploaded {file} to {self.remote_path}')
self.__disconnect()
@retry(Exception, logger=logger)
def execute_command(self, cmd):
"""
Execute command in ssh session.
:param cmd: unix command as string.
"""
if self.client is None:
self.client = self.__connect()
try:
stdin, stdout, stderr = self.client.exec_command(cmd)
stdout.channel.recv_exit_status()
response = stdout.readlines()
error = stderr.readlines()
if response or error:
msg = f'INPUT: {cmd}\n'
if response:
msg += 'STDOUT:\n{}'.format('\n'.join(response))
if error:
msg += 'STDERR:\n{}'.format('\n'.join(error))
logger.info(msg)
except Exception:
self.__disconnect()
raise
else:
self.__disconnect()
