def _setup_node(node, clmap):
if node["is_master"]:
script_body = clmap["master_script"]
else:
script_body = clmap["slave_script"]
ssh = SSHClient()
try:
_setup_ssh_connection(node["ip"], ssh)
sftp = ssh.open_sftp()
fl = sftp.file("/tmp/eho-hadoop-init.sh", "w")
fl.write(script_body)
fl.close()
sftp.chmod("/tmp/eho-hadoop-init.sh", 0500)
ret = _open_channel_and_execute(ssh, "/tmp/eho-hadoop-init.sh " ">>/tmp/eho-hadoop-init.log 2>&1")
_ensure_zero(ret)
finally:
ssh.close()
class SSHModule(Module):
def __init__(self, config):
self.config_data = super().\
__init__(config=config, defaults=self.defaults)
self.module_config = self.config_data['defaults']
self.connection = SSHClient()
self.connection.set_missing_host_key_policy(AutoAddPolicy())
self.__connect()
self.stdout, self.stdin = [], []
def __connect(self):
self.connection.connect(username=self.module_config['user'],
password=self.module_config['password'],
hostname=self.module_config['host'])
def __close(self):
self.connection.close()
def execute_command(self, command):
stdin, stdout, _ = self.connection.exec_command(command)
self.stdout = stdout.readlines()
self.stdin = stdin
self.__close()
print(self.stdout)
return ''.join(self.stdout).split('\n')[:-1]
def get_file(self, r_file_dir, l_file_dir):
self.sftp_connection = self.connection.open_sftp()
self.sftp_connection.get(r_file_dir, l_file_dir)
def put_file(self, l_file_dir, r_file_dir):
self.sftp_connection = self.connection.open_sftp()
self.sftp_connection.put(l_file_dir, r_file_dir)
class ssh_session:
def __init__(self):
_username="******"
_password="******"
vm_ip = "192.168.11.137"
self.client = SSHClient()
self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.client.connect(vm_ip, 22, username=_username, password=_password)
def run(self, cmd):
stdin, stdout, err = self.client.exec_command(cmd)
print(stdout.readline(2048))
return self
def upload(self, from_path, file):
sftp=self.client.open_sftp()
sftp.put(from_path, file)
print(f'file uploaded.')
#self.run(f"mv {file} ./uploaded")
#print("filed moved")
def download(self, from_path, to_path):
sftp=self.client.open_sftp()
sftp.get(from_path, to_path)
print(f'file downloaded.')
def logout(self):
self.client.close()
class SSH(object):
"""
package for ssh_client, in this class stores the sftp handlers and the
ssh_client
"""
def __init__(self, host, user, password, port=22):
self.host = host
self.user = user
self.password = password
self.port = port
def ssh_connect(self, task_list, ret_map):
try:
self.clt = SSHClient()
self.clt.load_system_host_keys()
self.clt.set_missing_host_key_policy(AutoAddPolicy())
self.clt.connect(hostname=self.host, username=self.user, password=self.password, port=self.port)
self.sftp = self.clt.open_sftp()
except Exception, e:
try:
self.clt.connect(hostname=self.host, username=self.user, password=self.password, port=22)
self.sftp = self.clt.open_sftp()
except Exception, e:
for task in task_list:
if len(task) > 1:
ret_map[task[0]] = (0, "ssh connect error")
return False
class FileService(object):
def __init__(self, storage_settings: StorageSettings):
self.ssh = SSHClient()
self.ssh.set_missing_host_key_policy(AutoAddPolicy())
self.ssh.connect(hostname='%s://%s' %
(storage_settings.protocol, storage_settings.host),
username=storage_settings.user,
port=storage_settings.port,
password=storage_settings.password)
def upload_file(self, local_file, remote_path, callback=None):
try:
sftp = self.ssh.open_sftp()
with sftp.open(remote_path, 'wb') as remote_file:
remote_file.write(local_file.read())
remote_file.close()
sftp.close()
except (BadHostKeyException, AuthenticationException, SSHException,
socket.error) as exception:
logging.error(exception)
def download_file(self, filename, remote_path, callback=None):
try:
sftp = self.ssh.open_sftp()
with sftp.open(filename, 'rb') as remote_file:
local_file = remote_file.read()
remote_file.close()
sftp.close()
return local_file
except (BadHostKeyException, AuthenticationException, SSHException,
socket.error) as exception:
logging.error(exception)
def remove_file(self, filename):
try:
sftp = self.ssh.open_sftp()
sftp.remove(filename)
sftp.close()
except (BadHostKeyException, AuthenticationException, SSHException,
socket.error) as exception:
logging.error(exception)
def open_file(self):
s3 = resource('s3', region_name='us-east-2')
bucket = s3.Bucket('brain-spark')
bucket_file = bucket.Object(
'models/parts_identification/test/0000001.jpg')
temporary = NamedTemporaryFile()
with open(temporary, 'wb') as binary:
bucket_file.download_file(binary)
image = imread(temporary.name)
return image
def clean_up_sftp(ssh_client: SSHClient,
ftp_config: FTPConfig) -> Generator[None, None, None]:
yield
ssh_client.connect(
hostname=ftp_config.host,
username=ftp_config.username,
password=ftp_config.password,
port=ftp_config.port,
)
sftp = ssh_client.open_sftp()
sftp.chdir(ftp_config.path)
sftp = ssh_client.open_sftp()
_sftp_rm(ftp_config.path, sftp)
sftp.close()
class DeployTool(object):
"""docstring for ClassName"""
def __init__(self, remote_user, remote_password, remote_server):
self.remote_user = remote_user
self.remote_password = remote_password
self.remote_server = remote_server
self.client = SSHClient()
self.client.load_system_host_keys()
self.client.connect(hostname=self.remote_server,
username=self.remote_user,
password=self.remote_password)
def _ssh(self, command):
stdin, stdout, stderr = self.client.exec_command(command)
ssh_output = stdout.read()
ssh_error = stderr.read()
return (ssh_output, ssh_error)
def _sftp(self, infile, outfile):
sftp = self.client.open_sftp()
sftp.put(infile, outfile)
def deploy(self):
print '### Downloading Configurator ###'
self._ssh('wget -P /opt/ http://10.100.10.146/config/configurator.sh')
self._ssh('chmod 777 /opt/configurator.sh')
print '### Running Configurator ###'
self._ssh('/opt/configurator.sh')
def _ssh(self, script_file, parameters):
"""
Copies a templated script to the EC2 instance and yields the SSH
session.
"""
import os.path
from paramiko import SSHClient
from paramiko.client import AutoAddPolicy
this_file_dir = os.path.dirname(__file__)
with open(os.path.join(this_file_dir, script_file)) as fh:
script = fh.read() % parameters
response = self.ec2.describe_instances(InstanceIds=[self.instance_id])
instance = response['Reservations'][0]['Instances'][0]
ip = instance['PublicIpAddress']
ssh = SSHClient()
# TODO: is there any way to get the host key via AWS APIs?
ssh.set_missing_host_key_policy(AutoAddPolicy)
ssh.connect(ip, username='******', pkey=self.key, look_for_keys=False)
try:
with ssh.open_sftp() as ftp:
with ftp.file(script_file, 'w') as fh:
fh.write(script)
yield ssh
except Exception as e:
raise RuntimeError('ERROR: %s (on ubuntu@%s)' % (e, ip))
finally:
ssh.close()
class SFTP(Operations):
"""A simple SFTP filesystem. Requires paramiko:
http://www.lag.net/paramiko/
You need to be able to login to remote host without entering a password.
"""
def __init__(self, host, path='.'):
self.client = SSHClient()
self.client.load_system_host_keys()
self.client.connect(host)
self.sftp = self.client.open_sftp()
self.root = path
def __del__(self):
self.sftp.close()
self.client.close()
def __call__(self, op, path, *args):
print '->', op, path, args[0] if args else ''
ret = '[Unhandled Exception]'
try:
ret = getattr(self, op)(self.root + path, *args)
return ret
except OSError, e:
ret = str(e)
raise
except IOError, e:
ret = str(e)
raise OSError(*e.args)
def transfer_file(host_dict, file_dict):
# create SSH client with paramiko and connect with system host keys
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(host_dict['hostname'], username=host_dict['username'])
sftp = ssh.open_sftp()
# see if the remote directory exists
try:
sftp.stat(file_dict['remote_directory'])
# if not, then recursivley add each file in the path
except FileNotFoundError:
f_path = ''
for f in file_dict['remote_directory'].replace(file_dict['remote_root'], '').split('/')[:-1]:
f_path += str('/' + f)
print(f_path)
try:
sftp.stat(str(file_dict['remote_root'] + f_path))
except FileNotFoundError:
sftp.mkdir(str(file_dict['remote_root'] + f_path))
# set up scp protocol and recursively push the directories across
# scp = SCPClient(ssh.get_transport())
print(file_dict['local_file'])
print(file_dict['remote_directory'])
sftp.put(file_dict['local_file'], file_dict['remote_directory'])
ssh.close()
class SSHConnection:
def connect(self, *args, **kwargs):
print("connect called...")
input()
from paramiko import SSHClient, AutoAddPolicy
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
self.ssh.set_missing_host_key_policy(AutoAddPolicy())
self.ssh.connect(*args, **kwargs)
self.sftp = self.ssh.open_sftp()
def invalid_command(self, *args, **kwargs):
print("Invalid command")
def run_commands(self, config):
for comm, args, kwargs in config:
getattr(self, comm)(*args, **kwargs)
def __getattr__(self, attr):
print("Processing command:", attr)
if hasattr(self.ssh, attr):
return getattr(self.ssh, attr)
elif hasattr(self.sftp, attr):
return getattr(self.sftp, attr)
else:
return self.invalid_command
def sshtungetip():
"""
Подключается по sftp к серверу получает данные JSON. Осуществляет поиск IP и возращает его в качестве параметра в
место вызова.
"""
global setstatus
global login
setstatus = "Получение ip адреса"
from paramiko import SSHClient
from paramiko import AutoAddPolicy
from paramiko import RSAKey
getipsftp = SSHClient()
pk = RSAKey.from_private_key_file('srv.key')
getipsftp.load_system_host_keys()
getipsftp.set_missing_host_key_policy(AutoAddPolicy())
getipsftp.connect(hostname=publicipadress[0],
port=str(publicipadress[1]),
username='******',
pkey=pk)
sftp = getipsftp.open_sftp()
import json
fip = None
with sftp.file('ip-client.json', 'r') as f:
dataip = json.load(f)
getipsftp.close()
for x in dataip:
if login == x["User"]["login"] or login in x["User"]["FullName"]:
fip = x["User"]["ipaddress"]["ip"][0]
fullname = x["User"]["login"]
if fip == "Не найдено" or fip is None:
fip = "IP не найден"
time.sleep(1)
return (fip)
def launch(server, remote_code):
client = SSHClient()
client.load_system_host_keys()
client.connect(server)
stdin, stdout, stderr = client.exec_command('python3')
stdin.write(remote_code)
stdin.close()
stdin.channel.shutdown_write() # Sends EOF
for line in stdout:
m = re.match(CONN_FILE_RE, line)
if m:
remote_conn_file = m.group(1)
break
else:
print('Exit status', stdin.channel.exit_status)
print("Stderr:")
print(stderr.read().decode())
print("Stdout:")
print(stdout.read())
raise RuntimeError("Remote kernel failed to start")
print("Remote connection file:", remote_conn_file)
with client.open_sftp() as sftp:
with sftp.open(remote_conn_file) as f:
return json.load(f)
def _uploadFiles(self, dstdir, **sshargs):
(yield TaskOutput(u"ENTER", OutputType.NOTIFY))
sshcli = SSHClient()
sftpcli = None
code = 0
try:
if not (yield TaskOutput(u"Conntecting to %s ..." % sshargs["hostname"])):
raise CommandTerminated()
sshcli.set_missing_host_key_policy(AutoAddPolicy())
sshcli.connect(**sshargs)
if not (yield TaskOutput(u"Connected, ready to upload ...")):
raise CommandTerminated()
ret = sshcli.exec_command("[ -d {0} ] && rm -rf {0}; mkdir -p {0}".format(dstdir))
errstr = ret[2].read()
if errstr != "":
raise Exception(errstr)
sftpcli = sshcli.open_sftp()
for f in os.listdir(self.hdiff_dir):
if f.lower().endswith(".html"):
localfile = os.path.join(self.hdiff_dir, f)
remotefile = os.path.join(dstdir, f).replace(os.sep, "/")
if not (yield TaskOutput(u"Uploading %s ..." % f)):
raise CommandTerminated()
sftpcli.put(localfile, remotefile)
except CommandTerminated:
code = -2
(yield TaskOutput(u"Uploading Terminited", OutputType.WARN))
except Exception as ex:
code = -1
(yield TaskOutput(ex.message, OutputType.ERROR))
finally:
if sftpcli:
sftpcli.close()
sshcli.close()
(yield TaskOutput(u"EXIT %d" % code, OutputType.NOTIFY))
class DeployTool(object):
"""docstring for ClassName"""
def __init__(self, remote_user, remote_password, remote_server):
self.remote_user = remote_user
self.remote_password = remote_password
self.remote_server = remote_server
self.client = SSHClient()
self.client.load_system_host_keys()
self.client.connect(
hostname=self.remote_server,
username=self.remote_user,
password=self.remote_password
)
def _ssh(self, command):
stdin, stdout, stderr = self.client.exec_command(command)
ssh_output = stdout.read()
ssh_error = stderr.read()
return (ssh_output, ssh_error)
def _sftp(self, infile, outfile):
sftp = self.client.open_sftp()
sftp.put(
infile,
outfile
)
def deploy(self):
print '### Downloading Configurator ###'
self._ssh('wget -P /opt/ http://10.100.10.146/config/configurator.sh')
self._ssh('chmod 777 /opt/configurator.sh')
print '### Running Configurator ###'
self._ssh('/opt/configurator.sh')
def startConnection(self):
"""
connection to remote server
can launch a thread checking every minute your mailbox...
"""
#if not self.directory:return
#self.parent().close()
self.close()
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
try:
client.connect(str(self.host.text()), username=str(self.username.text()), password=str(self.password.text()))
except SSHException: qApp.instance().showErrorMessage("Error", "The ssh session could not be established")
except AuthenticationException: qApp.instance().showErrorMessage("Error","Authentication failed")
except BadHostKeyException: qApp.instance().showErrorMessage("Error", "the server's host key could not be verified")
sftpCp = self.cleaningRepository(client)
if sftpCp:
sftp = client.open_sftp()
self.sftpCopying(sftp)
sftp.close()
else:
self.retrieveDirTree(c, out)
self.makeRScript(client)
self.makeShellScript(client)
if self.interactiveSession.isChecked():
import interactive
print ('invoking a shell')
chan = client.invoke_shell()
chan.send('bash\n')
interactive.interactive_shell(chan)
chan.close()
else:
self.launchCommand(client)
client.close()
def install(self):
filename = SyncGatewayInstaller._generate_filename(
self.__version, self.__build)
if not Path(filename).exists():
raise Exception(
"Unable to find installer, please call download first")
print("Installing Sync Gateway to {}...".format(self.__url))
ssh_client = SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(WarningPolicy())
ssh_connect(ssh_client, self.__url, self.__ssh_keyfile,
str(self.__ssh_keypass))
(_, stdout, _) = ssh_client.exec_command("test -f {}".format(filename))
if stdout.channel.recv_exit_status() == 0:
print(
"Install file already present on remote host, skipping upload..."
)
else:
print("Uploading file to remote host...")
sftp = ssh_client.open_sftp()
sftp_upload(sftp, filename, filename)
sftp.close()
ssh_command(ssh_client, self.__url,
"sudo yum install -y {}".format(filename))
print("Install finished!")
def get(self, remote_path, local_path=None, out_stream=sys.stdout, verbose=False):
"""
Copy a file from the remote system to the local system.
:param remote_path:
:param local_path:
:param out_stream:
:param verbose:
:return: :rtype:
"""
if local_path is None:
local_path = remote_path
self.display("scp '{src}' '{dest}'".format(src=remote_path, dest=local_path),
out_stream=out_stream, verbose=verbose)
names = self.run(['ls', '-1', remote_path]).split('\n')
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(Project.address, Project.port, Project.user, Project.password)
# scp = SFTPClient.from_transport(ssh.get_transport())
# output = scp.get(remote_path, local_path, recursive=True)
ftp = ssh.open_sftp()
for name in names:
print(name)
ftp.get(name, local_path)
output = repr(names)
self.display(output, out_stream=out_stream, verbose=verbose)
return output
def execute_gpu(img_name):
client = SSHClient()
client.load_system_host_keys()
print("gpu server connection..")
client.connect(host, username=user, port=port, password=password)
print("gpu server connection Ok.")
sftp = client.open_sftp()
source_img = os.path.join(source_img_dir, img_name + '.jpg')
target_img = os.path.join(target_img_dir, img_name + '.png')
result_img = os.path.join(result_img_dir, img_name + '.png')
sftp.put(source_img, STYLEGAN_FFHQ_DIR + img_name)
stdin, stdout, stderr = client.exec_command(RUN_COMMAND)
# 오류 발생 시 출력
for i in stderr.readlines():
print(i)
sftp.get(STYLEGAN_RESULT_IMAGE_PATH, target_img)
swap(source_img, target_img, result_img)
client.close()
sftp.close()
def get_fc_file(self, fc_ip, fc_username, fc_password):
"""
Connect to the Flow Connector scp the file:
'/lancope/var/sw/today/data/exporter_device_stats.txt' to /tmp.
"""
print(f"\nSSH connect to Flow Collector: {fc_ip}")
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(
fc_ip,
username=fc_username,
password=fc_password,
look_for_keys=False,
allow_agent=False,
)
sftp = ssh.open_sftp()
with sftp.open(self.fc_datafile_path) as tsv:
current_device = pd.read_csv(tsv, sep="\t")
print("File successfully retrieved and read...")
# Replace spaces with underscores in column names
current_device.columns = current_device.columns.str.replace(" ", "_")
# Add in the FC IP, this is useful for debugging
current_device["FC_IP"] = fc_ip
return current_device
def try_password(ip_addr, password):
try:
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
# print "当前处理的ip是",ip_addr,"密码是"+password,ctime()
ssh.connect(ip_addr,
port,
username,
password,
pkey=None,
timeout=2,
allow_agent=False,
look_for_keys=False,
banner_timeout=5)
print
"目标ip" + ip_addr + "password", ctime()
sftpclient = ssh.open_sftp()
sftpclient.put(Trojans_file_dir + "/bingtu.sh",
remote_dir + "/bingtu.sh")
ssh.close()
sftpclient.close()
except Exception as e:
print(e)
# print "%s processing %s %s" % (name, ip_addr,password),ctime(),"\n"
# print "匹配中\n"
pass
class SSHConnection:
attrs = "open", "get", "put", "chmod", "exec_command", "mkdir", "rmdir"
def __init__(self, hostname, username, password):
self.hostname, self.username, self.password = hostname, username, password
from paramiko import SSHClient, AutoAddPolicy
self.ssh_client = SSHClient()
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
def __enter__(self):
self.ssh_client.connect(hostname=self.hostname,
username=self.username,
password=self.password)
self.sftp_client = self.ssh_client.open_sftp()
return self
def __getattr__(self, attr):
if attr not in self.attrs:
raise AttributeError(
f"'{self.__class__}' object has no attribute '{attr}'")
elif hasattr(self.sftp_client, attr):
return getattr(self.sftp_client, attr)
elif hasattr(self.ssh_client, attr):
return getattr(self.ssh_client, attr)
def __exit__(self, et, ev, tb):
self.sftp_client.close()
self.ssh_client.close()
def sync_public_key(host, port=22, username=None, password=None):
try:
client = SSHClient()
client.connect(hostname=host, username=username, password=password)
except SSHException, e:
client.close()
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
client.connect(hostname=host, username=username, password=password)
sftp_client = client.open_sftp()
id_rsa_pub = "/home/%s/.ssh/id_rsa.pub" % LOCAL_USER_NAME
if username == "root":
remote_rsa_pub = "/root/.ssh/%s.pub" % (LOCAL_USER_NAME)
else:
remote_rsa_pub = "/home/%s/.ssh/%s.pub" % (username, LOCAL_USER_NAME)
print remote_rsa_pub
try:
sftp_client.put(id_rsa_pub , remote_rsa_pub)
except Exception, e:
"""
if the remote host did have .ssh dirctory
"""
print e
def run(self):
if 'no_transfer_done' in open(self.input().path, 'r').readlines():
with self.output().open('w') as f:
f.write('')
else:
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(self.hostname, username=self.username)
if os.path.isfile(os.path.join(os.getcwd(),
self.target_list_file)):
os.remove(os.path.join(os.getcwd(), self.target_list_file))
os.system('touch ' +
os.path.join(os.getcwd(), self.target_list_file))
verne_dirs = []
v_sftp = ssh.open_sftp()
v_sftp.chdir(os.path.join(self.remote_root, self.timestamp))
for i in v_sftp.listdir():
lstatout = str(v_sftp.lstat(i)).split()[0]
if 'd' in lstatout:
verne_dirs.append(str(i))
v_sftp.close()
verne_dirs.sort()
write_string = ' '.join(verne_dirs)
with open(os.path.join(os.getcwd(), self.target_list_file),
'w') as f:
f.write(write_string)
local_file = os.path.join(os.getcwd(), 'READY')
if not local_file:
with open(local_file, 'w') as f:
f.write('')
scp = SCPClient(ssh.get_transport())
scp.put(os.path.join(os.getcwd(), 'READY'),
recursive=True,
remote_path=os.path.join(self.remote_root, self.timestamp))
scp.put(os.path.join(os.getcwd(), self.target_list_file),
recursive=True,
remote_path=os.path.join(self.remote_root, self.timestamp))
scp.close()
ssh.exec_command(
str('chmod -R 775 ' +
os.path.join(self.remote_root, self.timestamp)))
curl_string = str(
'curl -X POST "https://' + self.user + ':' + self.rand_string +
'@jenkins-fragalysis-cicd.apps.xchem.diamond.ac.uk/job/Loader%20Image/build?token='
+ self.token + '" -k')
os.system(curl_string)
with self.output().open('w') as f:
f.write('')
def rollback_using_import_policy(self, version):
filenames = self.get_filenames(version)
# Create the tar file of the selected JSON files
self._generate_tar_gz(filenames, version)
# Put the tar file on the APIC
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(self.session.ipaddr, username=self.session.uid, password=self.session.pwd)
sftp = ssh.open_sftp()
sftp.chdir('/home/%s' % self.session.uid)
sftp.put('./ce_snapback.tar.gz', 'ce_snapback.tar.gz')
# Remove the local tar file
os.remove('./ce_snapback.tar.gz')
# Send the import policy to the APIC
url = '/api/node/mo/uni/fabric.json'
payload = {"configImportP": {"attributes": { "name": "snapback",
"fileName": "ce_snapback.tar.gz",
"adminSt": "triggered",
"importType": "replace",
"importMode": "atomic"},
"children": [{"configRsImportSource": {"attributes": {"tnFileRemotePathName": "snapback"},
"children": []
}}]}}
resp = self.session.push_to_apic(url, payload)
return resp
class Sender(object):
def __init__(self,server,username,password,dest_path,from_path):
self.dest_path = dest_path
self.from_path = from_path
self.recorder = Recorder.Recorder()
self.ssh = SSHClient()
self.ssh.load_system_host_keys()
self.ssh.connect(server,username=username,password=password)
self.sftp = self.ssh.open_sftp()
self.scp = SCPClient(self.ssh.get_transport())
def send(self,file_path):
if op.exists(file_path):
file_modify_time = time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(os.stat(file_path).st_mtime))
if not self.recorder.isSynced(file_path,file_modify_time):
new_file_path = os.path.join(self.dest_path+'/',file_path.split(self.from_path+os.sep)[1])
new_file_path = new_file_path.replace('\\','/')
new_file_dir,new_file = op.split(new_file_path)
if not rexists(self.sftp,new_file_dir):
rmkdir(self.sftp,new_file_dir)
print 'uploading %s .....' % (file_path)
self.scp.put(file_path,new_file_path)
self.recorder.record(file_path,file_modify_time)
else:
pass
def do_ansible(ssh: SSHClient):
script_dir = path.dirname(path.abspath(__file__))
ansible_dir = path.join(script_dir, 'ansible')
remote_ansible_dir = PRETTY_CONFIG['REMOTE_HOME'] + '/ansible'
sftp = ssh.open_sftp()
try:
try:
sftp.stat(remote_ansible_dir)
ssh_exec(ssh, 'rm -rf ' + remote_ansible_dir)
except IOError as ex:
# ファイルが存在しないと IOError になる
logger.debug(ex)
ssh_cpdir(sftp, ansible_dir, remote_ansible_dir)
finally:
sftp.close()
logger.info('リモートサーバ上でAnsibleを実行します')
ssh_exec(
ssh, 'ansible-playbook -i {inv} -e ssh_port={ssh_port} \
-e db_port={db_port} -e db_user={db_user} -e db_password={db_pwd} {book}'.
format(
inv=remote_ansible_dir + '/hosts.yml',
book=remote_ansible_dir + '/db.yml',
ssh_port=PRETTY_CONFIG['SSH_PORT'],
db_port=PRETTY_CONFIG['DB_PORT'],
db_user=PRETTY_CONFIG['DB_USER'],
db_pwd=PRETTY_CONFIG['DB_PASSWORD'],
))
def execute_remotely_on(instance_ip, scriptname, ssh_username, as_sudo=False):
client = None
sftp = None
try:
print "working on", instance_ip
client = SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(instance_ip, username=ssh_username)
sftp = client.open_sftp()
temp_file_name = "remove_me"
sftp.put(scriptname, '/tmp/%s' % temp_file_name)
command = 'bash /tmp/%s' % temp_file_name
if as_sudo:
command = 'sudo bash /tmp/%s' % temp_file_name
_, stdout, stderr = client.exec_command(command)
print "stdout: \n", stdout.read(), "stderr:", stderr.read()
sftp.remove('/tmp/%s' % temp_file_name)
except (paramiko.AuthenticationException,
paramiko.SSHException) as message:
print "ERROR: SSH connection to ", instance_ip, " failed: ", str(
message)
print "make sure you have you private key setup in ~/.ssh"
finally:
if client:
client.close()
if sftp:
sftp.close()
def create_backup(self):
current_backup_dir = None
out = []
if 'port' in self.connection:
ssh_port = self.connection['port']
else:
ssh_port = 22
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
vm = self.find_virtual_machine()
if vm is None:
logging.critical('Failed to obtain VM')
return False, current_backup_dir
else:
vm_status = vm.isActive()
logging.warning('VM "{}" found [Status: {}]'.format(
self.connection['vm_name'], vm_status))
if vm.isActive() == 1:
deactivation = self._deactivate_vm(vm)
if not deactivation:
logging.critical('Could not shutdown machine.')
return False, current_backup_dir
# return True
images_to_save = self._print_all_vm_disks(vm)
ssh.load_host_keys(
filename=path.join(path.expanduser('~'), '.ssh', 'known_hosts'))
try:
ssh.connect(hostname=self.connection['host'],
username=self.connection['user'],
port=ssh_port,
key_filename=self.connection['keyfile'])
except SSHException as e:
logging.critical('SSH Failed: {}'.format(e))
ssh.close()
return False, current_backup_dir
try:
current_backup_dir = datetime.today().strftime("backup-%Y%m%d%H%M")
stdin, stdout, ssh_stderr = ssh.exec_command('mkdir {}/{}'.format(
self.remote_path, current_backup_dir))
stdin.flush()
for image_to_save in images_to_save:
stdin, stdout, ssh_stderr = ssh.exec_command(
'cp -v {} {}/{}'.format(image_to_save, self.remote_path,
current_backup_dir))
out.append(stdout.readlines())
stdin.flush()
# Dump XML too
ftp = ssh.open_sftp()
ftp.chdir(self.remote_path + '/' + current_backup_dir)
with ftp.open('VMdump.xml') as xml_dump_fp:
xml_dump_fp.write(vm.XMLDesc())
ftp.close()
except SSHException as e:
logging.critical('SSH error: {}'.format(e))
return False, current_backup_dir
if not self._activate_vm(vm):
out.append("Failed to reactivate VM\n")
return out, current_backup_dir
def __init__(self,hostname='bayonet-08.ics.uci.edu',
username='******',password='******'):
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.connect(hostname,username=username,password=password)
self.ssh = ssh
self.scp = SCPClient(ssh.get_transport())
self.ftp = ssh.open_sftp()
def sftp_client_from_ssh_client(hostname, username, password):
from paramiko import SSHClient, AutoAddPolicy
sshclient = SSHClient()
sshclient.set_missing_host_key_policy(AutoAddPolicy())
sshclient.connect(hostname, username=username, password=password)
return sshclient.open_sftp()
def connect(self, host, username, password, port=22):
self.close(host)
try:
clt = SSHClient()
clt.load_system_host_keys()
clt.set_missing_host_key_policy(AutoAddPolicy())
clt.connect(hostname=host, username=username,
password=password, port=port)
stp = clt.open_sftp()
except Exception, e:
try:
clt.connect(hostname=host, username=username,
password=password, port=222)
stp = clt.open_sftp()
except Exception, e:
Log.error("Ssh connect to " + host + " failed: %s" % str(e))
return False
def copy_build(ssh_session: paramiko.SSHClient, path: str):
copy_func = info("Copying Build")
sftp: MySFTPClient = ssh_session.open_sftp()
sftp.__class__ = MySFTPClient
sftp.put_dir('build', path)
copy_func("Done", True)
class SSHUtilClient:
def __init__(self, settings):
self.settings = settings
logger.debug("Initializing SSH client ...")
self.ssh = SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
self.ssh.load_system_host_keys()
logger.info("Connecting to remote host ...")
try:
self.ssh.connect(self.settings.host, timeout=10, username=self.settings.user)
except gaierror as e:
logger.error(
"{err}, make sure you have connectivity to {h}".format(
h=self.settings.host,
err=e
)
)
sys.exit(1)
logger.info("Successfully connected to remote.")
logger.info("Successfully initialized SSH client.")
# Define scp-put progress callback that prints the current percentage completed for the file
def progress(self, filename, size, sent):
sys.stdout.write("%s\'s progress: %.2f%% \r" % (filename, float(sent)/float(size)*100) )
def sync(self):
"""pointer to scp module's sync function
"""
scp = SCPClient(self.ssh.get_transport(), progress = self.progress)
sftp = self.ssh.open_sftp()
sync = SSHSync(scp, sftp)
logger.info("Successfully initialized SCP client.")
return sync
def execute(self, remote_cmd, stream_stdout=False):
# Send the command (non-blocking)
logger.debug("Remote command is '{}'".format(remote_cmd.rstrip('\n')))
logger.info("Executing command on remote machine...")
stdin, stdout, stderr = self.ssh.exec_command(remote_cmd)
if stream_stdout:
# Wait for the command to terminate
while not stdout.channel.exit_status_ready():
# Only print(data if there is data to read in the channel)
if stdout.channel.recv_ready():
rl, wl, xl = select.select([stdout.channel], [], [], 0.0)
if len(rl) > 0:
# print(data from stdout)
line = stdout.channel.recv(2048).decode("utf-8")
print(line)
# Disconnect from the host
logger.info("Remote command done, closing SSH connection")
self.ssh.close()
return (stdin,stdout,stderr)
else:
return (stdin,stdout,stderr)
def connect(self):
client = SSHClient()
client.load_system_host_keys()
client.set_missing_host_key_policy(WarningPolicy())
client.connect(hostname=self.remote_host, username=self.remote_user)
LOG.debug(client)
self.client = client
self.sftp = client.open_sftp()
def scp_to(ip, local_path, user='******', password=None, remote_path=""):
"""
Send a file to a server
"""
ssh = SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
ssh.connect(ip, username=user, password=password, allow_agent=False)
sftp = ssh.open_sftp()
sftp.put(local_path, remote_path)
def start_scheduler():
cmd = [
'aws', 'ec2', 'run-instances', '--image-id', IMAGE_ID,
'--instance-type', INSTANCE_TYPE, '--key-name', KEY,
'--security-group-ids', SECURITY_GROUP, '--region', REGION,
'--tag-specifications',
'ResourceType=instance,Tags=[{}Key=Name,Value=scheduler{}]'.format(
'{', '}')
]
proc = subprocess.run(cmd, stdout=subprocess.PIPE)
iid = json.loads(proc.stdout)['Instances'][0]['InstanceId']
print("Starting {}...".format(iid), end="", flush=True)
instance = get_instances(REGION)[iid]
while instance['State']['Name'] != 'running':
sleep(3)
instance = get_instances(REGION)[iid]
print("Started!")
k = RSAKey.from_private_key_file(Path(KEYFILE).expanduser())
c = SSHClient()
c.set_missing_host_key_policy(AutoAddPolicy())
ip = instance['PublicIpAddress']
print("Connecting to {}".format(ip))
c.connect(hostname=ip, username="******", pkey=k)
# send progress if there is any
archives = glob.glob('archive/*.tar')
if len(archives) > 0:
tar = archives[0]
print("Sending {}...".format(tar), end="", flush=True)
sftp = c.open_sftp()
def transfer_progress(completed, todo):
print("\rSending {}...{:.2%}".format(tar, completed / todo),
end="",
flush=True)
sftp.put(tar, "{}".format(Path(tar).name), callback=transfer_progress)
print("\rSending {}...Done! ".format(tar))
print("Unpacking archives...", end="", flush=True)
cmd = 'tar -xf {}'.format(Path(tar).name)
c.exec_command(cmd)
print("Done!")
# start the scheduler
print("Starting scheduler...", end="", flush=True)
cmd = "tmux new -d -s scheduler './sklearn-pmlb-benchmarks/src/scheduler.py --resume {} --max-connections {}'".format(
OUTPUTDIR, NUMCLIENTS)
c.exec_command(cmd)
c.close()
print("Done!")
print("Started at {}:{}".format(instance['PrivateIpAddress'], PORT))
def __up(save_path, fp):
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(HOST, 22, USER, PASS)
sftp = ssh.open_sftp()
fp.seek(0)
sftp.put(fp.name, save_path)
sftp.close()
ssh.close()
class SSHConnection:
attrs = "open", "get", "put", "chmod", "exec_command", "mkdir", "rmdir"
def __init__(self, hostname, username, password, queue):
self.hostname, self.username, self.password = hostname, username, password
self.queue = queue
from paramiko import SSHClient, AutoAddPolicy
self.client = SSHClient()
self.client.set_missing_host_key_policy(AutoAddPolicy())
def __enter__(self):
self.client.connect(hostname=self.hostname,
username=self.username,
password=self.password)
self.channel = self.client.invoke_shell()
self.stdin = self.channel.makefile("w")
self.stdout = self.channel.makefile("r")
self.stderr = self.channel.makefile_stderr("r")
self.output_thread = Thread(target=self.output)
self.error_thread = Thread(target=self.error)
self.output_thread.start()
self.error_thread.start()
self.sftp_client = self.client.open_sftp()
return self
def __getattr__(self, attr):
if attr not in self.attrs:
raise AttributeError(
f"'{self.__class__}' object has no attribute '{attr}'")
elif hasattr(self.sftp_client, attr):
return getattr(self.sftp_client, attr)
elif hasattr(self.client, attr):
return getattr(self.client, attr)
def __exit__(self, et, ev, tb):
self.sftp_client.close()
self.stdout.close()
self.stderr.close()
self.stdin.close()
self.channel.close()
self.client.close()
def exec(self, command):
self.stdin.write(command + "\n")
def output(self):
for line in self.stdout:
self.queue.put(self.hostname, "stdout", line)
def error(self):
for line in self.stderr:
self.queue.put(self.hostname, "stderr", line)
def build(conn: paramiko.SSHClient, **extra) -> bool:
'''Build a general docker
:conn: An established paramiko connection.
:extra: Extra information.
{
'dockerprog': The docker program.
'user': The user name.
'hostid': The host where the docker is built in.
'content': The customized docker file content.
'himsg': The hello message from the administrator.
'apg': The apt souce type.
}
:return: if build successfully, return true, or else false.
'''
tm = str(time.time())
sftp = conn.open_sftp()
with sftp.file(f"/tmp/{tm}.sourcelist", "w") as srcfile:
with open(os.path.join(util.get_data_root(), 'ubuntu', extra['apt']), 'r') as _:
srcfile.write(_.read())
with sftp.file(f"/tmp/{tm}.motd", "w") as _:
_.write(motdfmt.format(himsg = extra['himsg'], user = extra['user'], hostid = extra['hostid']))
with sftp.file(f"/tmp/{tm}.supervisord", "w") as _:
_.write(sprvsrfile)
with sftp.file(f"/tmp/{tm}.init", "w") as _:
_.write(init)
with sftp.file(f"/tmp/{tm}.dockerfile", "w") as _:
dockerfile = dckrtpl.format(content = extra['content'],
sourcelist = f"{tm}.sourcelist",
init = f"{tm}.init",
motd = f"{tm}.motd",
sprvsr = f"{tm}.supervisord",
user = f"{extra['user']}",
psdlen = 15)
_.write(dockerfile)
sftp.close()
docker_build_cmd = (f'{extra["dockerprog"]} build'
f' --tag {extra["hostid"]}:{extra["user"]}'
f' --file /tmp/{tm}.dockerfile /tmp')
print(f"[+] {docker_build_cmd}")
stdin, stdout, stderr = conn.exec_command(docker_build_cmd)
err = stdout.channel.recv_exit_status()
# Remove unused files
conn.exec_command(f'rm -rf /tmp/{tm}.*')
if err:
print("[x] Cannot run docker build!")
for line in stderr.readlines():
print(line.strip('\n'))
conn.close()
return False
else:
return True
def scp_from(ip, remote_path, user='******', password=None, local_path=""):
"""
@param path_to_file: file to copy
@param copy_location: place on localhost to place file
"""
ssh = SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
ssh.connect(ip, username=user, password=password, allow_agent=False)
sftp = ssh.open_sftp()
sftp.get(remote_path, local_path)
def scp_to(ip, local_path, user='******', password=None, remote_path=""):
"""
Send a file to a server
"""
ssh = SSHClient()
ssh.set_missing_host_key_policy(WarningPolicy())
ssh.connect(ip, username=user, password=password, allow_agent=False)
sftp = ssh.open_sftp()
sftp.put(local_path, remote_path)
def get_file_list(conf, ip, dir_name, port):
options = conf.options("file")
file_list = []
if ip in options:
dir_list = conf.get("file", ip).split('|')
if dir_name in dir_list:
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(ip, username = get_user(conf, ip), port = int(port))
return list_sort(ssh.open_sftp().listdir(dir_name))
def getSftp(self):
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
try:
client.connect(str(self.host.text()), username=str(self.username.text()), password=str(self.password.text()))
except SSHException: qApp.instance().showErrorMessage("Error", "The ssh session could not be established")
except AuthenticationException: qApp.instance().showErrorMessage("Error","Authentication failed")
except BadHostKeyException: qApp.instance().showErrorMessage("Error", "the server's host key could not be verified")
sftp = client.open_sftp()
sftp.get('/work/matrice', './matrice')
sftp.close()
client.close()
def __init__(self):
path = os.path.expanduser("~/.ssh/config")
cfg = SSHConfig()
cfg.parse(open(path))
ssh = SSHClient()
ssh.load_system_host_keys()
(host, params) = self._ssh_cfg2params(cfg)
ssh.connect(host, **params)
self._ssh = ssh
self._sftp = ssh.open_sftp()
def upload(local_file, remote_file):
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(HOST, PORT, serialno, key_filename=PRIVATE_KEY)
sftp = ssh.open_sftp()
try:
sftp.stat(remote_file)
except IOError:
print 'no exist' + remote_file
else:
sftp.remove(remote_file)
sftp.put(local_file, remote_file)
sftp.close()
ssh.close()
def upload(local_file, remote_file):
print('Upload Start ...')
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(HOST, PORT, USER, key_filename=PRIVATE_KEY)
sftp = ssh.open_sftp()
sftp.put(local_file, remote_file)
info = sftp.stat(remote_file)
print("file size: %s byte" % info.st_size)
sftp.close()
ssh.close()
print('Upload Finish')
def uploadSFTP(username,serverAddress,serverDir,texfn):
imgfn = texfn.with_suffix('.png')
print(f'Uploading {imgfn} to {serverAddress} {serverDir}')
ssh = SSHClient()
#ssh.set_missing_host_key_policy(AutoAddPolicy())
#ssh.load_host_keys(expanduser(join("~", ".ssh", "known_hosts")))
ssh.load_system_host_keys()
ssh.connect(serverAddress, username=username, password=getpass())
sftp = ssh.open_sftp()
sftp.put(imgfn, serverDir+imgfn.name,confirm=True) #note that destination filename MUST be included!
sftp.close()
ssh.close()
#NOTE: I didn't enable a return code from paramiko, if any is available
class Uploader:
"""
This class is used for uploading current script to remote server and getting information about remote server
"""
def __init__(self, host, login, ssh_key=None):
if host is None or len(host) < 4:
raise ApplicationException("Can't connect to server. Host name is required")
if login is None:
raise ApplicationException("Can't connect to server. User name is required")
if ssh_key is None:
raise ApplicationException("Can't connect to server because credentials hasn't been provided")
self.host = host
self.login = login
self.ssh_key = ssh_key
from paramiko import SSHClient, AutoAddPolicy
self.ssh_client = SSHClient()
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
self.ssh_client.connect(self.host, username=self.login, key_filename=self.ssh_key)
pass
def __get_arguments(self):
if self.ssh_key is not None:
return "-i " + self.ssh_key
raise ApplicationException("Credentials not found")
pass
def upload(self):
with closing(self.ssh_client.open_sftp()) as sender:
sender.put("./collector.py", remotepath="/tmp/collector.py")
pass
def prnt(self):
stdin, stdout, stderr = self.ssh_client.exec_command("python /tmp/collector.py -r")
stdin.flush()
data = stdout.read()
print("RESULTS:\n\n\n")
print(data)
pass
class watchfile(threading.Thread):
def __init__(self, ip, user, port, file_path, line_list):
threading.Thread.__init__(self)
print file_path
self.ssh = SSHClient()
self.ssh.set_missing_host_key_policy(AutoAddPolicy())
self.ssh.connect(ip, username = user, port = int(port))
sftp = self.ssh.open_sftp()
self.remote_file = sftp.open(file_path)
self.remote_file.seek(0, 2)
fsize = self.remote_file.tell()
self.remote_file.seek (max (fsize-2048, 0), 0)
self.stop_signal = 0
self.line_list = line_list
def __watch__(self):
while True:
new = self.remote_file.readline()
# Once all lines are read this just returns ''
# until the file changes and a new line appears
if self.stop_signal != 0:
break
if new:
yield new
else:
time.sleep(0.5)
def run(self):
for line in self.__watch__():
if self.stop_signal != 0:
break
line = line.strip()
self.line_list.append(line)
print line
def stop(self):
self.stop_signal = 1
time.sleep(0.5)
self.ssh.close()
def deploy(ip, username, password, arduino_sketch_file=None):
print("==== Deploying to {} ====".format(ip.ljust(15)))
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
try:
ssh.connect(ip, SSH_PORT, username, password)
# Create deploy path
cmd = 'mkdir -p {}'.format(DEPLOY_PATH)
(stdin, stdout, stderr) = ssh.exec_command(cmd)
if stdout.channel.recv_exit_status() != 0:
raise DeployException('Could not execute {}'.format(cmd))
# Copy all files
print("Copying files..")
sftp = ssh.open_sftp()
def copy_file(local_path, remote_path):
print('{} --> {}:{}'.format(local_path.ljust(30), ip, remote_path))
sftp.put(local_path, remote_path, confirm=True)
for local_path in DEPLOY_FILES:
filename = path.basename(local_path)
remote_path = DEPLOY_PATH + filename
copy_file(local_path, remote_path)
if arduino_sketch_file is not None:
copy_file(arduino_sketch_file, DEPLOY_PATH + 'sketch.ino')
print("Running setup script..")
# Run setup script, pipe output back?
channel = ssh.get_transport().open_session()
channel.exec_command(POST_DEPLOY_COMMAND)
while not channel.exit_status_ready():
if channel.recv_stderr_ready():
os.write(sys.stderr.fileno(), channel.recv_stderr(4096))
if channel.recv_ready():
os.write(sys.stdout.fileno(), channel.recv(4096))
except SSHException as e:
raise DeployException(e.value)
finally:
ssh.close()
def _fetchJarFile(self, prefix, suffix, dstdir, **sshargs):
(yield TaskOutput(u'ENTER', OutputType.NOTIFY))
sshcli = SSHClient()
sftpcli = None
code = 0
try:
mkdir_p(self.backup_dir)
if not (yield TaskOutput(u'Conntecting to %s ...' % sshargs['hostname'])):
raise CommandTerminated()
sshcli.set_missing_host_key_policy(AutoAddPolicy())
sshcli.connect(**sshargs)
if not (yield TaskOutput(u'Connected, fetchting file ...')):
raise CommandTerminated()
ret = sshcli.exec_command("cd {}; ls -1 {}*.{}".format(dstdir, prefix, suffix))
errstr = ret[2].read()
if errstr != '':
raise Exception(errstr)
sftpcli = sshcli.open_sftp()
filenum = 0
self.cboFetchedFile.clear()
for l in ret[1].readlines():
f = l.rstrip()
remotefile = os.path.join(dstdir, f).replace(os.sep, '/')
if not (yield TaskOutput(u'Fetchting %s ...' % f)):
raise CommandTerminated()
sftpcli.get(remotefile, os.path.join(self.backup_dir, f))
filenum += 1
self.cboFetchedFile.addItem(f)
if filenum > 1:
(yield TaskOutput(
u'Fetched %d Files, Please Check Your Settings' % filenum,
OutputType.WARN))
except CommandTerminated:
code = -2
(yield TaskOutput(u'Fetching Terminited', OutputType.WARN))
except Exception as ex:
code = -1
(yield TaskOutput(ex.message, OutputType.ERROR))
finally:
if sftpcli: sftpcli.close()
sshcli.close()
(yield TaskOutput(u'EXIT %d' % code, OutputType.NOTIFY))
def connect(login):
client = SSHClient()
args = parse(login)
if "identityfile" in args:
args["key_filename"] = args["identityfile"]
del args["identityfile"]
client.set_missing_host_key_policy(AutoAddPolicy())
try:
client.connect(**args)
except AuthenticationException:
args['password'] = getpass()
try:
client.connect(**args)
except AuthenticationException as e:
print e.message
exit(1)
return (client, client.open_sftp())
def run(args, logging):
"""
Relieves a remote file using SFTP
:param args: A NameSpace object with the arguments required
:param logging: A python logging object
:return: An exit code
"""
hostname = build_data.read(args.build_id, 'instance', 'public_ip_address')
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
logging.info('Connecting to remote host "{0}"...'.format(hostname))
client.connect(hostname, port=args.port, username=args.username,
key_filename=args.key_filename, timeout=args.timeout)
logging.info('Getting file "{0}"...'.format(args.remote_path))
sftp_client = client.open_sftp()
sftp_client.get(args.remote_path, args.local_path)
return 0, None, None
def _setup_node(node, clmap):
if node['is_master']:
script_body = clmap['master_script']
else:
script_body = clmap['slave_script']
ssh = SSHClient()
try:
_setup_ssh_connection(node['ip'], ssh)
sftp = ssh.open_sftp()
fl = sftp.file('/tmp/savanna-hadoop-init.sh', 'w')
fl.write(script_body)
fl.close()
sftp.chmod('/tmp/savanna-hadoop-init.sh', 0500)
ret = _open_channel_and_execute(ssh,
'/tmp/savanna-hadoop-init.sh '
'>> /tmp/savanna-hadoop-init.log 2>&1')
_ensure_zero(ret)
finally:
ssh.close()
def sync_public_key(host, port=22, username=None, password=None):
if LOCAL_USER_NAME == "root":
rsa_pub_path = "/root/.ssh/id_rsa.pub"
else:
rsa_pub_path = "/home/{user}/.ssh/id_rsa.pub".format(user=LOCAL_USER_NAME)
if os.path.exists(rsa_pub_path):
PUB_KEY = rsa_pub_path
else:
print("本地没有可用的SSH公钥，请使用ssh-ken-gen生成")
sys.exit(-1)
try:
client = SSHClient()
client.connect(hostname=host, username=username, password=password)
except SSHException, e:
print e
client.close()
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy())
client.connect(hostname=host, username=username, password=password)
sftp_client = client.open_sftp()
if username == "root":
remote_home = "/root"
remote_rsa_pub = "/root/.ssh/%s.pub" % (LOCAL_USER_NAME)
else:
remote_home = "/home/{username}".format(username=username)
remote_ssh_home = "{remote_home}/.ssh".format(remote_home=remote_home)
remote_rsa_pub = "{remote_ssh_home}/{local_user}.pub".format(
remote_ssh_home=remote_ssh_home, local_user=LOCAL_USER_NAME)
try:
sftp_client.put(PUB_KEY , remote_rsa_pub)
except Exception, e:
"""
if the remote host did have .ssh dirctory
"""
print e
def _uploadDiffs(self, dstdir, sshargs):
st = self.settings
app = self.cboApp.itemText(self.cboApp.currentIndex())
hdiff = dpath(app, 'hdiff')
sshcli = SSHClient()
sftpcli = None
code = 0
try:
if not (yield TaskOutput(u'Connecting to %s ...' % sshargs['hostname'])):
raise CommandTerminated()
sshcli.set_missing_host_key_policy(AutoAddPolicy())
sshcli.connect(**sshargs)
if not (yield TaskOutput(u'Connected, ready to upload ...')):
raise CommandTerminated()
ret = sshcli.exec_command("[ -d {0} ] && rm -rf {0}; mkdir -p {0}".format(dstdir))
errstr = ret[2].read()
if errstr != '':
raise Exception(errstr)
sftpcli = sshcli.open_sftp()
for f in os.listdir(hdiff):
if f.lower().endswith('.html'):
localfile = os.path.join(hdiff, f)
remotefile = os.path.join(dstdir, f).replace(os.sep, '/')
if not (yield TaskOutput(u'Uploading %s ...' % f)):
raise CommandTerminated()
sftpcli.put(localfile, remotefile)
except CommandTerminated:
code = -2
(yield TaskOutput(u'Uploading Terminited', OutputType.WARN))
except Exception as ex:
code = -1
(yield TaskOutput(ex.message, OutputType.ERROR))
finally:
if sftpcli: sftpcli.close()
if sshcli: sshcli.close()
(yield TaskOutput(u'EXIT %d' % code, OutputType.NOTIFY))
class SFTP(LoggingMixIn, Operations):
'''
A simple SFTP filesystem. Requires paramiko: http://www.lag.net/paramiko/
You need to be able to login to remote host without entering a password.
'''
def __init__(self, host, path='.'):
self.client = SSHClient()
self.client.load_system_host_keys()
self.client.connect(host)
self.sftp = self.client.open_sftp()
self.root = path
def chmod(self, path, mode):
return self.sftp.chmod(path, mode)
def chown(self, path, uid, gid):
return self.sftp.chown(path, uid, gid)
def create(self, path, mode):
f = self.sftp.open(path, 'w')
f.chmod(mode)
f.close()
return 0
def destroy(self, path):
self.sftp.close()
self.client.close()
def getattr(self, path, fh=None):
st = self.sftp.lstat(path)
return dict((key, getattr(st, key)) for key in ('st_atime', 'st_gid',
'st_mode', 'st_mtime', 'st_size', 'st_uid'))
def mkdir(self, path, mode):
return self.sftp.mkdir(path, mode)
def read(self, path, size, offset, fh):
f = self.sftp.open(path)
f.seek(offset, 0)
buf = f.read(size)
f.close()
return buf
def readdir(self, path, fh):
return ['.', '..'] + [name.encode('utf-8')
for name in self.sftp.listdir(path)]
def readlink(self, path):
return self.sftp.readlink(path)
def rename(self, old, new):
return self.sftp.rename(old, self.root + new)
def rmdir(self, path):
return self.sftp.rmdir(path)
def symlink(self, target, source):
return self.sftp.symlink(source, target)
def truncate(self, path, length, fh=None):
return self.sftp.truncate(path, length)
def unlink(self, path):
return self.sftp.unlink(path)
def utimens(self, path, times=None):
return self.sftp.utime(path, times)
def write(self, path, data, offset, fh):
f = self.sftp.open(path, 'r+')
f.seek(offset, 0)
f.write(data)
f.close()
return len(data)
def take_snapshot_using_export_policy(self, callback=None):
"""
Perform an immediate snapshot of the APIC configuration.
:param callback: Optional callback function that can be used to notify
applications when a snapshot is taken. Used by the
GUI to update the snapshots view when recurring
snapshots are taken.
"""
tag_name = time.strftime("%Y-%m-%d_%H.%M.%S", time.localtime())
url = '/api/node/mo/uni/fabric.json'
remote_path_payload = {"fileRemotePath": {"attributes": {
"remotePort": "22",
"name": "snapback",
"host": "%s" % self.session.ipaddr,
"remotePath": "/home/%s" % self.session.uid,
"protocol": "scp",
"userName": "******" % self.session.uid,
"userPasswd": "%s" % self.session.pwd},
"children": []}}
resp = self.session.push_to_apic(url, remote_path_payload)
export_policy_payload = {"configExportP":{"attributes":{"name":"snapback",
"adminSt":"triggered"},
"children":[{"configRsRemotePath":
{"attributes":{"tnFileRemotePathName":"snapback"},
"children":[]}}]}}
resp = self.session.push_to_apic(url, export_policy_payload)
if not resp.ok:
print resp, resp.text
time.sleep(10)
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(self.session.ipaddr, username=self.session.uid, password=self.session.pwd)
sftp = ssh.open_sftp()
sftp.chdir('/home/%s' % self.session.uid)
file_names = sftp.listdir()
for file_name in file_names:
if str(file_name).startswith('ce_snapback-'):
sftp.get('/home/' + self.session.uid + '/' + file_name, './' + file_name)
sftp.remove('/home/' + self.session.uid + '/' + file_name)
with tarfile.open(file_name, 'r:gz') as tfile:
tfile.extractall(self.repo_dir)
os.remove(file_name)
for json_filename in os.listdir(self.repo_dir):
print 'checking', json_filename
if json_filename.startswith('ce_snapback') and json_filename.endswith('.json'):
new_filename = 'snapshot_' + self.session.ipaddr + '_' + json_filename.rpartition('_')[2]
new_filename = os.path.join(self.repo_dir, new_filename)
print 'renaming', json_filename, 'to', new_filename
json_filename = os.path.join(self.repo_dir, json_filename)
with open(json_filename, 'r') as old_file:
config = json.loads(old_file.read())
with open(new_filename, 'w') as new_file:
new_file.write(json.dumps(config, indent=4, separators=(',', ':')))
os.remove(json_filename)
# Add the file to Git
self.repo.index.add([new_filename])
# Commit the files and tag with the timestamp
self.repo.index.commit(tag_name)
self.repo.git.tag(tag_name)
if callback:
callback()
