def _sftp_rm(path: str, sftp: SFTPClient) -> None:
files = sftp.listdir(path)
for f in files:
filepath = f"{path}/{f}"
try:
sftp.remove(filepath)
except IOError:
_sftp_rm(filepath, sftp)
try:
sftp.rmdir(path)
except Exception:
pass
def sftpclient(sftpserver):
transport = Transport((sftpserver.host, sftpserver.port))
transport.connect(username="******", password="******")
sftpclient = SFTPClient.from_transport(transport)
yield sftpclient
sftpclient.close()
transport.close()
def __enter__(self):
assert self.sftp_clients is None, 'the context is already started'
sftp_clients = {}
for server in self.servers:
try:
transport = Transport((server.host, server.port))
transport.connect(username=server.user, pkey=self.old_key)
except (OSError, SSHException) as e:
for t, _, __ in sftp_clients.values():
t.close()
l = logging.getLogger(__name__ + '.TwoPhaseRenewal.__enter__')
l.exception(
'An exception rise during master key renewal '
'(%s -> %s, server: %s@%s:%d): %s',
get_key_fingerprint(self.old_key),
get_key_fingerprint(self.new_key),
server.user, server.host, server.port, str(e)
)
raise
except socket.gaierror as e:
raise ConnectionError(
'failed to connect: {0!s}\n{1!s}'.format(server, e)
) from e
sftp_client = SFTPClient.from_transport(transport)
authorized_keys = AuthorizedKeyList(sftp_client)
sftp_clients[server] = transport, sftp_client, authorized_keys
authorized_keys.append(self.new_key)
self.sftp_clients = sftp_clients
return self.servers
def get_files(self, sub_path, pattern='*'):
def progress_function(item):
if item:
return 'Parsing {0}'.format(item)
with SSHClient() as client:
client.set_missing_host_key_policy(WarningPolicy())
client.load_system_host_keys()
client.connect(**self.connect_settings)
with SFTPClient.from_transport(client.get_transport()) as ftpclient:
self.ftpclient = ftpclient
files = sorted(filter(lambda x: fnmatch(x, pattern), ftpclient.listdir(os.path.join(self.root, sub_path))))
with click.progressbar(files,
show_eta=False,
label='Parsing {0} logs from ssh'.format(len(files)),
item_show_func=progress_function) as progressbar:
for filename in progressbar:
path = os.path.join(self.root, sub_path, filename)
size = ftpclient.lstat(path).st_size
key = self.keys.log_file(filename='ssh://' + path)
prev = self.r.get(key)
if prev is None:
pos = 0
else:
pos = int(prev)
if size > pos:
if progressbar.is_hidden:
print('Reading {filename} from offset {pos}'.format(filename=path, pos=pos))
yield path, pos
self.r.set(key, size)
def connect_sftp(self, username, password, host, port):
sock = socket.socket()
try:
sock.connect((host, port))
except socket.error as e:
self._raise_connection_error(host, port=port, orig_error=e)
return SFTPClient(SocketAsChannelAdapter(sock))
def connect_sftp(connect_args):
"""Create an SFTP client."""
client = SSHClient()
client.set_missing_host_key_policy(AutoAddPolicy)
client.connect("localhost",
port=8022,
look_for_keys=False,
allow_agent=False,
**connect_args)
sftp = SFTPClient.from_transport(client.get_transport())
def rmdir(path, delete_root=True):
for f in sftp.listdir_attr(path=path):
childpath = join(path, f.filename)
if S_ISDIR(f.st_mode):
rmdir(childpath)
else:
sftp.remove(childpath)
if delete_root:
sftp.rmdir(path)
# Delete any files left over from previous tests :(
rmdir("/", delete_root=False)
return sftp
def upload_file(host, port, usr, psw, local_path, remote_path):
file_count = 0
print('-' * 50 + '\n')
print('Start uploading files')
transport = Transport((host, port))
transport.connect(username=usr, password=psw)
sftp = SFTPClient.from_transport(transport)
for file_name in listdir(local_path):
local_file = local_path + '\\' + file_name
remote_file = remote_path + '/' + file_name
if file_name.split('.')[-1] == 'css':
new_css = cssmini(open(local_file, 'r').read())
with open('tmp.css', 'w') as tmp_css:
tmp_css.write(new_css)
tmp_css.close()
sftp.put(cur_path + tmp_css.name, remote_file)
remove(cur_path + tmp_css.name)
elif file_name.split('.')[-1] == 'js':
new_js = jsmini(open(local_file, 'r', encoding='utf-8').read())
with open('tmp.js', 'w', encoding='utf-8') as tmp_js:
tmp_js.write(new_js)
tmp_js.close()
sftp.put(cur_path + tmp_js.name, remote_file)
remove(cur_path + tmp_js.name)
else:
sftp.put(local_file, remote_file)
print(file_name + 'Upload completed')
file_count += 1
transport.close()
print('All files have been uploaded, Connection has been closed, Number of files：{}'.format(file_count))
print('-' * 50 + '\n')
def connect_sftp(self, username, password, host, port):
try:
argv = self._get_vendor_specific_argv(username, host, port,
subsystem='sftp')
sock = self._connect(argv)
return SFTPClient(SocketAsChannelAdapter(sock))
except _ssh_connection_errors as e:
self._raise_connection_error(host, port=port, orig_error=e)
class LoopbackVendor(SSHVendor):
"""SSH "vendor" that connects over a plain TCP socket, not SSH."""
def connect_sftp(self, username, password, host, port):
sock = socket.socket()
try:
sock.connect((host, port))
except socket.error, e:
self._raise_connection_error(host, port=port, orig_error=e)
return SFTPClient(SocketAsChannelAdapter(sock))
def _create_copy_task(self, conn, hostname, direction, localpath,
remotepath):
chan = conn.get_transport()
sftp_client = SFTPClient.from_transport(chan)
task = self._pool.spawn(
self._sftp_transfer, direction, sftp_client, localpath, remotepath)
task._channel = chan
task._hostname = hostname
task._sftp_client = sftp_client
task._stdout = StringIO()
task._stderr = StringIO()
return task
def copy_worker(client, sftp, remote_paths, local_paths):
i = 1
for remote, local in zip(remote_paths, local_paths):
if i % 500 == 0:
print('{0}/{1} copied over {2}'.format(i, len(remote_paths),
current_thread()))
try:
sftp.get(remote, local)
except SSHException:
client.construct_client()
sftp = SFTPClient.from_transport(client.client.get_transport())
sftp.get(remote, local)
i = i + 1
def get_remote_file(ssh_config: SshClientConfig, local_path: Path, remote_path: Path):
"""Use SFTP to copy a file from a remote server to the local server.
:param ssh_config: Configuration of the SSH session used for SFTP
:param local_path: Destination path of the file being transferred
:param remote_path: Source path of the file being transferred
"""
ssh_transport = Transport((ssh_config.remote_server, int(ssh_config.ssh_port)))
ssh_key = RSAKey.from_private_key_file("/home/mycroft/.ssh/id_rsa")
ssh_transport.connect(hostkey=None, username=ssh_config.remote_user, pkey=ssh_key)
sftp_client = SFTPClient.from_transport(ssh_transport)
sftp_client.get(str(remote_path), str(local_path))
sftp_client.close()
ssh_transport.close()
def fx_authorized_sftp(fx_sftpd, fx_authorized_keys):
port, (thread, path, ev) = fx_sftpd.popitem()
thread.start()
key = RSAKey.generate(1024)
dot_ssh = path.mkdir('.ssh')
with dot_ssh.join('authorized_keys').open('w') as f:
print(format_openssh_pubkey(key), file=f)
for authorized_key in fx_authorized_keys:
print(format_openssh_pubkey(authorized_key), file=f)
transport = Transport(('127.0.0.1', port))
transport.connect(username='******', pkey=key)
sftp_client = SFTPClient.from_transport(transport)
yield sftp_client, path, [key] + fx_authorized_keys
sftp_client.close()
transport.close()
def main():
t = Transport(('127.0.0.1', 2222))
t.connect(None, 'foo', 'pass')
sftp = SFTPClient.from_transport(t)
loc = 'upload'
items_1 = []
items_2 = []
files = []
for item in sftp.listdir(loc):
items_1.append(item)
for item in sftp.listdir_iter(loc):
items_2.append(item.filename)
if stat.S_IFMT(item.st_mode) != stat.S_IFDIR:
files.append(item)
assert items_1 == items_2
print(f'{len(items_2)} items listed but only {len(files)} files')
def get_sftp_session():
"""Create SFTP Session based on env variable
:return: SFTP client connection
:rtype: SFTPClient
"""
conn = None
try:
transport = paramiko.Transport(host)
transport.connect(username=user, password=password)
conn = SFTPClient.from_transport(transport)
conn.chdir(working_dir)
print 'SFTP connection created'
except Exception as e:
print e.message
pass
return conn
def __enter__(self):
assert self.sftp_clients is None, 'the context is already started'
sftp_clients = {}
for server in self.servers:
try:
transport = Transport((server.host, server.port))
transport.connect(pkey=self.old_key)
except SSHException:
for t, _, __ in sftp_clients.values():
t.close()
raise
sftp_client = SFTPClient.from_transport(transport)
authorized_keys = AuthorizedKeyList(sftp_client)
sftp_clients[server] = transport, sftp_client, authorized_keys
authorized_keys.append(self.new_key)
self.sftp_clients = sftp_clients
return self.servers
def __enter__(self):
assert self.sftp_clients is None, "the context is already started"
sftp_clients = {}
for server in self.servers:
try:
transport = Transport((server.host, server.port))
transport.connect(pkey=self.old_key)
except SSHException:
for t, _, __ in sftp_clients.values():
t.close()
raise
sftp_client = SFTPClient.from_transport(transport)
authorized_keys = AuthorizedKeyList(sftp_client)
sftp_clients[server] = transport, sftp_client, authorized_keys
authorized_keys.append(self.new_key)
self.sftp_clients = sftp_clients
return self.servers
def get_sftp_session():
"""Create SFTP Session based on env variable
:return: SFTP client connection
:rtype: SFTPClient
"""
conn = None
try:
transport = paramiko.Transport(host)
transport.connect(username=user, password=password)
conn = SFTPClient.from_transport(transport)
conn.chdir(working_dir)
print 'SFTP connection created'
except Exception as e:
print e.message
pass
return conn
def __connect(self):
hostKey = self.__getHostKey()
if self.__transport:
self.__transport.close()
self.__transport = Transport( (self.__host, self.__port) )
if self._usePassword:
password = self.__password
if not password:
password = self.__promptPassword()
self.__transport.connect(hostKey, self.__username, password)
else:
privateKey = self.__loadKey()
self.__transport.connect(hostKey, self.__username, pkey=privateKey)
self.__sftp = SFTPClient.from_transport(self.__transport)
return self
def authorize(public_keys: collections.abc.Set,
master_key: PKey,
remote: Remote,
timeout: datetime.timedelta) -> datetime.datetime:
"""Make an one-time authorization to the ``remote``, and then revokes
it when ``timeout`` reaches soon.
:param public_keys: the set of public keys (:class:`paramiko.pkey.PKey`)
to authorize
:type public_keys: :class:`collections.abc.Set`
:param master_key: the master key (*not owner's key*)
:type master_key: :class:`paramiko.pkey.PKey`
:param remote: a remote to grant access permission
:type remote: :class:`~.remote.Remote`
:param timeout: the time an authorization keeps alive
:type timeout: :class:`datetime.timedelta`
:return: the expiration time
:rtype: :class:`datetime.datetime`
"""
transport = Transport((remote.host, remote.port))
transport.connect(username=remote.user, pkey=master_key)
try:
sftp_client = SFTPClient.from_transport(transport)
try:
authorized_keys = AuthorizedKeyList(sftp_client)
authorized_keys.extend(public_keys)
except:
sftp_client.close()
raise
except:
transport.close()
raise
def rollback():
time.sleep(timeout.total_seconds())
authorized_keys[:] = [master_key]
sftp_client.close()
transport.close()
timer = threading.Thread(target=rollback)
expires_at = datetime.datetime.now(datetime.timezone.utc) + timeout
timer.start()
return expires_at
def put_file(self, filename):
try:
self.future = TracebackFuture()
transport = self.client.get_transport()
self.sftp_client = SFTPClient.from_transport(transport)
self.remote = self.sftp_client.open(filename, 'wb')
self.local = open(filename, 'rb')
# self.remote.setblocking(0)
# _buf = self.local.read(100)
# self.remote.write(_buf)
self.io_loop.add_handler(self.sftp_client.sock, self._handle_write,
self.io_loop.WRITE | self.io_loop.ERROR)
return self.future
except Exception as e:
print(e)
pass
def run_task(self):
# this task has to run before configuration task
# copy base image
# make copy of base image for specific VNF
# generate bootstrap conf
# copy boostrap conf
# make bootstrap iso
# Done
_vnf = Vnf()
_configurator = Configuration()
datavars = self.sample_device.deviceConfigData
if datavars:
for vnf in datavars['device']['vnfs']:
self.logger.info(
Tools.create_log_msg(self.task_name,
self.sample_device.deviceSerial,
logmsg.VNFSTAGE_SETUP_DIR))
sftp = SFTPClient.from_transport(
self.sample_device.deviceConnection._conn._session.
_transport)
_vnf.mkdir_p(sftp, vnf['bootstrap_remote_dir'] + vnf['name'])
# Copy base image and boostrap conf
try:
with SCP(self.sample_device.deviceConnection,
progress=False) as scp:
self.update_task_state(
new_task_state=c.TASK_STATE_PROGRESS,
task_state_message=logmsg.VNFSTAGE_CP_BASE_IMG.
format(vnf['base_img'],
self.sample_device.deviceIP,
vnf['base_img_path']))
self.logger.info(
Tools.create_log_msg(
self.task_name,
self.sample_device.deviceSerial,
logmsg.VNFSTAGE_CP_BASE_IMG.format(
vnf['base_img'],
self.sample_device.deviceIP,
vnf['base_img_path'])))
scp.put('images/' + vnf['base_img'],
vnf['base_img_path'])
self.logger.info(
Tools.create_log_msg(
self.task_name,
self.sample_device.deviceSerial,
logmsg.VNFSTAGE_MAKE_CP_BASE_IMG_OK.format(
vnf['name'])))
self.logger.info(
Tools.create_log_msg(
self.task_name,
self.sample_device.deviceSerial,
logmsg.VNFSTAGE_GEN_BOOTSTRAP.format(
vnf['name'])))
except OSError as ose:
self.update_task_state(
new_task_state=c.TASK_STATE_FAILED,
task_state_message=logmsg.VNFSTAGE_CP_ERR.format(
vnf['name'], ose.strerror, ose.filename))
self.logger.info(
Tools.create_log_msg(
self.task_name, self.sample_device.deviceSerial,
logmsg.VNFSTAGE_CP_ERR.format(
vnf['name'], ose.strerror, ose.filename)))
return
vnf_conf_file = _configurator.prepare_vnf_boostrap_config(
serialnumber=vnf['deviceID'],
grp_cfg=self.grp_cfg,
vnf_type=vnf['vnf_type'])
if vnf_conf_file is not None:
full_path = self.grp_cfg.TASKS.Provision.Configuration.ConfigFileHistory + vnf_conf_file
with SCP(self.sample_device.deviceConnection,
progress=False) as scp:
self.update_task_state(
new_task_state=c.TASK_STATE_PROGRESS,
task_state_message=logmsg.VNFSTAGE_COPY_BOOTSTRAP.
format(
full_path, self.sample_device.deviceIP,
vnf['bootstrap_remote_dir'] + vnf['name'] +
'/'))
self.logger.info(
Tools.create_log_msg(
self.task_name,
self.sample_device.deviceSerial,
logmsg.VNFSTAGE_COPY_BOOTSTRAP.format(
full_path, self.sample_device.deviceIP,
vnf['bootstrap_remote_dir'] + vnf['name'] +
'/')))
try:
scp.put(
full_path, vnf['bootstrap_remote_dir'] +
vnf['name'] + '/' + 'juniper.conf')
self.logger.info(
Tools.create_log_msg(
self.task_name,
self.sample_device.deviceSerial,
logmsg.VNFSTAGE_COPY_BOOTSTRAP_OK.format(
vnf['name'])))
except OSError as ose:
self.update_task_state(
new_task_state=c.TASK_STATE_FAILED,
task_state_message=logmsg.VNFSTAGE_CP_ERR.
format(vnf['name'], ose.strerror,
ose.filename))
self.logger.info(
Tools.create_log_msg(
self.task_name,
self.sample_device.deviceSerial,
logmsg.VNFSTAGE_CP_ERR.format(
vnf['name'], ose.strerror,
ose.filename)))
return
# Make copy of base image
self.update_task_state(
new_task_state=c.TASK_STATE_PROGRESS,
task_state_message=logmsg.VNFSTAGE_MAKE_CP_BASE_IMG.
format(vnf['name']))
self.logger.info(
Tools.create_log_msg(
self.task_name, self.sample_device.deviceSerial,
logmsg.VNFSTAGE_MAKE_CP_BASE_IMG.format(
vnf['name'])))
req1 = '{0} {1}{2} {3}{4}{5}'.format(
'file copy', vnf['base_img_path'], vnf['base_img'],
vnf['base_img_path'], vnf['name'], '.qcow2')
self.sample_device.deviceConnection.cli(command=req1,
format='text',
warning=False)
self.logger.info(
Tools.create_log_msg(
self.task_name, self.sample_device.deviceSerial,
logmsg.VNFSTAGE_MAKE_CP_BASE_IMG_OK.format(
vnf['name'])))
# Genisoimage
self.update_task_state(
new_task_state=c.TASK_STATE_PROGRESS,
task_state_message=logmsg.VNFSTAGE_GEN_BOOTSTRAP.
format(vnf['name']))
self.logger.info(
Tools.create_log_msg(
self.task_name, self.sample_device.deviceSerial,
logmsg.VNFSTAGE_GEN_BOOTSTRAP.format(vnf['name'])))
req2 = '{0} {1}{2}{3}{4} {5}{6}{7}{8}{9}'.format(
'request genisoimage', vnf['bootstrap_remote_dir'],
vnf['name'], '/', 'juniper.conf',
vnf['bootstrap_remote_dir'], vnf['name'], '/',
vnf['name'], '.iso')
self.sample_device.deviceConnection.cli(command=req2,
format='text',
warning=False)
self.update_task_state(
new_task_state=c.TASK_STATE_PROGRESS,
task_state_message=logmsg.VNFSTAGE_BOOSTRAP_OK.format(
vnf['name']))
self.logger.info(
Tools.create_log_msg(
self.task_name, self.sample_device.deviceSerial,
logmsg.VNFSTAGE_BOOSTRAP_OK.format(vnf['name'])))
else:
self.update_task_state(
new_task_state=c.TASK_STATE_FAILED,
task_state_message=c.TASK_STATE_MSG_FAILED)
return
self.update_task_state(
new_task_state=c.TASK_STATE_DONE,
task_state_message=c.TASK_STATE_MSG_DONE)
else:
self.update_task_state(
new_task_state=c.TASK_STATE_FAILED,
task_state_message=logmsg.VNFSTAGE_DEV_CONF_READ_ERR)
self.logger.info(
Tools.create_log_msg(self.task_name,
self.sample_device.deviceSerial,
logmsg.VNFSTAGE_DEV_CONF_READ_ERR))
return
format(time() - start_time))
## NOTE
# This creates the ~10 SFTP objects that can be used for either single or multithreading
# the I/O jobs
client = Client(USERNAME,
SERVER_URL,
KEY_PATH,
PASSPHRASE,
optional_args=optional_client_build_args)
MAX_THREADS = 10
print('Attempting to open {0} channels'.format(MAX_THREADS))
sftps = []
for i in range(MAX_THREADS):
try:
sftps.append(SFTPClient.from_transport(client.client.get_transport()))
except ChannelException:
break
image_paths = get_all_image_paths(client, BANISHMENT_PATH)
with open('image_paths.log', 'w') as f:
for path in image_paths:
f.write(path + "\n")
print("All image paths on the server have been logged. {0} seconds so far.".
format(time() - start_time))
base_url = WORDPRESS_SITE_URL[(WORDPRESS_SITE_URL.find("//") + 2):]
unreferenced = get_unreferenced_image_paths(image_paths, image_urls, base_url,
YEAR_MONTH_IMAGES_ONLY)
with open('images_to_delete.log', 'w') as f:
for path in unreferenced:
def create_tmp_dir(self, prefix=''):
scp = SFTPClient.from_transport(self.ssh_client.get_transport())
dirpath = path_join('/tmp', prefix + str(int(time.time() * 1000)))
scp.mkdir(dirpath)
return dirpath
def copy(self, src, dst):
scp = SFTPClient.from_transport(self.ssh_client.get_transport())
scp.put(src, dst)
