def remote_exists(sftp_client: paramiko.SFTPClient, f: str) -> bool:
try:
sftp_client.stat(f)
return True
except IOError:
return False
def process_directory(
self,
sftp: paramiko.SFTPClient,
remote_root: Path,
local_targetdir: Path,
entry: BackupEntry
):
remotedir = Path(entry.get_path())
self.info("\tProcessing folder: '{}'".format(remotedir))
remote_exists = True
try:
stat_remote = sftp.stat(str(remotedir))
except:
remote_exists = False
if not remote_exists:
self.info("\tPath '{}' doesn't exist".format(
remotedir
))
elif not stat.S_ISDIR(stat_remote.st_mode):
self.info("\t{}Path '{}' is not a directory".format(
remotedir
))
else:
self._process_directory(0, sftp, remote_root, remotedir, local_targetdir, entry)
self.info("Finished\n")
def rcsv_listdir(self, sftp: paramiko.SFTPClient, path, dirs):
if stat.S_ISREG(sftp.stat('.').st_mode):
yield self.join_paths(sftp.getcwd(), sftp.stat('.').filename)
sftp.chdir('..')
return
sftp.chdir(path)
if not dirs:
yield from (self.join_paths(sftp.getcwd(), fn) for fn in sftp.listdir('.'))
sftp.chdir('..')
return
subdirs = sftp.listdir('.') if dirs[0] is None \
else set(sftp.listdir('.')).intersection(to_iter(dirs[0], ittype=iter))
for subdir in subdirs:
yield from self.rcsv_listdir(sftp, subdir, dirs[1:])
sftp.chdir('..')
def compare_files(ftp: paramiko.SFTPClient, ftp_path: str, local_path: str):
"""Returns True if remote file is newer, else False."""
if not os.path.exists(local_path):
return True
ftp_path = ftp_path.replace('\\', '/')
file_date = round(os.path.getmtime(local_path), 0)
ftp_date = ftp.stat(f'{ftp_path}').st_mtime
if ftp_date > file_date: # Be careful there's no time delta
return True
return False
def upload_file(sftp: paramiko.SFTPClient, local_path,
remote_path) -> paramiko.SFTPClient:
local_all = files_of_dir(local_path)
for file in local_all:
real_remote = os.path.join(
remote_path,
file.replace(os.path.dirname(local_path),
'').lstrip('\\')).replace("\\", "/")
remote_p = os.path.dirname(real_remote)
# 远程创建文件夹
try:
stat = sftp.stat(remote_p)
except FileNotFoundError:
sftp.mkdir(remote_p)
stat = sftp.stat(remote_p)
if S_ISDIR(stat.st_mode):
# sftp 的 put命令，需要在 remotepath 中，指定文件的名字
# f.put(r'D:\Python.zip', r'/10.130.218.69 sftp (10.130.218.69)/nginx/home/nginx/webapps/Python.zip')
sftp.put(file, real_remote)
return sftp
def compare_get_files(task: Task, sftp_client: paramiko.SFTPClient, src: str,
dst: str) -> List[str]:
changed = []
if stat.S_ISREG(sftp_client.stat(src).st_mode):
# is a file
src_hash = get_dst_hash(task, src)
try:
dst_hash = get_src_hash(dst)
except IOError:
dst_hash = ""
if src_hash != dst_hash:
changed.append(dst)
else:
if os.path.exists(dst):
for f in sftp_client.listdir(src):
s = os.path.join(src, f)
d = os.path.join(dst, f)
changed.extend(compare_get_files(task, sftp_client, s, d))
else:
changed.append(dst)
return changed
def discover_granules_sftp(self, sftp_client: paramiko.SFTPClient, path: str, file_reg_ex: str = None,
dir_reg_ex: str = None,
depth: int = 0):
"""
Discover granules on an SFTP provider
:param sftp_client: An initialized paramiko SFTP client
:param path: The location to begin discovery
:param file_reg_ex: Regular expression used to filter files
:param dir_reg_ex: Regular expression used to filter directories
:param depth: The positive number of levels to search down, will use the lesser of 3 or depth
:rtype: dictionary of discovered granules containing path, etag, and last modified dates
"""
directory_list = []
granule_dict = {}
rdg_logger.info(f'Exploring path {path} depth {depth}')
sftp_client.chdir(path)
for dir_file in sftp_client.listdir():
file_stat = sftp_client.stat(dir_file)
file_type = str(file_stat)[0]
if file_type == 'd' and (dir_reg_ex is None or re.search(dir_reg_ex, path)):
rdg_logger.info(f'Found directory: {dir_file}')
directory_list.append(dir_file)
elif file_reg_ex is None or re.search(file_reg_ex, dir_file):
populate_dict(granule_dict, f'{path}/{dir_file}', etag='N/A',
last_mod=file_stat.st_mtime, size=file_stat.st_size)
else:
rdg_logger.warning(f'Regex did not match dir_file: {dir_file}')
depth = min(abs(depth), 3)
if depth > 0:
for directory in directory_list:
granule_dict.update(
self.discover_granules_sftp(sftp_client, path=directory, file_reg_ex=file_reg_ex,
dir_reg_ex=dir_reg_ex, depth=(depth - 1))
)
sftp_client.chdir('../')
return granule_dict
def isfile(sftp_client: paramiko.SFTPClient, remotepath: str):
try:
return stat.S_ISREG(sftp_client.stat(remotepath).st_mode)
except IOError: # no such file
return False
def is_dir(path_: Path, sftp_client_: paramiko.SFTPClient):
return stat.S_ISDIR(sftp_client_.stat(str(path_)).st_mode)
