def rcsv_listdir(self, sftp: paramiko.SFTPClient, path, dirs):
if stat.S_ISREG(sftp.stat('.').st_mode):
yield self.join_paths(sftp.getcwd(), sftp.stat('.').filename)
sftp.chdir('..')
return
sftp.chdir(path)
if not dirs:
yield from (self.join_paths(sftp.getcwd(), fn) for fn in sftp.listdir('.'))
sftp.chdir('..')
return
subdirs = sftp.listdir('.') if dirs[0] is None \
else set(sftp.listdir('.')).intersection(to_iter(dirs[0], ittype=iter))
for subdir in subdirs:
yield from self.rcsv_listdir(sftp, subdir, dirs[1:])
sftp.chdir('..')
def recursive_list_filepaths(root_dirpath: Path, sftp_client_: paramiko.SFTPClient):
print('Go to {}'.format(str(root_dirpath)))
sftp_client_.chdir(str(root_dirpath))
for name in sftp_client_.listdir():
if name in ['.', '..']:
continue
new_path = root_dirpath / name
if is_dir(new_path, sftp_client_):
yield from recursive_list_filepaths(new_path, sftp_client_)
else:
yield Path(new_path)
def empty(sftp: SFTPClient, path):
if not SystemUtils.isdir(sftp, path):
sftp.mkdir(path)
files = sftp.listdir(path=path)
for f in files:
file_path = os.path.join(path, f)
if SystemUtils.isdir(sftp, file_path):
SystemUtils.empty(sftp, file_path)
else:
sftp.remove(file_path)
def compare_get_files(task: Task, sftp_client: paramiko.SFTPClient, src: str,
dst: str) -> List[str]:
changed = []
if stat.S_ISREG(sftp_client.stat(src).st_mode):
# is a file
src_hash = get_dst_hash(task, src)
try:
dst_hash = get_src_hash(dst)
except IOError:
dst_hash = ""
if src_hash != dst_hash:
changed.append(dst)
else:
if os.path.exists(dst):
for f in sftp_client.listdir(src):
s = os.path.join(src, f)
d = os.path.join(dst, f)
changed.extend(compare_get_files(task, sftp_client, s, d))
else:
changed.append(dst)
return changed
def discover_granules_sftp(self, sftp_client: paramiko.SFTPClient, path: str, file_reg_ex: str = None,
dir_reg_ex: str = None,
depth: int = 0):
"""
Discover granules on an SFTP provider
:param sftp_client: An initialized paramiko SFTP client
:param path: The location to begin discovery
:param file_reg_ex: Regular expression used to filter files
:param dir_reg_ex: Regular expression used to filter directories
:param depth: The positive number of levels to search down, will use the lesser of 3 or depth
:rtype: dictionary of discovered granules containing path, etag, and last modified dates
"""
directory_list = []
granule_dict = {}
rdg_logger.info(f'Exploring path {path} depth {depth}')
sftp_client.chdir(path)
for dir_file in sftp_client.listdir():
file_stat = sftp_client.stat(dir_file)
file_type = str(file_stat)[0]
if file_type == 'd' and (dir_reg_ex is None or re.search(dir_reg_ex, path)):
rdg_logger.info(f'Found directory: {dir_file}')
directory_list.append(dir_file)
elif file_reg_ex is None or re.search(file_reg_ex, dir_file):
populate_dict(granule_dict, f'{path}/{dir_file}', etag='N/A',
last_mod=file_stat.st_mtime, size=file_stat.st_size)
else:
rdg_logger.warning(f'Regex did not match dir_file: {dir_file}')
depth = min(abs(depth), 3)
if depth > 0:
for directory in directory_list:
granule_dict.update(
self.discover_granules_sftp(sftp_client, path=directory, file_reg_ex=file_reg_ex,
dir_reg_ex=dir_reg_ex, depth=(depth - 1))
)
sftp_client.chdir('../')
return granule_dict
def _process_directory(
self,
level: int,
sftp: paramiko.SFTPClient,
remote_root: Path,
remote_path: Path,
local_targetdir: Path,
entry: BackupEntry
):
isroot = False
options = entry.get_options()
has_options = not is_empty_dict(options)
path_rootindex = None
recurse = False
if has_options:
if "recurse" in options and not self._check_option_ignored("recures"):
recurse = options["recurse"]
if "path_rootindex" in options and not self._check_option_ignored("path_rootindex"):
path_rootindex = options["path_rootindex"]
if len(remote_path.parents) == 0:
isroot = True
if isroot:
localdir = local_targetdir
else:
if path_rootindex is not None and is_integer(path_rootindex):
f = path_from_partindex(remote_path, path_rootindex)
localdir = local_targetdir.joinpath(f)
else:
localdir = local_targetdir.joinpath(str(remote_path)[1:])
remote_filenode = None
tabs2 = repeat("\t", level + 1)
do_transfer = self._check_folder_with_options(remote_root, remote_path, options)
if do_transfer is not None:
self.info("{}Excluding '{}' due to json-folder-option {}".format(
tabs2,
remote_path,
do_transfer
))
return
if not localdir.exists():
self.info("{}Creating parent folders '{}'".format(tabs2, localdir))
makedirs(str(localdir))
try:
filelist = sftp.listdir(str(remote_path))
if len(filelist) > 0:
for filenode in filelist:
remote_filenode = remote_path.joinpath(filenode)
remote_stat = sftp.lstat(str(remote_filenode))
if stat.S_ISDIR(remote_stat.st_mode):
if recurse is True:
self.info("\n{}Recursing into sub-directory '{}'".format(
tabs2, remote_filenode
))
self._process_directory(
level + 1, sftp,
remote_path,
remote_filenode,
local_targetdir,
entry
)
else:
localfile = localdir.joinpath(filenode)
self._download_file(level, sftp, remote_root, localfile, remote_filenode, entry)
except Exception as e:
if isinstance(e, PermissionError):
self.info("{}PermissionError while listing contents of {}\n".format(
tabs2,
remote_filenode if remote_filenode is not None else remote_path
))
elif isinstance(e, JobException):
raise e
else:
raise JobException(e, 1)
