def recursive_list_filepaths(root_dirpath: Path, sftp_client_: paramiko.SFTPClient):
print('Go to {}'.format(str(root_dirpath)))
sftp_client_.chdir(str(root_dirpath))
for name in sftp_client_.listdir():
if name in ['.', '..']:
continue
new_path = root_dirpath / name
if is_dir(new_path, sftp_client_):
yield from recursive_list_filepaths(new_path, sftp_client_)
else:
yield Path(new_path)
def sftp_mkdir(client: paramiko.SFTPClient, dirpath: str):
"""Create a directory on the remote server.
----------
client: paramiko.SFTPClient
SFTP client.
dirpath: string
Path to the created directory on the remote server.
"""
try:
# Attempt to change into the directory. This will raise an error
# if the directory does not exist.
client.chdir(dirpath)
except IOError:
# Create directory if it does not exist.
client.mkdir(dirpath)
def discover_granules_sftp(self, sftp_client: paramiko.SFTPClient, path: str, file_reg_ex: str = None,
dir_reg_ex: str = None,
depth: int = 0):
"""
Discover granules on an SFTP provider
:param sftp_client: An initialized paramiko SFTP client
:param path: The location to begin discovery
:param file_reg_ex: Regular expression used to filter files
:param dir_reg_ex: Regular expression used to filter directories
:param depth: The positive number of levels to search down, will use the lesser of 3 or depth
:rtype: dictionary of discovered granules containing path, etag, and last modified dates
"""
directory_list = []
granule_dict = {}
rdg_logger.info(f'Exploring path {path} depth {depth}')
sftp_client.chdir(path)
for dir_file in sftp_client.listdir():
file_stat = sftp_client.stat(dir_file)
file_type = str(file_stat)[0]
if file_type == 'd' and (dir_reg_ex is None or re.search(dir_reg_ex, path)):
rdg_logger.info(f'Found directory: {dir_file}')
directory_list.append(dir_file)
elif file_reg_ex is None or re.search(file_reg_ex, dir_file):
populate_dict(granule_dict, f'{path}/{dir_file}', etag='N/A',
last_mod=file_stat.st_mtime, size=file_stat.st_size)
else:
rdg_logger.warning(f'Regex did not match dir_file: {dir_file}')
depth = min(abs(depth), 3)
if depth > 0:
for directory in directory_list:
granule_dict.update(
self.discover_granules_sftp(sftp_client, path=directory, file_reg_ex=file_reg_ex,
dir_reg_ex=dir_reg_ex, depth=(depth - 1))
)
sftp_client.chdir('../')
return granule_dict
def rcsv_listdir(self, sftp: paramiko.SFTPClient, path, dirs):
if stat.S_ISREG(sftp.stat('.').st_mode):
yield self.join_paths(sftp.getcwd(), sftp.stat('.').filename)
sftp.chdir('..')
return
sftp.chdir(path)
if not dirs:
yield from (self.join_paths(sftp.getcwd(), fn) for fn in sftp.listdir('.'))
sftp.chdir('..')
return
subdirs = sftp.listdir('.') if dirs[0] is None \
else set(sftp.listdir('.')).intersection(to_iter(dirs[0], ittype=iter))
for subdir in subdirs:
yield from self.rcsv_listdir(sftp, subdir, dirs[1:])
sftp.chdir('..')
def sftp_chdir(sftp: paramiko.SFTPClient, remote_directory: Path):
if remote_directory == '/':
sftp.chdir('/')
return
if remote_directory == '':
return
try:
sftp.chdir(str(remote_directory))
except IOError:
dirname, basename = os.path.split(str(remote_directory).rstrip('/'))
sftp_chdir(sftp, dirname)
sftp.mkdir(basename)
sftp.chdir(basename)
return True
