def recursive_list_filepaths(root_dirpath: Path, sftp_client_: paramiko.SFTPClient): print('Go to {}'.format(str(root_dirpath))) sftp_client_.chdir(str(root_dirpath)) for name in sftp_client_.listdir(): if name in ['.', '..']: continue new_path = root_dirpath / name if is_dir(new_path, sftp_client_): yield from recursive_list_filepaths(new_path, sftp_client_) else: yield Path(new_path)
def sftp_mkdir(client: paramiko.SFTPClient, dirpath: str): """Create a directory on the remote server. ---------- client: paramiko.SFTPClient SFTP client. dirpath: string Path to the created directory on the remote server. """ try: # Attempt to change into the directory. This will raise an error # if the directory does not exist. client.chdir(dirpath) except IOError: # Create directory if it does not exist. client.mkdir(dirpath)
def discover_granules_sftp(self, sftp_client: paramiko.SFTPClient, path: str, file_reg_ex: str = None, dir_reg_ex: str = None, depth: int = 0): """ Discover granules on an SFTP provider :param sftp_client: An initialized paramiko SFTP client :param path: The location to begin discovery :param file_reg_ex: Regular expression used to filter files :param dir_reg_ex: Regular expression used to filter directories :param depth: The positive number of levels to search down, will use the lesser of 3 or depth :rtype: dictionary of discovered granules containing path, etag, and last modified dates """ directory_list = [] granule_dict = {} rdg_logger.info(f'Exploring path {path} depth {depth}') sftp_client.chdir(path) for dir_file in sftp_client.listdir(): file_stat = sftp_client.stat(dir_file) file_type = str(file_stat)[0] if file_type == 'd' and (dir_reg_ex is None or re.search(dir_reg_ex, path)): rdg_logger.info(f'Found directory: {dir_file}') directory_list.append(dir_file) elif file_reg_ex is None or re.search(file_reg_ex, dir_file): populate_dict(granule_dict, f'{path}/{dir_file}', etag='N/A', last_mod=file_stat.st_mtime, size=file_stat.st_size) else: rdg_logger.warning(f'Regex did not match dir_file: {dir_file}') depth = min(abs(depth), 3) if depth > 0: for directory in directory_list: granule_dict.update( self.discover_granules_sftp(sftp_client, path=directory, file_reg_ex=file_reg_ex, dir_reg_ex=dir_reg_ex, depth=(depth - 1)) ) sftp_client.chdir('../') return granule_dict
def rcsv_listdir(self, sftp: paramiko.SFTPClient, path, dirs): if stat.S_ISREG(sftp.stat('.').st_mode): yield self.join_paths(sftp.getcwd(), sftp.stat('.').filename) sftp.chdir('..') return sftp.chdir(path) if not dirs: yield from (self.join_paths(sftp.getcwd(), fn) for fn in sftp.listdir('.')) sftp.chdir('..') return subdirs = sftp.listdir('.') if dirs[0] is None \ else set(sftp.listdir('.')).intersection(to_iter(dirs[0], ittype=iter)) for subdir in subdirs: yield from self.rcsv_listdir(sftp, subdir, dirs[1:]) sftp.chdir('..')
def sftp_chdir(sftp: paramiko.SFTPClient, remote_directory: Path): if remote_directory == '/': sftp.chdir('/') return if remote_directory == '': return try: sftp.chdir(str(remote_directory)) except IOError: dirname, basename = os.path.split(str(remote_directory).rstrip('/')) sftp_chdir(sftp, dirname) sftp.mkdir(basename) sftp.chdir(basename) return True