def rcsv_listdir(self, sftp: paramiko.SFTPClient, path, dirs): if stat.S_ISREG(sftp.stat('.').st_mode): yield self.join_paths(sftp.getcwd(), sftp.stat('.').filename) sftp.chdir('..') return sftp.chdir(path) if not dirs: yield from (self.join_paths(sftp.getcwd(), fn) for fn in sftp.listdir('.')) sftp.chdir('..') return subdirs = sftp.listdir('.') if dirs[0] is None \ else set(sftp.listdir('.')).intersection(to_iter(dirs[0], ittype=iter)) for subdir in subdirs: yield from self.rcsv_listdir(sftp, subdir, dirs[1:]) sftp.chdir('..')
def recursive_list_filepaths(root_dirpath: Path, sftp_client_: paramiko.SFTPClient): print('Go to {}'.format(str(root_dirpath))) sftp_client_.chdir(str(root_dirpath)) for name in sftp_client_.listdir(): if name in ['.', '..']: continue new_path = root_dirpath / name if is_dir(new_path, sftp_client_): yield from recursive_list_filepaths(new_path, sftp_client_) else: yield Path(new_path)
def empty(sftp: SFTPClient, path): if not SystemUtils.isdir(sftp, path): sftp.mkdir(path) files = sftp.listdir(path=path) for f in files: file_path = os.path.join(path, f) if SystemUtils.isdir(sftp, file_path): SystemUtils.empty(sftp, file_path) else: sftp.remove(file_path)
def compare_get_files(task: Task, sftp_client: paramiko.SFTPClient, src: str, dst: str) -> List[str]: changed = [] if stat.S_ISREG(sftp_client.stat(src).st_mode): # is a file src_hash = get_dst_hash(task, src) try: dst_hash = get_src_hash(dst) except IOError: dst_hash = "" if src_hash != dst_hash: changed.append(dst) else: if os.path.exists(dst): for f in sftp_client.listdir(src): s = os.path.join(src, f) d = os.path.join(dst, f) changed.extend(compare_get_files(task, sftp_client, s, d)) else: changed.append(dst) return changed
def discover_granules_sftp(self, sftp_client: paramiko.SFTPClient, path: str, file_reg_ex: str = None, dir_reg_ex: str = None, depth: int = 0): """ Discover granules on an SFTP provider :param sftp_client: An initialized paramiko SFTP client :param path: The location to begin discovery :param file_reg_ex: Regular expression used to filter files :param dir_reg_ex: Regular expression used to filter directories :param depth: The positive number of levels to search down, will use the lesser of 3 or depth :rtype: dictionary of discovered granules containing path, etag, and last modified dates """ directory_list = [] granule_dict = {} rdg_logger.info(f'Exploring path {path} depth {depth}') sftp_client.chdir(path) for dir_file in sftp_client.listdir(): file_stat = sftp_client.stat(dir_file) file_type = str(file_stat)[0] if file_type == 'd' and (dir_reg_ex is None or re.search(dir_reg_ex, path)): rdg_logger.info(f'Found directory: {dir_file}') directory_list.append(dir_file) elif file_reg_ex is None or re.search(file_reg_ex, dir_file): populate_dict(granule_dict, f'{path}/{dir_file}', etag='N/A', last_mod=file_stat.st_mtime, size=file_stat.st_size) else: rdg_logger.warning(f'Regex did not match dir_file: {dir_file}') depth = min(abs(depth), 3) if depth > 0: for directory in directory_list: granule_dict.update( self.discover_granules_sftp(sftp_client, path=directory, file_reg_ex=file_reg_ex, dir_reg_ex=dir_reg_ex, depth=(depth - 1)) ) sftp_client.chdir('../') return granule_dict
def _process_directory( self, level: int, sftp: paramiko.SFTPClient, remote_root: Path, remote_path: Path, local_targetdir: Path, entry: BackupEntry ): isroot = False options = entry.get_options() has_options = not is_empty_dict(options) path_rootindex = None recurse = False if has_options: if "recurse" in options and not self._check_option_ignored("recures"): recurse = options["recurse"] if "path_rootindex" in options and not self._check_option_ignored("path_rootindex"): path_rootindex = options["path_rootindex"] if len(remote_path.parents) == 0: isroot = True if isroot: localdir = local_targetdir else: if path_rootindex is not None and is_integer(path_rootindex): f = path_from_partindex(remote_path, path_rootindex) localdir = local_targetdir.joinpath(f) else: localdir = local_targetdir.joinpath(str(remote_path)[1:]) remote_filenode = None tabs2 = repeat("\t", level + 1) do_transfer = self._check_folder_with_options(remote_root, remote_path, options) if do_transfer is not None: self.info("{}Excluding '{}' due to json-folder-option {}".format( tabs2, remote_path, do_transfer )) return if not localdir.exists(): self.info("{}Creating parent folders '{}'".format(tabs2, localdir)) makedirs(str(localdir)) try: filelist = sftp.listdir(str(remote_path)) if len(filelist) > 0: for filenode in filelist: remote_filenode = remote_path.joinpath(filenode) remote_stat = sftp.lstat(str(remote_filenode)) if stat.S_ISDIR(remote_stat.st_mode): if recurse is True: self.info("\n{}Recursing into sub-directory '{}'".format( tabs2, remote_filenode )) self._process_directory( level + 1, sftp, remote_path, remote_filenode, local_targetdir, entry ) else: localfile = localdir.joinpath(filenode) self._download_file(level, sftp, remote_root, localfile, remote_filenode, entry) except Exception as e: if isinstance(e, PermissionError): self.info("{}PermissionError while listing contents of {}\n".format( tabs2, remote_filenode if remote_filenode is not None else remote_path )) elif isinstance(e, JobException): raise e else: raise JobException(e, 1)