def sftp_walk(self, remotepath): # Kindof a stripped down version of os.walk, implemented for # sftp. Tried running it flat without the yields, but it really # chokes on big directories. path = remotepath files = [] folders = [] for f in self.sftp.listdir_attr(remotepath): if S_ISDIR(f.st_mode): folders.append(f.filename) else: files.append(f.filename) #print (path,folders,files) yield path, folders, files for folder in folders: new_path = os.path.join(remotepath, folder) for x in self.sftp_walk(new_path): yield x
def _sftp_walk(self, remotepath): """ https://techtalkontv.wordpress.com/2016/11/05/python-pramiko-sftp-copydownload-all-files-in-a-folder-recursively-from-remote-server/ """ path = remotepath files = [] folders = [] for f in self._sftp.listdir_attr(remotepath): if S_ISDIR(f.st_mode): folders.append(f.filename) else: files.append(f.filename) if files: yield path, files for folder in folders: new_path = os.path.join(remotepath, folder) for x in self._sftp_walk(new_path): yield x
def walk_files(self, directory): from stat import S_ISLNK, S_ISDIR, S_ISREG self._sftp_connect() for entry in self._sftp.listdir_attr(directory): path = posixpath.join(directory, entry.filename) if S_ISLNK(entry.st_mode): path = self._sftp.readlink(directory) entry = self._sftp.stat(path) if S_ISDIR(entry.st_mode): for inner_path in self.walk_files(path): yield inner_path elif S_ISREG(entry.st_mode): yield path
def check_init_script(configurator, question, answer): try: from stat import ST_MODE, S_ISDIR if not os.path.lexists(answer): os.makedirs(answer) elif not S_ISDIR(os.stat(answer)[ST_MODE]): raise ValidationError( "Invalid path (%s): path must point to a directory not a file." % answer) with open(os.path.join(answer, 'plume'), 'w'): answer = os.path.realpath(answer) configurator.target_directory = answer return answer except OSError as e: raise ValidationError(e) except IOError as e: raise ValidationError(e)
def make_writable(path): try: mode = stat(path).st_mode if S_ISDIR(mode): chmod(path, S_IMODE(mode) | S_IWRITE | S_IEXEC) elif S_ISREG(mode) or S_ISLNK(mode): chmod(path, S_IMODE(mode) | S_IWRITE) else: log.debug("path cannot be made writable: %s", path) except Exception as e: eno = getattr(e, 'errno', None) if eno in (ENOENT, ): raise elif eno in (EACCES, EPERM): log.debug("tried make writable but failed: %s\n%r", path, e) else: log.error("Error making path writable: %s\n%r", path, e) raise
def _walk(self, remote_path): path=remote_path files=[] folders=[] for f in self.__sftp.listdir_attr(remote_path): if S_ISDIR(f.st_mode): folders.append(f.filename) else: files.append(f.filename) if files: yield path, files for folder in folders: new_path = os.path.join(remote_path, folder) for x in self._walk(new_path): yield x
def add_files(self, basePath, path, app): if self.debug: print(("add files " + path)) for f in os.listdir(path): pathname = os.path.join(path, f) mode = os.stat(pathname).st_mode if S_ISREG(mode): filename = os.path.join(os.path.relpath(path, basePath), f) if self.debug: print(("add " + pathname)) print(("name " + filename)) fileBuffer = open(pathname, 'rb').read() appFile = app.file.add() appFile.name = filename appFile.encoding = CLEARTEXT appFile.blob = fileBuffer elif S_ISDIR(mode): self.add_files(basePath, pathname, app)
def walk(self): #logger.debug('self path=%s, called path=%s', self.path, path) #npath = abspath_re.sub('', path) dirs = list() files = list() #npath = self.path_join(self.path) npath = self.path logger.debug('walking path %s', npath) for f in self.conn.listdir_attr(npath): if S_ISDIR(f.st_mode): dirs.append(f.filename) else: files.append(f.filename) yield self.path, dirs, files for d in dirs: _path = os.path.join(path, d) for f in self.walk(_path): yield f
def check_access_rights(top): for f in os.listdir(top): pathname = os.path.join(top, f) mode = os.stat(pathname).st_mode if S_ISDIR(mode): # directory, recurse into it check_access_rights(pathname) elif S_ISREG(mode): # file, check permissions permissions = oct(os.stat(pathname)[ST_MODE]) if PY3: self.assertEqual("0o100775", permissions) else: self.assertEqual("0100775", permissions) else: # unknown file type pass
def copystat(src, dst): """Copy stat bits from src to dst This should be used when shutil.copystat would be used on directories on win32 because win32 does not support utime() for directories. According to the official docs written by Microsoft, it returns ENOACCES if the supplied filename is a directory. Looks like a trainee implemented the function. """ if sys.platform == 'win32' and S_ISDIR(os.stat(dst)[ST_MODE]): if os.name == 'nt': st = os.stat(src) mode = S_IMODE(st[ST_MODE]) if hasattr(os, 'chmod'): os.chmod(dst, mode) # KEEP THIS ONE! #else: pass # we are on Win9x,ME - no chmod here else: shutil.copystat(src, dst)
def generateSolution(team, folder): print("Getting solutions for...", folder, team) solutions = os.listdir(folder) result = [] for solution in solutions: if S_ISDIR(os.stat("%s/%s" % (folder, solution)).st_mode): solution_page = open( "%s/%s_%s.md" % (solutionsFolder, team, solution), 'w') solution_page.write("---\n") solution_page.write("layout: solution\n") solution_page.write("team: %s\n" % (team, )) solution_page.write("sol: %s\n" % (solution, )) indexPreview = htmlPageIn("%s/%s" % (folder, solution)) if indexPreview: solution_page.write("demo: %s\n" % (indexPreview)) solution_page.write("---\n") readmeContent = getReadmeContent("%s/%s" % (folder, solution)) # copy folder try: shutil.copytree("%s/%s" % (folder, solution), "%s/%s_%s" % (solutionsFolder, team, solution)) except: pass if readmeContent: solution_page.write(readmeContent) else: solution_page.write(solution + "\n") solution_page.close() result.append("%s_%s" % (team, solution)) return result
def view_results(): dirpath = "output" entries = ( co2wui_fpath(dirpath, fn) for fn in os.listdir(co2wui_fpath(dirpath)) ) entries = ((os.stat(path), path) for path in entries) entries = ( (stat[ST_CTIME], path) for stat, path in entries if S_ISDIR(stat[ST_MODE]) ) results = [] for cdate, path in sorted(entries): dirname = osp.basename(path) output_files = [f.name for f in listdir_outputs("output", dirname)] summary = get_summary(dirname) outcome = "KO" if (summary is None or not summary or len(summary[0].keys()) <= 2) else "OK" results.append( { "datetime": time.ctime(cdate), "name": dirname, "files": output_files, "outcome": outcome, } ) running = False if ("active_pid" in session) and (session["active_pid"] is not None): running = True return render_template( "layout.html", action="view_results", data={ "breadcrumb": ["Co2mpas", _("View results")], "props": { "active": {"run": "active", "sync": "", "doc": "", "expert": ""} }, "results": reversed(results), "running": running, "texts": co2wui_texts, "globals": co2wui_globals, }, )
def walktree(self, remotepath, fcallback, dcallback, ucallback, recurse=True): '''recursively descend, depth first, the directory tree rooted at remotepath, calling discreet callback functions for each regular file, directory and unknown file type. :param str remotepath: root of remote directory to descend, use '.' to start at :attr:`.pwd` :param callable fcallback: callback function to invoke for a regular file. (form: ``func(str)``) :param callable dcallback: callback function to invoke for a directory. (form: ``func(str)``) :param callable ucallback: callback function to invoke for an unknown file type. (form: ``func(str)``) :param bool recurse: *Default: True* - should it recurse :returns: None :raises: ''' self._sftp_connect() for entry in self.listdir(remotepath): pathname = posixpath.join(remotepath, entry) mode = self._sftp.stat(pathname).st_mode if S_ISDIR(mode): # It's a directory, call the dcallback function dcallback(pathname) if recurse: # now, recurse into it self.walktree(pathname, fcallback, dcallback, ucallback) elif S_ISREG(mode): # It's a file, call the fcallback function fcallback(pathname) else: # Unknown file type ucallback(pathname)
def local_file_create(host_obj, directory_path, dir_node, filename, db): # type: (HostController, str, FileNode, str, SimpleDB) -> [(int, str)] # where (int, str): (FILE_CREATE, full_path) _log = get_mylog() _log.debug('Adding {} to filenode for the directory node {}'.format( filename, dir_node.name)) # print '\t\tAdding',filename,'to filenode for',dir_node.name file_pathname = os.path.join(directory_path, filename) file_stat = os.stat(file_pathname) file_modified = file_stat.st_mtime file_created = file_stat.st_ctime mode = file_stat.st_mode filenode = FileNode() db.session.add(filenode) filenode.name = filename filenode.created_on = datetime.utcfromtimestamp(file_created) filenode.last_modified = datetime.utcfromtimestamp(file_modified) # DO NOT try and set the new node's `cloud` setting. If that's set, then # we'll treat that node as a child of the cloud itself - as a child of # the host.models.Cloud. That's not what we want. # Appending the new filenode to the dir_node WILL work, because that will # append it to either the Cloud (if this is a child of the root) # or the FileNode correctly. dir_node.children.append(filenode) db.session.commit() # Theoretically, the private data should have already been loaded or # created. I guess the first time that the private data is generated, this # happens. Might actually cause the host to write the .nebs, then read it # right back in. # Not really the worst. if host_obj.is_private_data_file(file_pathname, filenode.cloud): host_obj.reload_private_data(filenode.cloud) updates = [(FILE_CREATE, file_pathname)] if S_ISDIR(mode): # It's a directory, recurse into it # use the directory's node as the new root. rec_updates = recursive_local_modifications_check( host_obj, file_pathname, filenode, db) updates.extend(rec_updates) db.session.commit() return updates
def __fetch_info(self): # lstat() does not follow sym links. try: info = os.lstat(self.__abs_path) is_symlink = True if S_ISLNK(info.st_mode) != 0 else False if Platform.isWindows: is_symlink = jaraco.windows.filesystem.is_reparse_point( self.__abs_path) dst_info = info if is_symlink: dst_info = os.stat(self.__abs_path) if info is not None: self.__dict.update({ 'exists': os.path.exists(self.__abs_path), 'size': info.st_size, 'r_size': 0, 'gid': info.st_gid, 'uid': info.st_uid, 'posix_perms': S_IMODE(info.st_mode), 'dir': S_ISDIR(dst_info.st_mode), 'create_date': info.st_ctime, 'mod_date': info.st_mtime, 'sym_link': is_symlink }) # if its a directory - collecting the size makes no sense, and in fact confuses the markup of checked/partially # modified scans - because changes to the posix perms can modify it? at least unit tests show that - anyway, we've # got no present use for directories with a size attribute. if self.is_dir: self.__dict.update({'size': 0}) else: # if its Mac, it might have the r_size and others if Platform.isMac and hasattr(info, 'r_size'): self.__dict.update({'r_size': info.r_size}) else: self.setDefaults() except OSError, err: # lookup on fs failed, fill in a blank directory logger.warn( "stat() on path '{0}' failed, using empty/blank defaults, error: {1}" .format(self.__abs_path, err)) self.setDefaults()
def _walk( self, directory: str ) -> Generator[Tuple[str, List[Any], List[str]], None, None]: dirs = [] nondirs = [] for entry in self.conn.listdir_attr(directory): if S_ISDIR(entry.st_mode or 0): dirs.append(entry.filename) else: nondirs.append(str(Path(directory).joinpath(entry.filename))) yield directory, dirs, nondirs # Recurse into sub-directories for dirname in dirs: new_path = str(Path(directory).joinpath(dirname)) yield from self._walk(new_path)
def sftp_tree(self, path='.'): main_tree = list() file_list = self.sftp.listdir_attr(path) for idx, entry in enumerate(file_list): mode = entry.st_mode filename = entry.filename if idx == len(file_list) - 1: start_str = '└─ ' sub_start_str = ' ' else: start_str = '├─ ' sub_start_str = '│ ' main_tree.append('{}{}'.format(start_str, filename)) if S_ISDIR(mode): sub_path = filename if path == '.' else '{}/{}'.format( path, filename) tree = self.sftp_sub_tree(sub_path, sub_start_str, 1) main_tree.extend(tree) return main_tree
def _tree_chunks(repo, tree, startofs): "Tree should be a sequence of (name, mode, hash) as per tree_decode()." assert (startofs >= 0) # name is the chunk's hex offset in the original file for mode, name, oid in _skip_chunks_before_offset(tree, startofs): ofs = int(name, 16) skipmore = startofs - ofs if skipmore < 0: skipmore = 0 it = repo.cat(hexlify(oid)) _, obj_t, size = next(it) data = b''.join(it) if S_ISDIR(mode): assert obj_t == b'tree' for b in _tree_chunks(repo, tree_decode(data), skipmore): yield b else: assert obj_t == b'blob' yield data[skipmore:]
def walktree(top): '''parcourt récursivement le dossier et renvoit une liste avec tous les fichiers normaux''' liste = [] for f in os.listdir(top): pathname = os.path.join(top, f) mode = os.stat(pathname).st_mode # give a code that # says whether it's a directory or a file if S_ISDIR(mode): # It's a directory, recurse into it result = walktree(pathname) for _ in result: liste.append(_) elif S_ISREG(mode): liste.append(pathname.split('/')[-1]) else: # Unknown file type, print a message print('Skipping %s' % pathname) return liste
def info(self, path, **kwargs): wpath = _as_unc_path(self.host, path) stats = smbclient.stat(wpath, **kwargs) if S_ISDIR(stats.st_mode): stype = "directory" elif S_ISLNK(stats.st_mode): stype = "link" else: stype = "file" res = { "name": path + "/" if stype == "directory" else path, "size": stats.st_size, "type": stype, "uid": stats.st_uid, "gid": stats.st_gid, "time": stats.st_atime, "mtime": stats.st_mtime, } return res
def sftp_walk(sftp, remotePath): #建立一个sftp客户端对象,通过ssh transport操作远程文件 files = [] folders = [] # Copy a remote file (remotePath) from the SFTP server to the local host try: for f in sftp.listdir_attr(remotePath): if S_ISDIR(f.st_mode): folders.append(f.filename) else: files.append(f.filename) yield remotePath, folders, files for folder in folders: new_path = os.path.join(remotePath, folder) for x in sftp_walk(sftp, new_path): yield x except (Exception) as e: print(e)
def __get_all_files_in_remote_dir(sftp, remote_dir): # 保存所有文件的列表 all_files = list() # 去掉路径字符串最后的字符'/',如果有的话 if remote_dir[-1] == '/': remote_dir = remote_dir[0:-1] # 获取当前指定目录下的所有目录及文件,包含属性值 files = sftp.listdir_attr(remote_dir) for x in files: # remote_dir目录中每一个文件或目录的完整路径 filename = remote_dir + '/' + x.filename # 如果是目录,则递归处理该目录,这里用到了stat库中的S_ISDIR方法,与linux中的宏的名字完全一致 if S_ISDIR(x.st_mode): all_files.extend(__get_all_files_in_remote_dir(sftp, filename)) else: all_files.append(filename) return all_files
def get_ftype_and_perm(mode): """ Returns a tuple of whether the file is: file(regular file)/dir/slink /char. spec/block spec/pipe/socket and the permission bits of the file. If it does not match any of the cases below (it will return "unknown" twice)""" if S_ISREG(mode): return "file", S_IMODE(mode) if S_ISDIR(mode): return "dir", S_IMODE(mode) if S_ISLNK(mode): return "slink", S_IMODE(mode) if S_ISCHR(mode): return "character special", S_IMODE(mode) if S_ISBLK(mode): return "block special", S_IMODE(mode) if S_ISFIFO(mode): return "pipe", S_IMODE(mode) if S_ISSOCK(mode): return "socket", S_IMODE(mode) return "unknown", "unknown"
def sftp_walk(remote_path): """Walks the remote path. Parameters: dir path on remote Returns: path, sub-dir, list of files """ path = remote_path files = [] folders = [] for file in SFTP.listdir_attr(remote_path): if S_ISDIR(file.st_mode): folders.append(file.filename) else: files.append(file.filename) if files: yield path, files for folder in folders: new_path = os.path.join(remote_path, folder) for n_file in sftp_walk(new_path): yield n_file
def _walk_dir(file_path, file_list=[]): for f in os.listdir(file_path): try: path_name = os.path.join(file_path, f) except UnicodeDecodeError: continue islink = os.path.islink(path_name) is_exists = os.path.exists(path_name) if not islink and is_exists: mode = os.stat(path_name).st_mode if S_ISDIR(mode): _walk_dir(path_name, file_list) elif S_ISREG(mode): size = os.stat(path_name).st_size file_list.append(size) else: #pass logging.info('invalid path: %s' % path_name)
def walk(self, remotepath): """Taken from https://gist.github.com/johnfink8/2190472 A stripped down version of os.walk implemented for sftp which yields tuples: (path, folders, files) """ path = remotepath files = [] folders = [] for f in self.listdir_attr(remotepath): if S_ISDIR(f.st_mode): folders.append(f.filename) else: # non-dir files.append(f.filename) yield path, folders, files for folder in folders: new_path = os.path.join(remotepath, folder) for x in self.walk(new_path): yield x
def sftp(files_from, files_to, mode, deleteAfterCopy=False): with dog_stats_api.timer('pearson.{0}'.format(mode), tags='sftp'): try: t = paramiko.Transport( (settings.PEARSON['SFTP_HOSTNAME'], 22)) t.connect(username=settings.PEARSON['SFTP_USERNAME'], password=settings.PEARSON['SFTP_PASSWORD']) sftp = paramiko.SFTPClient.from_transport(t) if mode == 'export': try: sftp.chdir(files_to) except IOError: raise CommandError( 'SFTP destination path does not exist: {}'. format(files_to)) for filename in os.listdir(files_from): sftp.put(files_from + '/' + filename, filename) if deleteAfterCopy: os.remove(os.path.join(files_from, filename)) else: try: sftp.chdir(files_from) except IOError: raise CommandError( 'SFTP source path does not exist: {}'.format( files_from)) for filename in sftp.listdir('.'): # skip subdirectories if not S_ISDIR(sftp.stat(filename).st_mode): sftp.get(filename, files_to + '/' + filename) # delete files from sftp server once they are # successfully pulled off: if deleteAfterCopy: sftp.remove(filename) except: dog_http_api.event('pearson {0}'.format(mode), 'sftp uploading failed', alert_type='error') raise finally: sftp.close() t.close()
def stat(_endpoint, filepath, _userid): '''Stat a file and returns (size, mtime) as well as other extended info. This method assumes that the given userid has access.''' try: tstart = time.time() statInfo = os.stat(_getfilepath(filepath)) tend = time.time() log.info('msg="Invoked stat" filepath="%s" elapsedTimems="%.1f"' % (_getfilepath(filepath), (tend - tstart) * 1000)) if S_ISDIR(statInfo.st_mode): raise IOError('Is a directory') return { 'inode': str(statInfo.st_ino), 'filepath': filepath, 'userid': str(statInfo.st_uid) + ':' + str(statInfo.st_gid), 'size': statInfo.st_size, 'mtime': statInfo.st_mtime } except (FileNotFoundError, PermissionError) as e: raise IOError(e)
def walk(self, root): ''' Упрощенный аналог os.walk. Для каждой итерации возвращает кортеж - (root, folders, files) Для всех folders, files возвращает полные пути на базе root ''' files = [] folders = [] for f in self.listdir_attr(root): if S_ISDIR(f.st_mode): folders.append(tools.join_unix(root, f.filename)) else: files.append(tools.join_unix(root, f.filename)) yield root, folders, files for folder in folders: new_root = tools.join_unix(root, folder).replace('\\', '/') for x in self.walk(new_root): yield x
def BuildTestList(): """Returns a list of all available system tests. Returns: A list of test_names, constructed from the set of directory names in the current directory. All directories containing the substring 'test' are included in the returned set. """ testing_directory = os.listdir('.') test_list = [] for test in testing_directory: mode = os.stat(test)[ST_MODE] if ('test' in test and test != 'unittest_data' and test != 'bitmap_test' and test != 'conditioner_test_data' and S_ISDIR(mode)): test_list += [test] return test_list