def _ProcessNode(self): """Process a node in the directory tree. If the node is another directory, enumerate its contents and add it to the list of nodes to be processed in the future.""" filename, filetype = self.items.pop() try: # If the filesystem supports readdir d_type, then we will know if the node is # a file or a directory without doing any extra work. If it does not, we have # to do a stat. if filetype == 0: s = safestat.safestat(filename) if stat.S_ISDIR(s.st_mode): filetype = readdir.dirent.DT_DIR else: filetype = readdir.dirent.DT_REG # If we a directory, enumerate its contents and add them to the list of nodes # to be processed. if filetype == readdir.dirent.DT_DIR: for node in readdir.readdir(filename): if not node.d_name in (".",".."): fullname = os.path.join(filename, node.d_name) self.items.appendleft((fullname, node.d_type)) # Call the processing functions on the directory or file. self.ProcessDir(filename) else: self.ProcessFile(filename) except OSError as error: print "cannot access `%s':" % filename, print os.strerror(error.errno) return()
def _ProcessNode(self): """Process a node in the directory tree. If the node is another directory, enumerate its contents and add it to the list of nodes to be processed in the future.""" filename, filetype = self.items.pop() try: # If the filesystem supports readdir d_type, then we will know if the node is # a file or a directory without doing any extra work. If it does not, we have # to do a stat. if filetype == 0: s = safestat.safestat(filename) if stat.S_ISDIR(s.st_mode): filetype = readdir.dirent.DT_DIR else: filetype = readdir.dirent.DT_REG # If we a directory, enumerate its contents and add them to the list of nodes # to be processed. if filetype == readdir.dirent.DT_DIR: for node in readdir.readdir(filename): if not node.d_name in (".", ".."): fullname = os.path.join(filename, node.d_name) self.items.appendleft((fullname, node.d_type)) # Call the processing functions on the directory or file. self.ProcessDir(filename) else: self.ProcessFile(filename) except OSError as error: print "cannot access `%s':" % filename, print os.strerror(error.errno) return ()
def fastwalk(sourcedir, onerror=None, topdown=True): """Improved version of os.walk: generates a tuple of (sourcedir,[dirs], [files]). This version tries to use readdir to avoid expensive stat operations on lustre.""" dirlist = [] filelist = [] try: entries = readdir.readdir(sourcedir) except Exception as err: if onerror is not None: onerror(err) return for entry in entries: name = entry.d_name filetype = entry.d_type if not name in (".", ".."): if filetype == readdir.dirent.DT_UNKNOWN: fullname = os.path.join(sourcedir, name) mode = safestat.safestat(fullname).st_mode if stat.S_ISDIR(mode): filetype = readdir.dirent.DT_DIR else: filetype = readdir.dirent.DT_REG if filetype == readdir.dirent.DT_DIR: dirlist.append(name) else: filelist.append(name) if topdown: yield sourcedir, dirlist, filelist for d in dirlist: fullname = os.path.join(sourcedir, d) for entries in fastwalk(fullname, onerror, topdown): yield entries if not topdown: yield sourcedir, dirlist, filelist