def _ProcessNode(self):
"""Process a node in the directory tree. If the node is another directory,
enumerate its contents and add it to the list of nodes to be processed in the
future."""
filename, filetype = self.items.pop()
try:
# If the filesystem supports readdir d_type, then we will know if the node is
# a file or a directory without doing any extra work. If it does not, we have
# to do a stat.
if filetype == 0:
s = safestat.safestat(filename)
if stat.S_ISDIR(s.st_mode):
filetype = readdir.dirent.DT_DIR
else:
filetype = readdir.dirent.DT_REG
# If we a directory, enumerate its contents and add them to the list of nodes
# to be processed.
if filetype == readdir.dirent.DT_DIR:
for node in readdir.readdir(filename):
if not node.d_name in (".",".."):
fullname = os.path.join(filename, node.d_name)
self.items.appendleft((fullname, node.d_type))
# Call the processing functions on the directory or file.
self.ProcessDir(filename)
else:
self.ProcessFile(filename)
except OSError as error:
print "cannot access `%s':" % filename,
print os.strerror(error.errno)
return()
def _ProcessNode(self):
"""Process a node in the directory tree. If the node is another directory,
enumerate its contents and add it to the list of nodes to be processed in the
future."""
filename, filetype = self.items.pop()
try:
# If the filesystem supports readdir d_type, then we will know if the node is
# a file or a directory without doing any extra work. If it does not, we have
# to do a stat.
if filetype == 0:
s = safestat.safestat(filename)
if stat.S_ISDIR(s.st_mode):
filetype = readdir.dirent.DT_DIR
else:
filetype = readdir.dirent.DT_REG
# If we a directory, enumerate its contents and add them to the list of nodes
# to be processed.
if filetype == readdir.dirent.DT_DIR:
for node in readdir.readdir(filename):
if not node.d_name in (".", ".."):
fullname = os.path.join(filename, node.d_name)
self.items.appendleft((fullname, node.d_type))
# Call the processing functions on the directory or file.
self.ProcessDir(filename)
else:
self.ProcessFile(filename)
except OSError as error:
print "cannot access `%s':" % filename,
print os.strerror(error.errno)
return ()
def fastwalk(sourcedir, onerror=None, topdown=True):
"""Improved version of os.walk: generates a tuple of (sourcedir,[dirs],
[files]). This version tries to use readdir to avoid expensive stat
operations on lustre."""
dirlist = []
filelist = []
try:
entries = readdir.readdir(sourcedir)
except Exception as err:
if onerror is not None:
onerror(err)
return
for entry in entries:
name = entry.d_name
filetype = entry.d_type
if not name in (".", ".."):
if filetype == readdir.dirent.DT_UNKNOWN:
fullname = os.path.join(sourcedir, name)
mode = safestat.safestat(fullname).st_mode
if stat.S_ISDIR(mode):
filetype = readdir.dirent.DT_DIR
else:
filetype = readdir.dirent.DT_REG
if filetype == readdir.dirent.DT_DIR:
dirlist.append(name)
else:
filelist.append(name)
if topdown:
yield sourcedir, dirlist, filelist
for d in dirlist:
fullname = os.path.join(sourcedir, d)
for entries in fastwalk(fullname, onerror, topdown):
yield entries
if not topdown:
yield sourcedir, dirlist, filelist
