def walk(top, topdown=True):
# Quit if top directory doesn't exist.
if not exists(top):
return
# Get contents of top directory using either ifdh or posix.
dirs = []
files = []
if top.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
if debug:
print(
'*** Larbatch_posix: Walk directory tree for %s using ifdh.' %
top)
# Retrieve the contents of this directory using ifdh.
lines = larbatch_utilities.ifdh_ll(top, 1)
for line in lines:
words = line.split()
if len(words) > 5 and words[-1] != '__pycache__':
if words[0][0] == 'd':
dirs.append(words[-1])
else:
files.append(words[-1])
else:
if debug:
print(
'*** Larbatch_posix: Walk directory tree for %s using posix.' %
top)
contents = listdir(top)
for obj in contents:
if obj != '__pycache__':
if isdir(os.path.join(top, obj)):
dirs.append(obj)
else:
files.append(obj)
if topdown:
yield top, dirs, files
# Recursively descend into subdirectories.
for dir in dirs:
for result in walk(os.path.join(top, dir), topdown):
yield result
if not topdown:
yield top, dirs, files
# Done.
return
def stat(path):
result = None
if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
if debug:
print '*** Larbatch_posix: Stat %s using ifdh.' % path
# The only reliable way to get information about a directory is
# to get a listing of the parent directory.
npath = os.path.normpath(path) # Strip trailing '/'
name = os.path.basename(npath)
dir = os.path.dirname(npath)
lines = larbatch_utilities.ifdh_ll(dir, 1)
for line in lines:
words = line.split()
if len(words) > 5 and words[-1] == name:
# Found thie path. Fill result.
# Interpret mode string.
mode = larbatch_utilities.parse_mode(words[0])
# Get remaining fields.
nlinks = int(words[1])
size = int(words[4])
result = os.stat_result((
mode, # Mode
0, # Inode
0, # Device
nlinks, # Number of links
os.getuid(), # Uid
os.getgid(), # Gid
size, # Size
0, # Access time
0, # Mod time
0)) # Creation time
else:
if debug:
print '*** Larbatch_posix: Stat %s using posix.' % path
result = os.stat(path)
if result == None:
raise OSError, 'No such file or directory.'
# Done.
return result
def isdir(path):
result = False
# Optimizations for speed and to reduce hang risk by not stat'ing every file.
if path[-5:] == '.list' or \
path[-5:] == '.root' or \
path[-5:] == '.json' or \
path[-4:] == '.txt' or \
path[-4:] == '.fcl' or \
path[-4:] == '.out' or \
path[-4:] == '.err' or \
path[-3:] == '.sh' or \
path[-5:] == '.stat':
return False
if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
if debug:
print(
'*** Larbatch_posix: Check existence of directory %s using ifdh.'
% path)
# Make sure path exists before trying to determine if it is a directory.
if exists(path):
# The only reliable way to get information about a directory is
# to get a listing of the parent directory.
npath = os.path.normpath(path) # Strip trailing '/'
name = os.path.basename(npath)
dir = os.path.dirname(npath)
lines = larbatch_utilities.ifdh_ll(dir, 1)
for line in lines:
words = line.split()
if len(words) > 5 and words[-1] == name:
if words[0][0] == 'd':
result = True
else:
if debug:
print(
'*** Larbatch_posix: Check existence of directory %s using posix.'
% path)
result = os.path.isdir(path)
# Done.
return result
def rmtree(path):
if path.startswith('/pnfs/') and (prefer_grid or not pnfs_is_mounted):
if debug:
print('*** Larbatch_posix: Delete directoroy tree %s using ifdh.' %
path)
# Delete contents recursively.
lines = larbatch_utilities.ifdh_ll(path, 1)
for line in lines:
words = line.split()
if len(words) > 5:
if words[0][0] == 'd':
rmtree(os.path.join(path, words[-1]))
else:
remove(os.path.join(path, words[-1]))
# Directory should be empty when we get to here.
rmdir(path)
else:
if debug:
print(
'*** Larbatch_posix: Delete directoroy tree %s using posix.' %
path)
# Deleting a directory tree is a hang risk, especially, but not only, in dCache.
# Therefore, use the following procedure.
#
# 1. Rename directory to a random name (this can usually be done, even
# for undeletable directories).
#
# 2. Delete renamed directory in a subprocess. No need to wait for
# subprocess to finish, or check its exit status.
#shutil.rmtree(path)
npath = os.path.normpath(path) # Strip trailing '/'
newpath = npath + '_' + str(uuid.uuid4())
os.rename(npath, newpath)
os.system('rm -rf %s &' % newpath)
# Done
return