def __getStorageDirectoryContents(self, lfnDir, storageElement):
""" Obtians the contents of the supplied directory on the storage
"""
gLogger.info('Obtaining the contents for %s directories at %s' %
(len(lfnDir), storageElement))
se = StorageElement(storageElement)
res = se.exists(lfnDir)
if not res['OK']:
gLogger.error("Failed to obtain existance of directories",
res['Message'])
return res
for directory, error in res['Value']['Failed'].items():
gLogger.error('Failed to determine existance of directory',
'%s %s' % (directory, error))
if res['Value']['Failed']:
return S_ERROR('Failed to determine existance of directory')
directoryExists = res['Value']['Successful']
activeDirs = []
for directory in sorted(directoryExists):
exists = directoryExists[directory]
if exists:
activeDirs.append(directory)
allFiles = {}
while len(activeDirs) > 0:
currentDir = activeDirs[0]
res = se.listDirectory(currentDir)
activeDirs.remove(currentDir)
if not res['OK']:
gLogger.error('Failed to get directory contents',
res['Message'])
return res
elif currentDir in res['Value']['Failed']:
gLogger.error(
'Failed to get directory contents',
'%s %s' % (currentDir, res['Value']['Failed'][currentDir]))
return S_ERROR(res['Value']['Failed'][currentDir])
else:
dirContents = res['Value']['Successful'][currentDir]
activeDirs.extend(
se.getLFNFromURL(dirContents['SubDirs']).get(
'Value', {}).get('Successful', []))
fileURLMetadata = dirContents['Files']
fileMetadata = {}
res = se.getLFNFromURL(fileURLMetadata)
if not res['OK']:
gLogger.error('Failed to get directory content LFNs',
res['Message'])
return res
for url, error in res['Value']['Failed'].items():
gLogger.error("Failed to get LFN for URL",
"%s %s" % (url, error))
if res['Value']['Failed']:
return S_ERROR("Failed to get LFNs for PFNs")
urlLfns = res['Value']['Successful']
for urlLfn, lfn in urlLfns.items():
fileMetadata[lfn] = fileURLMetadata[urlLfn]
allFiles.update(fileMetadata)
zeroSizeFiles = []
for lfn in sorted(allFiles):
if os.path.basename(lfn) == 'dirac_directory':
allFiles.pop(lfn)
else:
metadata = allFiles[lfn]
if metadata['Size'] == 0:
zeroSizeFiles.append(
(lfn, 'deprecatedUrl', storageElement, 'PFNZeroSize'))
if zeroSizeFiles:
self.__reportProblematicReplicas(zeroSizeFiles, storageElement,
'PFNZeroSize')
gLogger.info('Obtained at total of %s files for directories at %s' %
(len(allFiles), storageElement))
return S_OK(allFiles)
def getStorageDirectoryContents(self, lfnDir, storageElement):
""" This takes the supplied lfn directories and recursively obtains the files in the supplied storage element
"""
gLogger.info('Obtaining the contents for %s directories at %s' %
(len(lfnDir), storageElement))
se = StorageElement(storageElement)
res = se.exists(lfnDir)
if not res['OK']:
gLogger.error(
"Failed to obtain existance of directories", res['Message'])
return res
for directory, error in res['Value']['Failed'].iteritems():
gLogger.error('Failed to determine existance of directory',
'%s %s' % (directory, error))
if res['Value']['Failed']:
return S_ERROR(errno.ENOENT, 'Failed to determine existance of directory')
directoryExists = res['Value']['Successful']
activeDirs = []
for directory in sorted(directoryExists):
exists = directoryExists[directory]
if exists:
activeDirs.append(directory)
allFiles = {}
while len(activeDirs) > 0:
currentDir = activeDirs[0]
res = se.listDirectory(currentDir)
activeDirs.remove(currentDir)
if not res['OK']:
gLogger.error('Failed to get directory contents', res['Message'])
return res
elif currentDir in res['Value']['Failed']:
gLogger.error('Failed to get directory contents', '%s %s' %
(currentDir, res['Value']['Failed'][currentDir]))
return S_ERROR(errno.ENOENT, res['Value']['Failed'][currentDir])
else:
dirContents = res['Value']['Successful'][currentDir]
activeDirs.extend(se.getLFNFromURL(dirContents['SubDirs']).get(
'Value', {}).get('Successful', []))
fileURLMetadata = dirContents['Files']
fileMetadata = {}
res = se.getLFNFromURL(fileURLMetadata)
if not res['OK']:
gLogger.error('Failed to get directory content LFNs', res['Message'])
return res
for url, error in res['Value']['Failed'].iteritems():
gLogger.error("Failed to get LFN for URL", "%s %s" % (url, error))
if res['Value']['Failed']:
return S_ERROR(errno.ENOENT, "Failed to get LFNs for PFNs")
urlLfns = res['Value']['Successful']
for urlLfn, lfn in urlLfns.iteritems():
fileMetadata[lfn] = fileURLMetadata[urlLfn]
allFiles.update(fileMetadata)
zeroSizeFiles = []
for lfn in sorted(allFiles):
if os.path.basename(lfn) == 'dirac_directory':
allFiles.pop(lfn)
else:
metadata = allFiles[lfn]
if not metadata['Size']:
zeroSizeFiles.append(
(lfn, 'deprecatedUrl', storageElement, 'PFNZeroSize'))
if zeroSizeFiles:
self.dic.reportProblematicReplicas(
zeroSizeFiles, storageElement, 'PFNZeroSize')
gLogger.info('Obtained at total of %s files for directories at %s' %
(len(allFiles), storageElement))
return S_OK(allFiles)
