def __removeStorageDirectory(self, directory, storageElement):
""" wipe out all contents from :directory: at :storageElement:
:param self: self reference
:param str directory: path
:param str storageElement: SE name
"""
self.log.info('Removing the contents of %s at %s' %
(directory, storageElement))
se = StorageElement(storageElement)
res = returnSingleResult(se.exists(directory))
if not res['OK']:
self.log.error("Failed to obtain existance of directory",
res['Message'])
return res
exists = res['Value']
if not exists:
self.log.info("The directory %s does not exist at %s " %
(directory, storageElement))
return S_OK()
res = returnSingleResult(se.removeDirectory(directory, recursive=True))
if not res['OK']:
self.log.error("Failed to remove storage directory",
res['Message'])
return res
self.log.info(
"Successfully removed %d files from %s at %s" %
(res['Value']['FilesRemoved'], directory, storageElement))
return S_OK()
def resolveLFNZeroReplicas( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolves the LFNZeroReplicas prognosis
"""
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult( self.fc.getReplicas( lfn, allStatus = True ) )
if res['OK'] and res['Value']:
gLogger.info( "LFNZeroReplicas file (%d) found to have replicas" % fileID )
else:
gLogger.info( "LFNZeroReplicas file (%d) does not have replicas. Checking storage..." % fileID )
pfnsFound = False
for storageElementName in sortList( gConfig.getValue( 'Resources/StorageElementGroups/Tier1_MC_M-DST', [] ) ):
res = self.__getStoragePathExists( [lfn], storageElementName )
if res['Value'].has_key( lfn ):
gLogger.info( "LFNZeroReplicas file (%d) found storage file at %s" % ( fileID, storageElementName ) )
pfn = res['Value'][lfn]
self.__reportProblematicReplicas( [( lfn, pfn, storageElementName, 'PFNNotRegistered' )], storageElementName, 'PFNNotRegistered' )
pfnsFound = True
if not pfnsFound:
gLogger.info( "LFNZeroReplicas file (%d) did not have storage files. Removing..." % fileID )
res = returnSingleResult( self.fc.removeFile( lfn ) )
if not res['OK']:
gLogger.error( res['Message'] )
# Increment the number of retries for this file
self.server.incrementProblematicRetry( fileID )
return res
gLogger.info( "LFNZeroReplicas file (%d) removed from catalog" % fileID )
# If we get here the problem is solved so we can update the integrityDB
return self.__updateCompletedFiles( 'LFNZeroReplicas', fileID )
def resolveBKReplicaNo(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the BKReplicaNo prognosis
"""
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult(self.fc.exists(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
# If the file exists in the catalog
if not res['Value']:
return self.__updateCompletedFiles('BKReplicaNo', fileID)
gLogger.info("BKReplicaNo file (%d) found to exist in the catalog" %
fileID)
# and has available replicas
res = returnSingleResult(self.fc.getCatalogReplicas(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if not res['Value']:
gLogger.info("BKReplicaNo file (%d) found to have no replicas" %
fileID)
return self.changeProblematicPrognosis(fileID, 'LFNZeroReplicas')
gLogger.info("BKReplicaNo file (%d) found to have replicas" % fileID)
res = returnSingleResult(
FileCatalog(catalogs=['BookkeepingDB']).addFile(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
return self.__updateCompletedFiles('BKReplicaNo', fileID)
def test_getFile(self):
print('\n\n#########################################################'
'################\n\n\t\t\tGet file test\n')
destinationFilePath = '%s/testFile.%s' % (self.destDirectory,
time.time())
# pfnForLfnRes = returnSingleResult( self.storageElement.getURL( destinationFilePath ) )
# destinationPfn = pfnForLfnRes['Value']
fileDict = {destinationFilePath: self.localSourceFile}
putFileRes = returnSingleResult(self.storageElement.putFile(fileDict))
# Now get a local copy of the file
getFileRes = returnSingleResult(
self.storageElement.getFile(destinationFilePath))
# Now remove the destination file
removeFileRes = returnSingleResult(
self.storageElement.removeFile(destinationFilePath))
# Clean up the local mess
os.remove(os.path.basename(destinationFilePath))
# Check that the put was done correctly
self.assertTrue(putFileRes['OK'])
self.assertTrue(putFileRes['Value'])
self.assertEqual(putFileRes['Value'], self.localFileSize)
# Check that we got the file correctly
self.assertTrue(getFileRes['OK'])
self.assertEqual(getFileRes['Value'], self.localFileSize)
# Check that the removal was done correctly
self.assertTrue(removeFileRes['OK'])
self.assertTrue(removeFileRes['Value'])
def test_getURL( self ):
print '\n\n#########################################################################\n\n\t\tGet access url test\n'
destinationFilePath = '%s/testFile.%s' % ( self.destDirectory, time.time() )
# pfnForLfnRes = returnSingleResult( self.storageElement.getURL( destinationFilePath ) )
#destinationPfn = pfnForLfnRes['Value']
fileDict = {destinationFilePath:self.localSourceFile}
putFileRes = returnSingleResult( self.storageElement.putFile( fileDict ) )
# Get a transfer url for the file
getTurlRes = self.storageElement.getURL( destinationFilePath, protocol = 'dips' )
# Remove the destination file
removeFileRes = returnSingleResult( self.storageElement.removeFile( destinationFilePath ) )
# Get missing turl res
getMissingTurlRes = self.storageElement.getURL( destinationFilePath, protocol = 'dips' )
# Check that the put was done correctly
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value'] )
self.assertEqual( putFileRes['Value'], self.localFileSize )
# Check that we can get the tURL properly
self.assert_( getTurlRes['OK'] )
self.assert_( getTurlRes['Value'] )
self.assert_( type( getTurlRes['Value'] ) == types.DictType )
self.assert_( type( getTurlRes['Value']['Successful'][destinationFilePath] ) in types.StringTypes )
# Check that the removal was done correctly
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value'] )
def test_getURL(self):
print('\n\n#########################################################'
'################\n\n\t\tGet access url test\n')
destinationFilePath = '%s/testFile.%s' % (self.destDirectory,
time.time())
# pfnForLfnRes = returnSingleResult( self.storageElement.getURL( destinationFilePath ) )
# destinationPfn = pfnForLfnRes['Value']
fileDict = {destinationFilePath: self.localSourceFile}
putFileRes = returnSingleResult(self.storageElement.putFile(fileDict))
# Get a transfer url for the file
getTurlRes = self.storageElement.getURL(destinationFilePath,
protocol='dips')
# Remove the destination file
removeFileRes = returnSingleResult(
self.storageElement.removeFile(destinationFilePath))
# Get missing turl res
getMissingTurlRes = self.storageElement.getURL(destinationFilePath,
protocol='dips')
# Check that the put was done correctly
self.assertTrue(putFileRes['OK'])
self.assertTrue(putFileRes['Value'])
self.assertEqual(putFileRes['Value'], self.localFileSize)
# Check that we can get the tURL properly
self.assertTrue(getTurlRes['OK'])
self.assertTrue(getTurlRes['Value'])
self.assertTrue(isinstance(getTurlRes['Value'], dict))
self.assertTrue(
type(getTurlRes['Value']['Successful'][destinationFilePath]) in
six.string_types)
# Check that the removal was done correctly
self.assertTrue(removeFileRes['OK'])
self.assertTrue(removeFileRes['Value'])
def __removeStorageDirectory( self, directory, storageElement ):
""" wipe out all contents from :directory: at :storageElement:
:param self: self reference
:param str directory: path
:param str storageElement: SE name
"""
self.log.info( 'Removing the contents of %s at %s' % ( directory, storageElement ) )
se = StorageElement( storageElement )
res = returnSingleResult( se.exists( directory ) )
if not res['OK']:
self.log.error( "Failed to obtain existance of directory", res['Message'] )
return res
exists = res['Value']
if not exists:
self.log.info( "The directory %s does not exist at %s " % ( directory, storageElement ) )
return S_OK()
res = returnSingleResult( se.removeDirectory( directory, recursive = True ) )
if not res['OK']:
self.log.error( "Failed to remove storage directory", res['Message'] )
return res
self.log.info( "Successfully removed %d files from %s at %s" % ( res['Value']['FilesRemoved'],
directory,
storageElement ) )
return S_OK()
def test_getFileSize( self ):
print '\n\n#########################################################################\n\n\t\t\tGet file size test\n'
destinationFilePath = '%s/testFile.%s' % ( self.destDirectory, time.time() )
# pfnForLfnRes = returnSingleResult( self.storageElement.getURL( destinationFilePath ) )
#destinationPfn = pfnForLfnRes['Value']
fileDict = {destinationFilePath:self.localSourceFile}
putFileRes = returnSingleResult( self.storageElement.putFile( fileDict ) )
# Get the file metadata
getFileSizeRes = returnSingleResult( self.storageElement.getFileSize( destinationFilePath ) )
# Now remove the destination file
removeFileRes = returnSingleResult( self.storageElement.removeFile( destinationFilePath ) )
# Get metadata for a removed file
getMissingFileSizeRes = returnSingleResult( self.storageElement.getFileSize( destinationFilePath ) )
# Check directories are handled properly
destinationDir = os.path.dirname( destinationFilePath )
directorySizeRes = returnSingleResult( self.storageElement.getFileSize( destinationDir ) )
# Check that the put was done correctly
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value'] )
self.assertEqual( putFileRes['Value'], self.localFileSize )
# Check that the metadata was done correctly
self.assert_( getFileSizeRes['OK'] )
self.assertEqual( getFileSizeRes['Value'], self.localFileSize )
# Check that the removal was done correctly
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value'] )
# Check the get metadata for non existant file
self.assertFalse( getMissingFileSizeRes['OK'] )
expectedError = "File does not exist"
self.assert_( expectedError in getMissingFileSizeRes['Message'] )
# Check that metadata operation with a directory
self.assertFalse( directorySizeRes['OK'] )
expectedError = "Supplied path is not a file"
self.assert_( expectedError in directorySizeRes['Message'] )
def test_exists( self ):
print '\n\n#########################################################################\n\n\t\t\tExists test\n'
destinationFilePath = '%s/testFile.%s' % ( self.destDirectory, time.time() )
# pfnForLfnRes = self.storageElement.getURL( destinationFilePath )
#destinationPfn = pfnForLfnRes['Value']['Successful'].values()[0]
fileDict = {destinationFilePath:self.localSourceFile}
putFileRes = returnSingleResult( self.storageElement.putFile( fileDict ) )
# File exists
existsRes = returnSingleResult( self.storageElement.exists( destinationFilePath ) )
# Now remove the destination file
removeFileRes = returnSingleResult( self.storageElement.removeFile( destinationFilePath ) )
# Check removed file
missingExistsRes = returnSingleResult( self.storageElement.exists( destinationFilePath ) )
# Check directories are handled properly
destinationDir = os.path.dirname( destinationFilePath )
directoryExistsRes = returnSingleResult( self.storageElement.exists( destinationDir ) )
# Check that the put was done correctly
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value'] )
self.assertEqual( putFileRes['Value'], self.localFileSize )
# Check that we checked the file correctly
self.assert_( existsRes['OK'] )
self.assert_( existsRes['Value'] )
# Check that the removal was done correctly
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value'] )
# Check the exists for non existant file
self.assert_( missingExistsRes['OK'] )
self.assertFalse( missingExistsRes['Value'] )
# Check that directories exist
self.assert_( directoryExistsRes['OK'] )
self.assert_( directoryExistsRes['Value'] )
def __getFilePermission(self, path, credDict, noExistStrategy=None):
"""Checks POSIX permission for a file using the VOMS roles.
That is, if the owner group of the file shares the same vomsRole as the requesting user,
we check the permission as if the request was done with the real owner group.
:param path : file path (string)
:param credDict : credential of the user
:param noExistStrategy : If the directory does not exist, we can
* True : allow the access
* False : forbid the access
* None : return the error as is
:returns S_OK structure with a dictionary ( Read/Write/Execute : True/False)
"""
if not path:
return S_ERROR("Empty path")
# We check what is the group stored in the DB for the given path
res = returnSingleResult(self.db.fileManager.getFileMetadata([path]))
if not res["OK"]:
# If the error is not due to the directory not existing, we return
if not self.__isNotExistError(res["Message"]):
return res
# From now on, we know that the error is due to the file not existing
# If we have no strategy regarding non existing files, then just return the error
if noExistStrategy is None:
return res
# Finally, follow the strategy
return S_OK(
dict.fromkeys(["Read", "Write", "Execute"], noExistStrategy))
# ===========================================================================
# # This does not seem necessary since we add the OwnerGroup in the query behind the scene
# origGrp = 'unknown'
# res = self.db.ugManager.getGroupName( res['Value']['GID'] )
# if res['OK']:
# origGrp = res['Value']
# ===========================================================================
origGrp = res["Value"].get("OwnerGroup", "unknown")
# If the two group share the same voms role, we do the query like if we were
# the group stored in the DB
# useless in eiscat
# if self.__shareVomsRole( credDict.get( 'group', 'anon' ), origGrp ):
# credDict = { 'username' : credDict.get( 'username', 'anon' ), 'group' : origGrp}
# eiscat rules,
credDict = self.__eiscatFilesMatching(path,
credDict,
origGrp,
isDir=False)
return returnSingleResult(
self.db.fileManager.getPathPermissions([path], credDict))
def resolvePFNMissing(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNMissing prognosis
"""
pfn = problematicDict['PFN']
se = problematicDict['SE']
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult(self.fc.exists(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if not res['Value']:
gLogger.info("PFNMissing file (%d) no longer exists in catalog" %
fileID)
return self.__updateCompletedFiles('PFNMissing', fileID)
res = returnSingleResult(StorageElement(se).exists(pfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if res['Value']:
gLogger.info("PFNMissing replica (%d) is no longer missing" %
fileID)
return self.__updateReplicaToChecked(problematicDict)
gLogger.info("PFNMissing replica (%d) does not exist" % fileID)
res = returnSingleResult(self.fc.getReplicas(lfn, allStatus=True))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
replicas = res['Value']
seSite = se.split('_')[0].split('-')[0]
found = False
print replicas
for replicaSE in replicas.keys():
if re.search(seSite, replicaSE):
found = True
problematicDict['SE'] = replicaSE
se = replicaSE
if not found:
gLogger.info(
"PFNMissing replica (%d) is no longer registered at SE. Resolved."
% fileID)
return self.__updateCompletedFiles('PFNMissing', fileID)
gLogger.info(
"PFNMissing replica (%d) does not exist. Removing from catalog..."
% fileID)
res = returnSingleResult(self.fc.removeReplica({lfn: problematicDict}))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if len(replicas) == 1:
gLogger.info(
"PFNMissing replica (%d) had a single replica. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID, 'LFNZeroReplicas')
res = self.dm.replicateAndRegister(problematicDict['LFN'], se)
if not res['OK']:
return self.__returnProblematicError(fileID, res)
# If we get here the problem is solved so we can update the integrityDB
return self.__updateCompletedFiles('PFNMissing', fileID)
def checkArchive(self, archiveLFN):
"""Check that archiveLFN does not exist yet."""
LOG.notice('Using Tarball: %s' % archiveLFN)
exists = returnSingleResult(self.fcClient.isFile(archiveLFN))
LOG.debug('Checking for Tarball existence %r' % exists)
if exists['OK'] and exists['Value']:
raise RuntimeError('Tarball %r already exists' % archiveLFN)
LOG.debug('Checking permissions for %r' % archiveLFN)
hasAccess = returnSingleResult(self.fcClient.hasAccess(archiveLFN, 'addFile'))
if not archiveLFN or not hasAccess['OK'] or not hasAccess['Value']:
LOG.error('Error checking tarball location: %r' % hasAccess)
raise ValueError('%s is not a valid path, parameter "Name" must be correct' % archiveLFN)
def __getFilePermission( self, path, credDict, noExistStrategy = None ):
""" Checks POSIX permission for a file using the VOMS roles.
That is, if the owner group of the file shares the same vomsRole as the requesting user,
we check the permission as if the request was done with the real owner group.
:param path : file path (string)
:param credDict : credential of the user
:param noExistStrategy : If the directory does not exist, we can
* True : allow the access
* False : forbid the access
* None : return the error as is
:returns S_OK structure with a dictionary ( Read/Write/Execute : True/False)
"""
if not path:
return S_ERROR( 'Empty path' )
# We check what is the group stored in the DB for the given path
res = returnSingleResult( self.db.fileManager.getFileMetadata( [path] ) )
if not res['OK']:
# If the error is not due to the directory not existing, we return
if not self.__isNotExistError( res['Message'] ):
return res
# From now on, we know that the error is due to the file not existing
# If we have no strategy regarding non existing files, then just return the error
if noExistStrategy is None:
return res
# Finally, follow the strategy
return S_OK( dict.fromkeys( ['Read', 'Write', 'Execute'], noExistStrategy ) )
#===========================================================================
# # This does not seem necessary since we add the OwnerGroup in the query behind the scene
# origGrp = 'unknown'
# res = self.db.ugManager.getGroupName( res['Value']['GID'] )
# if res['OK']:
# origGrp = res['Value']
#===========================================================================
origGrp = res['Value'].get( 'OwnerGroup', 'unknown' )
# If the two group share the same voms role, we do the query like if we were
# the group stored in the DB
if self.__shareVomsRole( credDict.get( 'group', 'anon' ), origGrp ):
credDict = { 'username' : credDict.get( 'username', 'anon' ), 'group' : origGrp}
return returnSingleResult( self.db.fileManager.getPathPermissions( [path], credDict ) )
def resolvePFNMissing( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNMissing prognosis
"""
pfn = problematicDict['PFN']
se = problematicDict['SE']
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult( self.fc.exists( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if not res['Value']:
gLogger.info( "PFNMissing file (%d) no longer exists in catalog" % fileID )
return self.__updateCompletedFiles( 'PFNMissing', fileID )
res = returnSingleResult( StorageElement( se ).exists( pfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if res['Value']:
gLogger.info( "PFNMissing replica (%d) is no longer missing" % fileID )
return self.__updateReplicaToChecked( problematicDict )
gLogger.info( "PFNMissing replica (%d) does not exist" % fileID )
res = returnSingleResult( self.fc.getReplicas( lfn, allStatus = True ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
replicas = res['Value']
seSite = se.split( '_' )[0].split( '-' )[0]
found = False
print replicas
for replicaSE in replicas.keys():
if re.search( seSite, replicaSE ):
found = True
problematicDict['SE'] = replicaSE
se = replicaSE
if not found:
gLogger.info( "PFNMissing replica (%d) is no longer registered at SE. Resolved." % fileID )
return self.__updateCompletedFiles( 'PFNMissing', fileID )
gLogger.info( "PFNMissing replica (%d) does not exist. Removing from catalog..." % fileID )
res = returnSingleResult( self.fc.removeReplica( {lfn:problematicDict} ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if len( replicas ) == 1:
gLogger.info( "PFNMissing replica (%d) had a single replica. Updating prognosis" % fileID )
return self.changeProblematicPrognosis( fileID, 'LFNZeroReplicas' )
res = self.dm.replicateAndRegister( problematicDict['LFN'], se )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
# If we get here the problem is solved so we can update the integrityDB
return self.__updateCompletedFiles( 'PFNMissing', fileID )
def _run(self):
"""Check for migration bit, set file done when migrated."""
self.waitingFiles = self.getWaitingFilesList()
self.log.notice("Waiting files:", len(self.waitingFiles))
targetSESet = set(self.operation.targetSEList)
self.log.notice("Target SEs:", ",".join(targetSESet))
migrated = True
for opFile in self.waitingFiles:
self.log.notice("Checking:", opFile.LFN)
for targetSE in targetSESet:
se = StorageElement(targetSE)
if not se.status()["TapeSE"]:
migrated = True and migrated
continue
metaData = returnSingleResult(se.getFileMetadata(opFile.LFN))
self.log.debug("MetaData: %s" % pformat(metaData))
if not metaData["OK"]:
self.log.error("Failed to get metadata:", "%s: %s" % (opFile.LFN, metaData["Message"]))
migrated = False
continue
migrated = metaData["Value"].get("Migrated", 0) == 1 and migrated
if migrated:
self.log.notice("File has been migrated:", opFile.LFN)
opFile.Status = "Done"
else:
self.log.notice("File has NOT been migrated:", opFile.LFN)
now = datetime.datetime.utcnow().replace(microsecond=0)
extraDelay = datetime.timedelta(minutes=20)
self.request.NotBefore = now + extraDelay
def _registerDescendent(self):
"""Register the tarball as a descendent of the archived LFNs.
Actually registers all LFNs as an ancestor to the Tarball.
"""
registerDescendents = self.parameterDict.get("RegisterDescendent",
None)
if not registerDescendents:
self.log.verbose(
"Will not register tarball as descendent to the Archived LFNs."
)
return
self.log.info(
"Will register tarball as descendent to the Archived LFNs.")
tarballLFN = self.parameterDict["ArchiveLFN"]
ancestorDict = {tarballLFN: {"Ancestors": self.lfns}}
for _trial in range(3):
resAncestors = returnSingleResult(
self.fc.addFileAncestors(ancestorDict))
if resAncestors["OK"]:
break
else:
self.log.error("Failed to register ancestors",
resAncestors["Message"])
raise RuntimeError("Failed to register ancestors")
self.log.info("Successfully registered ancestors")
def downloadFile(dm, lfn, localFolder):
res = returnSingleResult(dm.getFile(lfn, localFolder))
if not res or not res['OK']:
print "Error with file %s"%lfn
print res
else:
print "Downloaded %s"%lfn
def _checkArchiveLFN(self):
"""Make sure the archive LFN does not exist yet."""
archiveLFN = self.parameterDict["ArchiveLFN"]
exists = returnSingleResult(self.fc.isFile(archiveLFN))
self.log.debug("Checking for Tarball existence %r" % exists)
if exists["OK"] and exists["Value"]:
raise RuntimeError("Tarball %r already exists" % archiveLFN)
def _run(self):
"""Check for migration bit, set file done when migrated."""
self.waitingFiles = self.getWaitingFilesList()
self.log.notice('Waiting files:', len(self.waitingFiles))
targetSESet = set(self.operation.targetSEList)
self.log.notice('Target SEs:', ','.join(targetSESet))
migrated = True
for opFile in self.waitingFiles:
self.log.notice('Checking:', opFile.LFN)
for targetSE in targetSESet:
se = StorageElement(targetSE)
if not se.status()['TapeSE']:
migrated = True and migrated
continue
metaData = returnSingleResult(se.getFileMetadata(opFile.LFN))
self.log.debug('MetaData: %s' % pformat(metaData))
if not metaData['OK']:
self.log.error(
'Failed to get metadata:',
'%s: %s' % (opFile.LFN, metaData['Message']))
migrated = False
continue
migrated = metaData['Value'].get('Migrated',
0) == 1 and migrated
if migrated:
self.log.notice('File has been migrated:', opFile.LFN)
opFile.Status = 'Done'
else:
self.log.notice('File has NOT been migrated:', opFile.LFN)
now = datetime.datetime.utcnow().replace(microsecond=0)
extraDelay = datetime.timedelta(minutes=20)
self.request.NotBefore = now + extraDelay
def test_isFile( self ):
print '\n\n#########################################################################\n\n\t\t\tIs file size test\n'
destinationFilePath = '%s/testFile.%s' % ( self.destDirectory, time.time() )
pfnForLfnRes = returnSingleResult( self.storageElement.getPfnForLfn( destinationFilePath ) )
destinationPfn = pfnForLfnRes['Value']
fileDict = {destinationPfn:self.localSourceFile}
putFileRes = self.storageElement.putFile( fileDict, singleFile = True )
# Is a file
isFileRes = self.storageElement.isFile( destinationPfn, singleFile = True )
# Now remove the destination file
removeFileRes = self.storageElement.removeFile( destinationPfn, singleFile = True )
# Get metadata for a removed file
missingIsFileRes = self.storageElement.isFile( destinationPfn, singleFile = True )
# Check directories are handled properly
destinationDir = os.path.dirname( destinationPfn )
directoryIsFileRes = self.storageElement.isFile( destinationDir, singleFile = True )
# Check that the put was done correctly
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value'] )
self.assertEqual( putFileRes['Value'], self.localFileSize )
# Check that we checked the file correctly
self.assert_( isFileRes['OK'] )
self.assert_( isFileRes['Value'] )
# Check that the removal was done correctly
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value'] )
# Check the is file for non existant file
self.assertFalse( missingIsFileRes['OK'] )
expectedError = "File does not exist"
self.assert_( expectedError in missingIsFileRes['Message'] )
# Check that is file operation with a directory
self.assert_( directoryIsFileRes['OK'] )
self.assertFalse( directoryIsFileRes['Value'] )
def __insertRegisterOperation( self, request, operation, toRegister ):
""" add RegisterReplica operation
:param Request request: request instance
:param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
:param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
"""
log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )
byTarget = {}
for ftsFile in toRegister:
if ftsFile.TargetSE not in byTarget:
byTarget.setdefault( ftsFile.TargetSE, [] )
byTarget[ftsFile.TargetSE].append( ftsFile )
log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )
for target, ftsFileList in byTarget.iteritems():
log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
len( ftsFileList ) ) )
registerOperation = Operation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
targetSE = StorageElement( target )
for ftsFile in ftsFileList:
opFile = File()
opFile.LFN = ftsFile.LFN
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
if not pfn["OK"]:
continue
opFile.PFN = pfn["Value"]
registerOperation.addFile( opFile )
request.insertBefore( registerOperation, operation )
return S_OK()
def __uploadFile(self, se, pfn):
"""proxied upload file"""
res = self.__prepareSecurityDetails()
if not res["OK"]:
return res
# Put file to the SE
try:
storageElement = StorageElement(se)
except AttributeError as x:
errStr = "__uploadFile: Exception while instantiating the Storage Element."
gLogger.exception(errStr, se, str(x))
return S_ERROR(errStr)
putFileDir = "%s/putFile" % BASE_PATH
localFileName = "%s/%s" % (putFileDir, os.path.basename(pfn))
res = returnSingleResult(storageElement.putFile({pfn: localFileName}))
if not res["OK"]:
gLogger.error("prepareFile: Failed to put local file to storage.",
res["Message"])
# Clear the local cache
try:
gLogger.debug("Removing temporary file", localFileName)
os.remove(localFileName)
except Exception as x:
gLogger.exception("Failed to remove local file", localFileName, x)
return res
def finalize( self ):
""" register successfully transferred files """
if self.Status not in FTSJob.FINALSTATES:
return S_OK()
startTime = time.time()
targetSE = StorageElement( self.TargetSE )
toRegister = [ ftsFile for ftsFile in self if ftsFile.Status == "Finished" ]
toRegisterDict = {}
for ftsFile in toRegister:
pfn = returnSingleResult( targetSE.getPfnForProtocol( ftsFile.TargetSURL, protocol = "SRM2", withPort = False ) )
if not pfn["OK"]:
continue
pfn = pfn["Value"]
toRegisterDict[ ftsFile.LFN ] = { "PFN": pfn, "SE": self.TargetSE }
if toRegisterDict:
self._regTotal += len( toRegisterDict )
register = self._fc.addReplica( toRegisterDict )
self._regTime += time.time() - startTime
if not register["OK"]:
for ftsFile in toRegister:
ftsFile.Error = "AddCatalogReplicaFailed"
return register
register = register["Value"]
self._regSuccess += len( register.get( 'Successful', {} ) )
failedFiles = register.get( "Failed", {} )
for ftsFile in toRegister:
if ftsFile.LFN in failedFiles:
ftsFile.Error = "AddCatalogReplicaFailed"
return S_OK()
def downloadFile(dm, lfn, localFolder):
res = returnSingleResult(dm.getFile(lfn, localFolder))
if not res or not res['OK']:
print "Error with file %s" % lfn
print res
else:
print "Downloaded %s" % lfn
def test_getAccessUrl(self):
print '\n\n#########################################################################\n\n\t\tGet access url test\n'
destinationFilePath = '%s/testFile.%s' % (self.destDirectory,
time.time())
pfnForLfnRes = returnSingleResult(
self.storageElement.getPfnForLfn(destinationFilePath))
destinationPfn = pfnForLfnRes['Value']
fileDict = {destinationPfn: self.localSourceFile}
putFileRes = self.storageElement.putFile(fileDict, singleFile=True)
# Get a transfer url for the file
getTurlRes = self.storageElement.getAccessUrl(destinationPfn,
singleFile=True)
# Remove the destination file
removeFileRes = self.storageElement.removeFile(destinationPfn,
singleFile=True)
# Get missing turl res
getMissingTurlRes = self.storageElement.getAccessUrl(destinationPfn,
singleFile=True)
# Check that the put was done correctly
self.assert_(putFileRes['OK'])
self.assert_(putFileRes['Value'])
self.assertEqual(putFileRes['Value'], self.localFileSize)
# Check that we can get the tURL properly
self.assert_(getTurlRes['OK'])
self.assert_(getTurlRes['Value'])
self.assert_(type(getTurlRes['Value']) in types.StringTypes)
# Check that the removal was done correctly
self.assert_(removeFileRes['OK'])
self.assert_(removeFileRes['Value'])
def __getCatalogDirectoryContents(self, directories):
""" get catalog contents under paths :directories:
:param self: self reference
:param list directories: list of paths in catalog
"""
self.log.info('Obtaining the catalog contents for %d directories:' % len(directories))
for directory in directories:
self.log.info(directory)
activeDirs = directories
allFiles = {}
fc = FileCatalog()
while activeDirs:
currentDir = activeDirs[0]
res = returnSingleResult(fc.listDirectory(currentDir))
activeDirs.remove(currentDir)
if not res['OK'] and 'Directory does not exist' in res['Message']: # FIXME: DFC should return errno
self.log.info("The supplied directory %s does not exist" % currentDir)
elif not res['OK']:
if "No such file or directory" in res['Message']:
self.log.info("%s: %s" % (currentDir, res['Message']))
else:
self.log.error("Failed to get directory %s content: %s" % (currentDir, res['Message']))
else:
dirContents = res['Value']
activeDirs.extend(dirContents['SubDirs'])
allFiles.update(dirContents['Files'])
self.log.info("Found %d files" % len(allFiles))
return S_OK(allFiles.keys())
def __insertRegisterOperation( self, request, operation, toRegister ):
""" add RegisterReplica operation
:param Request request: request instance
:param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
:param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
"""
log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )
byTarget = {}
for ftsFile in toRegister:
if ftsFile.TargetSE not in byTarget:
byTarget.setdefault( ftsFile.TargetSE, [] )
byTarget[ftsFile.TargetSE].append( ftsFile )
log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )
for target, ftsFileList in byTarget.iteritems():
log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
len( ftsFileList ) ) )
registerOperation = Operation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
targetSE = StorageElement( target )
for ftsFile in ftsFileList:
opFile = File()
opFile.LFN = ftsFile.LFN
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
if not pfn["OK"]:
continue
opFile.PFN = pfn["Value"]
registerOperation.addFile( opFile )
request.insertBefore( registerOperation, operation )
return S_OK()
def checkArchive(self, archiveLFN):
"""Check that archiveLFN does not exist yet."""
sLog.notice("Using Tarball: %s" % archiveLFN)
exists = returnSingleResult(self.fcClient.isFile(archiveLFN))
sLog.debug("Checking for Tarball existence %r" % exists)
if exists["OK"] and exists["Value"]:
raise RuntimeError("Tarball %r already exists" % archiveLFN)
sLog.debug("Checking permissions for %r" % archiveLFN)
hasAccess = returnSingleResult(
self.fcClient.hasAccess(archiveLFN, "addFile"))
if not archiveLFN or not hasAccess["OK"] or not hasAccess["Value"]:
sLog.error("Error checking tarball location: %r" % hasAccess)
raise ValueError(
'%s is not a valid path, parameter "Name" must be correct' %
archiveLFN)
def _registerDescendent(self):
"""Register the tarball as a descendent of the archived LFNs.
Actually registers all LFNs as an ancestor to the Tarball.
"""
registerDescendents = self.parameterDict.get('RegisterDescendent',
None)
if not registerDescendents:
self.log.verbose(
'Will not register tarball as descendent to the Archived LFNs.'
)
return
self.log.info(
'Will register tarball as descendent to the Archived LFNs.')
tarballLFN = self.parameterDict['ArchiveLFN']
ancestorDict = {tarballLFN: {'Ancestors': self.lfns}}
for _trial in range(3):
resAncestors = returnSingleResult(
self.fc.addFileAncestors(ancestorDict))
if resAncestors['OK']:
break
else:
self.log.error('Failed to register ancestors',
resAncestors['Message'])
raise RuntimeError('Failed to register ancestors')
self.log.info('Successfully registered ancestors')
def __registerSuccessful( self, transLFNs ):
""" register successfully transferred files to the catalogs,
fill failedRegistrations dict for files that failed to register
:param self: self reference
:param list transLFNs: LFNs in FTS job
"""
self.failedRegistrations = {}
toRegister = {}
for lfn in transLFNs:
res = returnSingleResult( self.oTargetSE.getPfnForProtocol( self.fileDict[lfn].get( 'Target' ), protocol = 'SRM2', withPort = False ) )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE }
if not toRegister:
return S_OK( ( 0, 0 ) )
res = self.__getCatalogObject()
if not res['OK']:
for lfn in toRegister:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
res = self.oCatalog.addReplica( toRegister )
if not res['OK']:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
for lfn, error in res['Value']['Failed'].items():
self.failedRegistrations[lfn] = toRegister[lfn]
self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) )
return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) )
def _downloadFiles(self):
"""Download the files."""
self._checkFilePermissions()
for index, opFile in enumerate(self.waitingFiles):
lfn = opFile.LFN
self.log.info('Processing file (%d/%d) %r' % (index, len(self.waitingFiles), lfn))
sourceSE = self.parameterDict['SourceSE']
attempts = 0
destFolder = os.path.join(self.cacheFolder, os.path.dirname(lfn)[1:])
self.log.debug('Local Cache Folder: %s' % destFolder)
if not os.path.exists(destFolder):
os.makedirs(destFolder)
while True:
attempts += 1
download = returnSingleResult(self.dm.getFile(lfn, destinationDir=destFolder, sourceSE=sourceSE))
if download['OK']:
self.log.info('Downloaded file %r to %r' % (lfn, destFolder))
break
errorString = download['Message']
self.log.error('Failed to download file:', errorString)
opFile.Error = errorString
opFile.Attempt += 1
self.operation.Error = opFile.Error
if 'No such file or directory' in opFile.Error:
# The File does not exist, we just ignore this and continue, otherwise we never archive the other files
opFile.Status = 'Done'
download = S_OK()
break
if attempts > 10:
self.log.error('Completely failed to download file:', errorString)
raise RuntimeError('Completely failed to download file: %s' % errorString)
return
def _checkArchiveLFN(self):
"""Make sure the archive LFN does not exist yet."""
archiveLFN = self.parameterDict['ArchiveLFN']
exists = returnSingleResult(self.fc.isFile(archiveLFN))
self.log.debug('Checking for Tarball existence %r' % exists)
if exists['OK'] and exists['Value']:
raise RuntimeError('Tarball %r already exists' % archiveLFN)
def __getCatalogDirectoryContents(self, directories):
""" get catalog contents under paths :directories:
:param self: self reference
:param list directories: list of paths in catalog
"""
self.log.info('Obtaining the catalog contents for %d directories:' %
len(directories))
for directory in directories:
self.log.info(directory)
activeDirs = directories
allFiles = {}
fc = FileCatalog()
while activeDirs:
currentDir = activeDirs[0]
res = returnSingleResult(fc.listDirectory(currentDir))
activeDirs.remove(currentDir)
if not res['OK'] and 'Directory does not exist' in res[
'Message']: # FIXME: DFC should return errno
self.log.info("The supplied directory %s does not exist" %
currentDir)
elif not res['OK']:
if "No such file or directory" in res['Message']:
self.log.info("%s: %s" % (currentDir, res['Message']))
else:
self.log.error("Failed to get directory %s content: %s" %
(currentDir, res['Message']))
else:
dirContents = res['Value']
activeDirs.extend(dirContents['SubDirs'])
allFiles.update(dirContents['Files'])
self.log.info("Found %d files" % len(allFiles))
return S_OK(allFiles.keys())
def tearDown(self):
destinationDir = returnSingleResult(
self.storageElement.getPfnForLfn(self.destDirectory))
res = self.storageElement.removeDirectory(destinationDir,
recursive=True,
singleDirectory=True)
self.assert_(res['OK'])
def __registerSuccessful( self, transLFNs ):
""" register successfully transferred files to the catalogs,
fill failedRegistrations dict for files that failed to register
:param self: self reference
:param list transLFNs: LFNs in FTS job
"""
self.failedRegistrations = {}
toRegister = {}
for lfn in transLFNs:
res = returnSingleResult( self.oTargetSE.getURL( self.fileDict[lfn].get( 'Target' ), protocol = 'srm' ) )
if not res['OK']:
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRegister[lfn] = { 'PFN' : res['Value'], 'SE' : self.targetSE }
if not toRegister:
return S_OK( ( 0, 0 ) )
res = self.__getCatalogObject()
if not res['OK']:
for lfn in toRegister:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
res = self.oCatalog.addReplica( toRegister )
if not res['OK']:
self.failedRegistrations = toRegister
self.log.error( 'Failed to get Catalog Object', res['Message'] )
return S_OK( ( 0, len( toRegister ) ) )
for lfn, error in res['Value']['Failed'].items():
self.failedRegistrations[lfn] = toRegister[lfn]
self.log.error( 'Registration of Replica failed', '%s : %s' % ( lfn, str( error ) ) )
return S_OK( ( len( res['Value']['Successful'] ), len( toRegister ) ) )
def resolvePFNUnavailable(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNUnavailable prognosis
"""
lfn = problematicDict['LFN']
se = problematicDict['SE']
fileID = problematicDict['FileID']
res = returnSingleResult(StorageElement(se).getFileMetadata(lfn))
if (not res['OK']) and (re.search('File does not exist',
res['Message'])):
# The file is no longer Unavailable but has now dissapeared completely
gLogger.info(
"PFNUnavailable replica (%d) found to be missing. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID, 'PFNMissing')
if (not res['OK']) or res['Value']['Unavailable']:
gLogger.info(
"PFNUnavailable replica (%d) found to still be Unavailable" %
fileID)
return self.incrementProblematicRetry(fileID)
if res['Value']['Lost']:
gLogger.info(
"PFNUnavailable replica (%d) is now found to be Lost. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID, 'PFNLost')
gLogger.info("PFNUnavailable replica (%d) is no longer Unavailable" %
fileID)
# Need to make the replica okay in the Catalog
return self.__updateReplicaToChecked(problematicDict)
def __prepareFile(self, se, pfn):
"""proxied prepare file"""
res = self.__prepareSecurityDetails()
if not res["OK"]:
return res
# Clear the local cache
getFileDir = "%s/getFile" % BASE_PATH
if not os.path.exists(getFileDir):
os.mkdir(getFileDir)
# Get the file to the cache
try:
storageElement = StorageElement(se)
except AttributeError as x:
errStr = "prepareFile: Exception while instantiating the Storage Element."
gLogger.exception(errStr, se, str(x))
return S_ERROR(errStr)
res = returnSingleResult(
storageElement.getFile(pfn, localPath="%s/getFile" % BASE_PATH))
if not res["OK"]:
gLogger.error("prepareFile: Failed to get local copy of file.",
res["Message"])
return res
return S_OK()
def resolveCatalogPFNSizeMismatch( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the CatalogPFNSizeMismatch prognosis
"""
lfn = problematicDict['LFN']
pfn = problematicDict['PFN']
se = problematicDict['SE']
fileID = problematicDict['FileID']
res = returnSingleResult( self.fc.getFileSize( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
catalogSize = res['Value']
res = returnSingleResult( StorageElement( se ).getFileSize( pfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
storageSize = res['Value']
bkKCatalog = FileCatalog( ['BookkeepingDB'] )
res = returnSingleResult( bkKCatalog.getFileSize( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
bookkeepingSize = res['Value']
if bookkeepingSize == catalogSize == storageSize:
gLogger.info( "CatalogPFNSizeMismatch replica (%d) matched all registered sizes." % fileID )
return self.__updateReplicaToChecked( problematicDict )
if ( catalogSize == bookkeepingSize ):
gLogger.info( "CatalogPFNSizeMismatch replica (%d) found to mismatch the bookkeeping also" % fileID )
res = returnSingleResult( self.fc.getReplicas( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if len( res['Value'] ) <= 1:
gLogger.info( "CatalogPFNSizeMismatch replica (%d) has no other replicas." % fileID )
return S_ERROR( "Not removing catalog file mismatch since the only replica" )
else:
gLogger.info( "CatalogPFNSizeMismatch replica (%d) has other replicas. Removing..." % fileID )
res = self.dm.removeReplica( se, lfn )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.__updateCompletedFiles( 'CatalogPFNSizeMismatch', fileID )
if ( catalogSize != bookkeepingSize ) and ( bookkeepingSize == storageSize ):
gLogger.info( "CatalogPFNSizeMismatch replica (%d) found to match the bookkeeping size" % fileID )
res = self.__updateReplicaToChecked( problematicDict )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.changeProblematicPrognosis( fileID, 'BKCatalogSizeMismatch' )
gLogger.info( "CatalogPFNSizeMismatch replica (%d) all sizes found mismatch. Updating retry count" % fileID )
return self.incrementProblematicRetry( fileID )
def resolvePFNZeroSize(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolves the PFNZeroSize prognosis
"""
lfn = problematicDict['LFN']
seName = problematicDict['SE']
fileID = problematicDict['FileID']
se = StorageElement(seName)
res = returnSingleResult(se.getFileSize(lfn))
if (not res['OK']) and (re.search('File does not exist',
res['Message'])):
gLogger.info(
"PFNZeroSize replica (%d) found to be missing. Updating prognosis"
% problematicDict['FileID'])
return self.changeProblematicPrognosis(fileID, 'PFNMissing')
storageSize = res['Value']
if storageSize == 0:
res = returnSingleResult(se.removeFile(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
gLogger.info(
"PFNZeroSize replica (%d) removed. Updating prognosis" %
problematicDict['FileID'])
return self.changeProblematicPrognosis(fileID, 'PFNMissing')
res = returnSingleResult(self.fc.getReplicas(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if seName not in res['Value']:
gLogger.info(
"PFNZeroSize replica (%d) not registered in catalog. Updating prognosis"
% problematicDict['FileID'])
return self.changeProblematicPrognosis(fileID, 'PFNNotRegistered')
res = returnSingleResult(self.fc.getFileMetadata(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
catalogSize = res['Value']['Size']
if catalogSize != storageSize:
gLogger.info(
"PFNZeroSize replica (%d) size found to differ from registered metadata. Updating prognosis"
% problematicDict['FileID'])
return self.changeProblematicPrognosis(fileID,
'CatalogPFNSizeMismatch')
return self.__updateCompletedFiles('PFNZeroSize', fileID)
def _hasAccess(self, lfn, s3_method):
"""Check if we have permission to execute given operation on the given file (if exists) or its directory"""
opType = self._s3ToFC_methods.get(s3_method)
if not opType:
return S_ERROR(errno.EINVAL, "Unknown S3 method %s" % s3_method)
return returnSingleResult(self._fc.hasAccess(lfn, opType))
def test_putFile( self ):
print '\n\n#########################################################################\n\n\t\t\tPut file test\n'
destinationFilePath = '%s/testFile.%s' % ( self.destDirectory, time.time() )
# pfnForLfnRes = returnSingleResult( self.storageElement.getURL( destinationFilePath ) )
#destinationPfn = pfnForLfnRes['Value']
fileDict = {destinationFilePath:self.localSourceFile}
putFileRes = returnSingleResult( self.storageElement.putFile( fileDict ) )
# Now remove the destination file
removeFileRes = returnSingleResult( self.storageElement.removeFile( destinationFilePath ) )
# Check that the put was done correctly
self.assert_( putFileRes['OK'] )
self.assert_( putFileRes['Value'] )
self.assertEqual( putFileRes['Value'], self.localFileSize )
# Check that the removal was done correctly
self.assert_( removeFileRes['OK'] )
self.assert_( removeFileRes['Value'] )
def resolveLFNCatalogMissing( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolved the LFNCatalogMissing prognosis
"""
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult( self.fc.exists( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if res['Value']:
return self.__updateCompletedFiles( 'LFNCatalogMissing', fileID )
# Remove the file from all catalogs
# RF_NOTE : here I can do it because it's a single file, but otherwise I would need to sort the path
res = returnSingleResult( self.fc.removeFile( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
return self.__updateCompletedFiles( 'LFNCatalogMissing', fileID )
def resolveLFNCatalogMissing(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the LFNCatalogMissing prognosis
"""
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult(self.fc.exists(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if res['Value']:
return self.__updateCompletedFiles('LFNCatalogMissing', fileID)
# Remove the file from all catalogs
# RF_NOTE : here I can do it because it's a single file, but otherwise I would need to sort the path
res = returnSingleResult(self.fc.removeFile(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
return self.__updateCompletedFiles('LFNCatalogMissing', fileID)
def setUp( self ): self.numberOfFiles = 1 self.storageElement = StorageElement( storageElementToTest ) self.localSourceFile = "/etc/group" self.localFileSize = getSize( self.localSourceFile ) self.destDirectory = "/lhcb/test/unit-test/TestStorageElement" destinationDir = returnSingleResult( self.storageElement.getPfnForLfn( self.destDirectory ) ) res = self.storageElement.createDirectory( destinationDir, singleDirectory = True ) self.assert_( res['OK'] )
def resolveTarget( self ):
""" find target SE eligible for submission
:param self: self reference
"""
toResolve = [ lfn for lfn in self.fileDict
if self.fileDict[lfn].get( 'Status' ) not in self.noSubmitStatus ]
if not toResolve:
return S_OK()
res = self.__updateReplicaCache( toResolve )
if not res['OK']:
return res
for lfn in toResolve:
res = returnSingleResult( self.oTargetSE.getURL( lfn, protocol = 'srm' ) )
if not res['OK']:
reason = res.get( 'Message', res['Message'] )
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, reason ) )
self.__setFileParameter( lfn, 'Reason', reason )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
res = self.setTargetSURL( lfn, res['Value'] )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - %s" % ( lfn, res["Message"] ) )
self.__setFileParameter( lfn, 'Reason', res['Message'] )
self.__setFileParameter( lfn, 'Status', 'Failed' )
continue
toResolve = []
for lfn in self.fileDict:
if "Target" in self.fileDict[lfn]:
toResolve.append( lfn )
if not toResolve:
return S_ERROR( "No eligible Target files" )
res = self.oTargetSE.exists( toResolve )
if not res['OK']:
return S_ERROR( "Failed to check target existence" )
for lfn, error in res['Value']['Failed'].items():
self.__setFileParameter( lfn, 'Reason', error )
self.__setFileParameter( lfn, 'Status', 'Failed' )
toRemove = []
for lfn, exists in res['Value']['Successful'].items():
if exists:
res = self.getSourceSURL( lfn )
if not res['OK']:
gLogger.warn( "resolveTarget: skipping %s - target exists" % lfn )
self.__setFileParameter( lfn, 'Reason', "Target exists" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
elif res['Value'] == self.fileDict[lfn]['Target']:
gLogger.warn( "resolveTarget: skipping %s - source and target pfns are the same" % lfn )
self.__setFileParameter( lfn, 'Reason', "Source and Target the same" )
self.__setFileParameter( lfn, 'Status', 'Failed' )
else:
toRemove.append( lfn )
if toRemove:
self.oTargetSE.removeFile( toRemove )
return S_OK()
def removeStorageDirectoryFromSE( directory, storageElement ):
"""
Delete directory on selected storage element
"""
se = StorageElement( storageElement, False )
res = returnSingleResult( se.exists( directory ) )
if not res['OK']:
return S_ERROR( "Failed to obtain existence of directory" + res['Message'] )
exists = res['Value']
if not exists:
return S_OK( "The directory %s does not exist at %s " % ( directory, storageElement ) )
res = returnSingleResult( se.removeDirectory( directory, recursive = True ) )
if not res['OK']:
return S_ERROR( "Failed to remove storage directory" + res['Message'] )
return S_OK()
def __updateReplicaToChecked( self, problematicDict ):
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
prognosis = problematicDict['Prognosis']
problematicDict['Status'] = 'Checked'
res = returnSingleResult( self.fc.setReplicaStatus( {lfn:problematicDict} ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
gLogger.info( "%s replica (%d) is updated to Checked status" % ( prognosis, fileID ) )
return self.__updateCompletedFiles( prognosis, fileID )
def cleanTransformationLogFiles( self, directory ):
""" clean up transformation logs from directory :directory:
:param self: self reference
:param str directory: folder name
"""
self.log.verbose( "Removing log files found in the directory %s" % directory )
res = returnSingleResult( StorageElement( self.logSE ).removeDirectory( directory ) )
if not res['OK']:
self.log.error( "Failed to remove log files", res['Message'] )
return res
self.log.info( "Successfully removed transformation log directory" )
return S_OK()
def __generateURLDict(self, lfns, storage, replicaDict=None):
""" Generates a dictionary (url : lfn ), where the url are constructed
from the lfn using the constructURLFromLFN method of the storage plugins.
:param: lfns : dictionary {lfn:whatever}
:returns dictionary {constructed url : lfn}
"""
log = self.log.getSubLogger("__generateURLDict")
log.verbose("generating url dict for %s lfn in %s." % (len(lfns), self.name))
if not replicaDict:
replicaDict = {}
urlDict = {} # url : lfn
failed = {} # lfn : string with errors
for lfn in lfns:
if self.useCatalogURL:
# Is this self.name alias proof?
url = replicaDict.get(lfn, {}).get(self.name, '')
if url:
urlDict[url] = lfn
continue
else:
fc = self.__getFileCatalog()
result = fc.getReplicas()
if not result['OK']:
failed[lfn] = result['Message']
url = result['Value']['Successful'].get(lfn, {}).get(self.name, '')
if not url:
failed[lfn] = 'Failed to get catalog replica'
else:
# Update the URL according to the current SE description
result = returnSingleResult(storage.updateURL(url))
if not result['OK']:
failed[lfn] = result['Message']
else:
urlDict[result['Value']] = lfn
else:
result = storage.constructURLFromLFN(lfn, withWSUrl=True)
if not result['OK']:
errStr = result['Message']
log.debug(errStr, 'for %s' % (lfn))
failed[lfn] = "%s %s" % (failed[lfn], errStr) if lfn in failed else errStr
else:
urlDict[result['Value']] = lfn
res = S_OK({'Successful': urlDict, 'Failed': failed})
# res['Failed'] = failed
return res
def resolvePFNZeroSize( self, problematicDict ):
""" This takes the problematic dictionary returned by the integrity DB and resolves the PFNZeroSize prognosis
"""
lfn = problematicDict['LFN']
seName = problematicDict['SE']
fileID = problematicDict['FileID']
se = StorageElement( seName )
res = returnSingleResult( se.getFileSize( lfn ) )
if ( not res['OK'] ) and ( re.search( 'File does not exist', res['Message'] ) ):
gLogger.info( "PFNZeroSize replica (%d) found to be missing. Updating prognosis" % problematicDict['FileID'] )
return self.changeProblematicPrognosis( fileID, 'PFNMissing' )
storageSize = res['Value']
if storageSize == 0:
res = returnSingleResult( se.removeFile( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
gLogger.info( "PFNZeroSize replica (%d) removed. Updating prognosis" % problematicDict['FileID'] )
return self.changeProblematicPrognosis( fileID, 'PFNMissing' )
res = returnSingleResult( self.fc.getReplicas( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
if seName not in res['Value']:
gLogger.info( "PFNZeroSize replica (%d) not registered in catalog. Updating prognosis" % problematicDict['FileID'] )
return self.changeProblematicPrognosis( fileID, 'PFNNotRegistered' )
res = returnSingleResult( self.fc.getFileMetadata( lfn ) )
if not res['OK']:
return self.__returnProblematicError( fileID, res )
catalogSize = res['Value']['Size']
if catalogSize != storageSize:
gLogger.info( "PFNZeroSize replica (%d) size found to differ from registered metadata. Updating prognosis" % problematicDict['FileID'] )
return self.changeProblematicPrognosis( fileID, 'CatalogPFNSizeMismatch' )
return self.__updateCompletedFiles( 'PFNZeroSize', fileID )
def __getRegisteredPFNLFN( self, pfn, storageElement ):
res = StorageElement( storageElement ).getPfnForProtocol( pfn, withPort = False )
if not res['OK']:
gLogger.error( "Failed to get registered PFN for physical files", res['Message'] )
return res
for pfn, error in res['Value']['Failed'].items():
gLogger.error( 'Failed to obtain registered PFN for physical file', '%s %s' % ( pfn, error ) )
return S_ERROR( 'Failed to obtain registered PFNs from physical file' )
registeredPFN = res['Value']['Successful'][pfn]
res = returnSingleResult( self.fc.getLFNForPFN( registeredPFN ) )
if ( not res['OK'] ) and re.search( 'No such file or directory', res['Message'] ):
return S_OK( False )
return S_OK( res['Value'] )
def finalize( self ):
""" register successfully transferred files """
if self.Status not in FTSJob.FINALSTATES:
return S_OK()
if not len( self ):
return S_ERROR( "Empty job in finalize" )
startTime = time.time()
targetSE = StorageElement( self.TargetSE )
toRegister = [ ftsFile for ftsFile in self if ftsFile.Status == "Finished" ]
toRegisterDict = {}
for ftsFile in toRegister:
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = 'srm' ) )
if pfn["OK"]:
pfn = pfn["Value"]
toRegisterDict[ ftsFile.LFN ] = { "PFN": pfn, "SE": self.TargetSE }
else:
self._log.error( "Error getting SRM URL", pfn['Message'] )
if toRegisterDict:
self._regTotal += len( toRegisterDict )
register = self._fc.addReplica( toRegisterDict )
self._regTime += time.time() - startTime
if not register["OK"]:
self._log.error( 'Error registering replica', register['Message'] )
for ftsFile in toRegister:
ftsFile.Error = "AddCatalogReplicaFailed"
return register
register = register["Value"]
self._regSuccess += len( register.get( 'Successful', {} ) )
if self._regSuccess:
self._log.info( 'Successfully registered %d replicas' % self._regSuccess )
failedFiles = register.get( "Failed", {} )
errorReason = {}
for lfn, reason in failedFiles.items():
errorReason.setdefault( str( reason ), [] ).append( lfn )
for reason in errorReason:
self._log.error( 'Error registering %d replicas' % len( errorReason[reason] ), reason )
for ftsFile in toRegister:
if ftsFile.LFN in failedFiles:
ftsFile.Error = "AddCatalogReplicaFailed"
else:
statuses = set( [ftsFile.Status for ftsFile in self] )
self._log.warn( "No replicas to register for FTSJob (%s) - Files status: '%s'" % \
( self.Status, ','.join( sorted( statuses ) ) ) )
return S_OK()
def test_getDirectory( self ):
print '\n\n#########################################################################\n\n\t\t\tGet directory test\n'
directory = "%s/%s" % ( self.destDirectory, 'getDirectoryTest' )
destDirectory = returnSingleResult( self.storageElement.getPfnForLfn( directory ) )['Value']
# Create a local directory to upload
localDir = '/tmp/unit-test'
srcFile = '/etc/group'
sizeOfLocalFile = getSize( srcFile )
if not os.path.exists( localDir ):
os.mkdir( localDir )
for i in range( self.numberOfFiles ):
shutil.copy( srcFile, '%s/testFile.%s' % ( localDir, time.time() ) )
time.sleep( 1 )
# Check that we can successfully upload the directory to the storage element
dirDict = {destDirectory:localDir}
putDirRes = self.storageElement.putDirectory( dirDict, singleDirectory = True )
# Get the directory metadata
#Clean up the locally created directory
shutil.rmtree( localDir )
getDirRes = self.storageElement.getDirectory( destDirectory, localPath = localDir, singleDirectory = True )
# Now remove the remove directory
removeDirRes = self.storageElement.removeDirectory( destDirectory, recursive = True, singleDirectory = True )
#Clean up the locally created directory
shutil.rmtree( localDir )
# Perform the checks for the put dir operation
self.assert_( putDirRes['OK'] )
self.assert_( putDirRes['Value'] )
if putDirRes['Value']['Files']:
self.assertEqual( putDirRes['Value']['Files'], self.numberOfFiles )
self.assertEqual( putDirRes['Value']['Size'], self.numberOfFiles * sizeOfLocalFile )
self.assert_( type( putDirRes['Value']['Files'] ) in [types.IntType, types.LongType] )
self.assert_( type( putDirRes['Value']['Size'] ) in [types.IntType, types.LongType] )
# Perform the checks for the get directory operation
self.assert_( getDirRes['OK'] )
self.assert_( getDirRes['Value'] )
if getDirRes['Value']['Files']:
self.assertEqual( getDirRes['Value']['Files'], self.numberOfFiles )
self.assertEqual( getDirRes['Value']['Size'], self.numberOfFiles * sizeOfLocalFile )
self.assert_( type( getDirRes['Value']['Files'] ) in [types.IntType, types.LongType] )
self.assert_( type( getDirRes['Value']['Size'] ) in [types.IntType, types.LongType] )
# Perform the checks for the remove directory operation
self.assert_( removeDirRes['OK'] )
self.assert_( removeDirRes['Value'] )
if removeDirRes['Value']['FilesRemoved']:
self.assertEqual( removeDirRes['Value']['FilesRemoved'], self.numberOfFiles )
self.assertEqual( removeDirRes['Value']['SizeRemoved'], self.numberOfFiles * sizeOfLocalFile )
self.assert_( type( removeDirRes['Value']['FilesRemoved'] ) in [types.IntType, types.LongType] )
self.assert_( type( removeDirRes['Value']['SizeRemoved'] ) in [types.IntType, types.LongType] )
def test_createDirectory( self ):
print '\n\n#########################################################################\n\n\t\t\tCreate directory test\n'
directory = "%s/%s" % ( self.destDirectory, 'createDirectoryTest' )
pfnForLfnRes = returnSingleResult( self.storageElement.getPfnForLfn( directory ) )
directoryPfn = pfnForLfnRes['Value']
createDirRes = self.storageElement.createDirectory( directoryPfn, singleDirectory = True )
# Remove the target dir
removeDirRes = self.storageElement.removeDirectory( directoryPfn, recursive = True, singleDirectory = True )
# Check that the creation was done correctly
self.assert_( createDirRes['OK'] )
self.assert_( createDirRes['Value'] )
# Remove the directory
self.assert_( removeDirRes['OK'] )
self.assert_( removeDirRes['Value'] )
def test_getFile( self ):
print '\n\n#########################################################################\n\n\t\t\tGet file test\n'
destinationFilePath = '%s/testFile.%s' % ( self.destDirectory, time.time() )
# pfnForLfnRes = returnSingleResult( self.storageElement.getURL( destinationFilePath ) )
#destinationPfn = pfnForLfnRes['Value']
fileDict = {destinationFilePath:self.localSourceFile}
putFileRes = returnSingleResult( self.storageElement.putFile( fileDict ) )
# Now get a local copy of the file
getFileRes = returnSingleResult( self.storageElement.getFile( destinationFilePath ) )
# Now remove the destination file
removeFileRes = returnSingleResult( self.storageElement.removeFile( destinationFilePath ) )
# Clean up the local mess
os.remove( os.path.basename( destinationFilePath ) )
# Check that the put was done correctly
self.assertTrue( putFileRes['OK'] )
self.assertTrue( putFileRes['Value'] )
self.assertEqual( putFileRes['Value'], self.localFileSize )
# Check that we got the file correctly
self.assertTrue( getFileRes['OK'] )
self.assertEqual( getFileRes['Value'], self.localFileSize )
# Check that the removal was done correctly
self.assertTrue( removeFileRes['OK'] )
self.assertTrue( removeFileRes['Value'] )
def _createDirectory(self, path, failed, mode=0o775):
"""Create the directory at path if it does not exist.
:param str path: path to check
:param list failed: list of failed paths
:param int mode: mode to set for directory
"""
exists = returnSingleResult(self.fc.isDirectory(path))
if exists['OK'] and exists['Value']:
LOG.verbose('Directory already exists:', path)
return S_OK()
result = returnSingleResult(self.fc.createDirectory(path))
if not result['OK']:
LOG.error('Failed to create directory:', '%s: %s' % (path, result['Message']))
failed[path].append(result['Message'])
return S_ERROR()
LOG.verbose('Successfully created directory:', path)
res = self.fc.changePathMode({path: mode}, False)
if not res['OK']:
LOG.error(res['Message'])
failed[path].append(res['Message'])
return S_ERROR()
LOG.verbose('Successfully changed mode:', path)
return S_OK()
def removeRemoteDirectory(fc,lfn):
"""
Remove file from the catalog
"""
storageElements = gConfig.getValue( 'Resources/StorageElementGroups/SE_Cleaning_List', [] )
for storageElement in sorted( storageElements ):
res = removeStorageDirectoryFromSE( lfn, storageElement )
if not res['OK']:
return S_ERROR( "Failed to clean storage directory at all SE:" + res['Message'] )
res = returnSingleResult( fc.removeDirectory( lfn, recursive = True ) )
if not res['OK']:
return S_ERROR( "Failed to clean storage directory at all SE:" + res['Message'] )
return S_OK("Successfully removed directory")
