def getActiveSEs(seList, access='Write'):
""" Utility function - uses the StorageElement cached status
"""
return [
se for se in seList
if StorageElement(se).getStatus().get('Value', {}).get(access, False)
]
def _run(self):
"""Check for migration bit, set file done when migrated."""
self.waitingFiles = self.getWaitingFilesList()
self.log.notice('Waiting files:', len(self.waitingFiles))
targetSESet = set(self.operation.targetSEList)
self.log.notice('Target SEs:', ','.join(targetSESet))
migrated = True
for opFile in self.waitingFiles:
self.log.notice('Checking:', opFile.LFN)
for targetSE in targetSESet:
se = StorageElement(targetSE)
if not se.status()['TapeSE']:
migrated = True and migrated
continue
metaData = returnSingleResult(se.getFileMetadata(opFile.LFN))
self.log.debug('MetaData: %s' % pformat(metaData))
if not metaData['OK']:
self.log.error(
'Failed to get metadata:',
'%s: %s' % (opFile.LFN, metaData['Message']))
migrated = False
continue
migrated = metaData['Value'].get('Migrated',
0) == 1 and migrated
if migrated:
self.log.notice('File has been migrated:', opFile.LFN)
opFile.Status = 'Done'
else:
self.log.notice('File has NOT been migrated:', opFile.LFN)
now = datetime.datetime.utcnow().replace(microsecond=0)
extraDelay = datetime.timedelta(minutes=20)
self.request.NotBefore = now + extraDelay
def printSEInfo(voName):
result = Resources.getStorageElements(vo=voName)
if not result['OK']:
gLogger.error('Failed to get SE information')
DIRACExit(-1)
seDict = result['Value']
fields = ('SE', 'Status', 'Protocols', 'Aliases')
records = []
for se in seDict:
seObject = StorageElement(se)
result = seObject.status()
status = []
for statusType in ['Write', 'Read']:
if result[statusType]:
status.append(statusType)
if status:
status = '/'.join(status)
else:
status = "InActive"
records.append((se, status, ",".join(seDict[se]["Protocols"]),
",".join(seDict[se]["Aliases"])))
gLogger.notice(
printTable(fields, records, printOut=False, columnSeparator=' '))
return S_OK()
def _downloadFromBestSE(self, lfn, reps, guid):
""" Download a local copy of a single LFN from a list of Storage Elements.
This is used as a last resort to attempt to retrieve the file.
"""
self.log.verbose(
"Attempting to download file from all SEs (%s):" % ','.join(reps),
lfn)
diskSEs = set()
tapeSEs = set()
for seName in reps:
seStatus = StorageElement(seName).status()
# FIXME: This is simply terrible - this notion of "DiskSE" vs "TapeSE" should NOT be used here!
if seStatus['Read'] and seStatus['DiskSE']:
diskSEs.add(seName)
elif seStatus['Read'] and seStatus['TapeSE']:
tapeSEs.add(seName)
for seName in list(diskSEs) + list(tapeSEs):
if seName in diskSEs or _isCached(lfn, seName):
# On disk or cached from tape
result = self._downloadFromSE(lfn, seName, reps, guid)
if result['OK']:
return result
else:
self.log.error(
"Download failed",
"Tried downloading %s from SE %s: %s" %
(lfn, seName, result['Message']))
return S_ERROR("Unable to download the file from any SE")
def __prepareFile(self, se, pfn):
"""proxied prepare file"""
res = self.__prepareSecurityDetails()
if not res["OK"]:
return res
# Clear the local cache
getFileDir = "%s/getFile" % BASE_PATH
if not os.path.exists(getFileDir):
os.mkdir(getFileDir)
# Get the file to the cache
try:
storageElement = StorageElement(se)
except AttributeError as x:
errStr = "prepareFile: Exception while instantiating the Storage Element."
gLogger.exception(errStr, se, str(x))
return S_ERROR(errStr)
res = returnSingleResult(
storageElement.getFile(pfn, localPath="%s/getFile" % BASE_PATH))
if not res["OK"]:
gLogger.error("prepareFile: Failed to get local copy of file.",
res["Message"])
return res
return S_OK()
def printSEInfo(voName):
fields = ('SE', 'Status', 'Protocols', 'Aliases')
records = []
for se in DMSHelpers(voName).getStorageElements(): # this will get the full list of SEs, not only the vo's ones.
seObject = StorageElement(se)
if not (seObject.vo and voName in seObject.vo.strip().split(',') or not seObject.voName):
continue
result = seObject.status()
status = []
for statusType in ['Write', 'Read']:
if result[statusType]:
status.append(statusType)
if status:
status = '/'.join(status)
else:
status = "InActive"
records.append((se, status,
",".join([seProtocol['Protocol'] for seProtocol in seObject.protocolOptions])))
gLogger.notice(printTable(fields, records, printOut=False, columnSeparator=' '))
return S_OK()
def getRegisterOperation(self,
opFile,
targetSE,
type='RegisterFile',
catalog=None):
""" add RegisterReplica operation for file
:param ~DIRAC.RequestManagementSystem.Client.File.File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = StorageElement(targetSE).getURL(
opFile.LFN, protocol=self.registrationProtocols).get(
'Value', {}).get('Successful', {}).get(opFile.LFN)
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
return registerOperation
def __removeStorageDirectory(self, directory, storageElement):
""" wipe out all contents from :directory: at :storageElement:
:param self: self reference
:param str directory: path
:param str storageElement: SE name
"""
self.log.info('Removing the contents of %s at %s' %
(directory, storageElement))
se = StorageElement(storageElement)
res = returnSingleResult(se.exists(directory))
if not res['OK']:
self.log.error("Failed to obtain existance of directory",
res['Message'])
return res
exists = res['Value']
if not exists:
self.log.info("The directory %s does not exist at %s " %
(directory, storageElement))
return S_OK()
res = returnSingleResult(se.removeDirectory(directory, recursive=True))
if not res['OK']:
self.log.error("Failed to remove storage directory",
res['Message'])
return res
self.log.info(
"Successfully removed %d files from %s at %s" %
(res['Value']['FilesRemoved'], directory, storageElement))
return S_OK()
def __uploadFile(self, se, pfn):
""" proxied upload file """
res = self.__prepareSecurityDetails()
if not res['OK']:
return res
# Put file to the SE
try:
storageElement = StorageElement(se)
except AttributeError as x:
errStr = "__uploadFile: Exception while instantiating the Storage Element."
gLogger.exception(errStr, se, str(x))
return S_ERROR(errStr)
putFileDir = "%s/putFile" % BASE_PATH
localFileName = "%s/%s" % (putFileDir, os.path.basename(pfn))
res = returnSingleResult(storageElement.putFile({pfn: localFileName}))
if not res['OK']:
gLogger.error("prepareFile: Failed to put local file to storage.", res['Message'])
# Clear the local cache
try:
gLogger.debug("Removing temporary file", localFileName)
os.remove(localFileName)
except Exception as x:
gLogger.exception("Failed to remove local file", localFileName, x)
return res
def getFilesToStage( lfnList ):
""" Utility that returns out of a list of LFNs those files that are offline,
and those for which at least one copy is online
"""
onlineLFNs = set()
offlineLFNsDict = {}
if not lfnList:
return S_OK( {'onlineLFNs':list( onlineLFNs ), 'offlineLFNs': offlineLFNsDict} )
dm = DataManager()
lfnListReplicas = dm.getActiveReplicas( lfnList, getUrl = False )
if not lfnListReplicas['OK']:
return lfnListReplicas
seToLFNs = dict()
if lfnListReplicas['Value']['Failed']:
return S_ERROR( "Failures in getting replicas" )
for lfn, ld in lfnListReplicas['Value']['Successful'].iteritems():
for se in ld:
seToLFNs.setdefault( se, list() ).append( lfn )
failed = {}
for se, lfnsInSEList in seToLFNs.iteritems():
fileMetadata = StorageElement( se ).getFileMetadata( lfnsInSEList )
if not fileMetadata['OK']:
failed[se] = dict.fromkeys( lfnsInSEList, fileMetadata['Message'] )
else:
failed[se] = fileMetadata['Value']['Failed']
# is there at least one online?
for lfn, mDict in fileMetadata['Value']['Successful'].iteritems():
if mDict['Cached']:
onlineLFNs.add( lfn )
# If the file was found staged, ignore possible errors, but print out errors
if failed:
for se, seFailed in failed.items():
gLogger.error( "Errors when getting files metadata", 'at %s' % se )
for lfn, reason in seFailed.items():
gLogger.info( '%s: %s' % ( lfn, reason ) )
if lfn in onlineLFNs:
failed[se].pop( lfn )
if not failed[se]:
failed.pop( se )
if failed:
return S_ERROR( 'Could not get metadata for %d files' % \
len( set( [lfn for lfnList in failed.values() for lfn in lfnList] ) ) )
offlineLFNs = set( lfnList ) - onlineLFNs
for offlineLFN in offlineLFNs:
ses = lfnListReplicas['Value']['Successful'][offlineLFN].keys()
random.shuffle( ses )
se = ses[0]
offlineLFNsDict.setdefault( se, list() ).append( offlineLFN )
return S_OK( {'onlineLFNs':list( onlineLFNs ), 'offlineLFNs': offlineLFNsDict} )
def _downloadFromSE(self, lfn, seName, reps, guid):
""" Download a local copy from the specified Storage Element.
"""
if not lfn:
return S_ERROR(
"LFN not specified: assume file is not at this site")
self.log.verbose("Attempting to download file %s from %s:" %
(lfn, seName))
downloadDir = self.__getDownloadDir()
fileName = os.path.basename(lfn)
for localFile in (os.path.join(os.getcwd(), fileName),
os.path.join(downloadDir, fileName)):
if os.path.exists(localFile):
self.log.info("File %s already exists locally as %s" %
(fileName, localFile))
fileDict = {
'turl': 'LocalData',
'protocol': 'LocalData',
'se': seName,
'pfn': reps[seName],
'guid': guid,
'path': localFile
}
return S_OK(fileDict)
localFile = os.path.join(downloadDir, fileName)
result = StorageElement(seName).getFile(lfn, localPath=downloadDir)
if not result['OK']:
self.log.warn('Problem getting %s from %s:\n%s' %
(lfn, seName, result['Message']))
return result
if lfn in result['Value']['Failed']:
self.log.warn('Problem getting %s from %s:\n%s' %
(lfn, seName, result['Value']['Failed'][lfn]))
return S_ERROR(result['Value']['Failed'][lfn])
if lfn not in result['Value']['Successful']:
self.log.warn("%s got from %s not in Failed nor Successful???\n" %
(lfn, seName))
return S_ERROR("Return from StorageElement.getFile() incomplete")
if os.path.exists(localFile):
self.log.verbose("File %s successfully downloaded locally to %s" %
(lfn, localFile))
fileDict = {
'turl': 'Downloaded',
'protocol': 'Downloaded',
'se': seName,
'pfn': reps[seName],
'guid': guid,
'path': localFile
}
return S_OK(fileDict)
else:
self.log.warn(
'File does not exist in local directory after download')
return S_ERROR(
'OK download result but file missing in current directory')
def _isCached(lfn, seName):
result = StorageElement(seName).getFileMetadata(lfn)
if not result['OK']:
return False
if lfn in result['Value']['Failed']:
return False
metadata = result['Value']['Successful'][lfn]
return metadata.get('Cached', metadata['Accessible'])
def __cache(self, storageElement):
"""Retrieve cache size for SE"""
if storageElement not in self.storageElementCache:
diskCacheTB = float(
StorageElement(storageElement).options.get("DiskCacheTB", 1.0))
self.storageElementCache[
storageElement] = diskCacheTB * 1000.0 / THROTTLING_STEPS
return self.storageElementCache[storageElement]
def __updateSharedSESites( self, jobState, stageSite, stagedLFNs, opData ):
siteCandidates = opData[ 'SiteCandidates' ]
seStatus = {}
result = jobState.getManifest()
if not result['OK']:
return result
manifest = result['Value']
vo = manifest.getOption( 'VirtualOrganization' )
for siteName in siteCandidates:
if siteName == stageSite:
continue
self.jobLog.verbose( "Checking %s for shared SEs" % siteName )
siteData = siteCandidates[ siteName ]
result = getSEsForSite( siteName )
if not result[ 'OK' ]:
continue
closeSEs = result[ 'Value' ]
diskSEs = []
for seName in closeSEs:
# If we don't have the SE status get it and store it
if seName not in seStatus:
seObj = StorageElement( seName, vo = vo )
result = seObj.getStatus()
if not result['OK' ]:
self.jobLog.error( "Cannot retrieve SE %s status: %s" % ( seName, result[ 'Message' ] ) )
continue
seStatus[ seName ] = result[ 'Value' ]
# get the SE status from mem and add it if its disk
status = seStatus[ seName ]
if status['Read'] and status['DiskSE']:
diskSEs.append( seName )
self.jobLog.verbose( "Disk SEs for %s are %s" % ( siteName, ", ".join( diskSEs ) ) )
# Hell again to the dev of this crappy value of value of successful of ...
lfnData = opData['Value']['Value']['Successful']
for seName in stagedLFNs:
# If the SE is not close then skip it
if seName not in closeSEs:
continue
for lfn in stagedLFNs[ seName ]:
self.jobLog.verbose( "Checking %s for %s" % ( seName, lfn ) )
# I'm pretty sure that this cannot happen :P
if lfn not in lfnData:
continue
# Check if it's already on disk at the site
onDisk = False
for siteSE in lfnData[ lfn ]:
if siteSE in diskSEs:
self.jobLog.verbose( "%s on disk for %s" % ( lfn, siteSE ) )
onDisk = True
# If not on disk, then update!
if not onDisk:
self.jobLog.verbose( "Setting LFN to disk for %s" % ( seName ) )
siteData[ 'disk' ] += 1
siteData[ 'tape' ] -= 1
return S_OK()
def bulkRemoval(self, toRemoveDict, targetSE):
""" bulk removal of lfns from :targetSE:
:param dict toRemoveDict: { lfn : opFile, ... }
:param str targetSE: target SE name
"""
bulkRemoval = StorageElement(targetSE).removeFile(toRemoveDict)
return bulkRemoval
def resolvePFNMissing(self, problematicDict):
""" This takes the problematic dictionary returned by the integrity DB and resolved the PFNMissing prognosis
"""
pfn = problematicDict['PFN']
se = problematicDict['SE']
lfn = problematicDict['LFN']
fileID = problematicDict['FileID']
res = returnSingleResult(self.fc.exists(lfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if not res['Value']:
gLogger.info("PFNMissing file (%d) no longer exists in catalog" %
fileID)
return self.__updateCompletedFiles('PFNMissing', fileID)
res = returnSingleResult(StorageElement(se).exists(pfn))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if res['Value']:
gLogger.info("PFNMissing replica (%d) is no longer missing" %
fileID)
return self.__updateReplicaToChecked(problematicDict)
gLogger.info("PFNMissing replica (%d) does not exist" % fileID)
res = returnSingleResult(self.fc.getReplicas(lfn, allStatus=True))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
replicas = res['Value']
seSite = se.split('_')[0].split('-')[0]
found = False
print replicas
for replicaSE in replicas.keys():
if re.search(seSite, replicaSE):
found = True
problematicDict['SE'] = replicaSE
se = replicaSE
if not found:
gLogger.info(
"PFNMissing replica (%d) is no longer registered at SE. Resolved."
% fileID)
return self.__updateCompletedFiles('PFNMissing', fileID)
gLogger.info(
"PFNMissing replica (%d) does not exist. Removing from catalog..."
% fileID)
res = returnSingleResult(self.fc.removeReplica({lfn: problematicDict}))
if not res['OK']:
return self.__returnProblematicError(fileID, res)
if len(replicas) == 1:
gLogger.info(
"PFNMissing replica (%d) had a single replica. Updating prognosis"
% fileID)
return self.changeProblematicPrognosis(fileID, 'LFNZeroReplicas')
res = self.dm.replicateAndRegister(problematicDict['LFN'], se)
if not res['OK']:
return self.__returnProblematicError(fileID, res)
# If we get here the problem is solved so we can update the integrityDB
return self.__updateCompletedFiles('PFNMissing', fileID)
def __getSEStatus( self, seName ):
result = self.__SEStatus.get( seName )
if result == False:
seObj = StorageElement( seName )
result = seObj.getStatus()
if not result[ 'OK' ]:
return result
self.__SEStatus.add( seName, 600, result )
return result
def removeRegisteredFiles(self, filesNewlyRegistered, registeredFiles, allUnmigratedFilesMeta):
"""
Remove successfuly registered files (newly, or in Registered status in the DB)
from the OnlineStorage
:param filesNewlyCopied: [lfns] of files newly copied
:param copiedFiles: {lfn:RIDb metadata} of files that were in Copied state.
:param allUnmigratedFilesMeta: {lfn:RI Db metadata} for all lfns non migrated at
the beginning of the loop.
:return: {lfn:True} for successfuly registered lfns
"""
if filesNewlyRegistered or registeredFiles:
self.log.info("Attempting to remove %s newly registered and %s previously registered files" %
(len(filesNewlyRegistered), len(registeredFiles)))
else:
self.log.info("No files to be removed")
# Update registeredFiles to also contain the newly registered files
registeredFiles.update(dict((lfn, allUnmigratedFilesMeta[lfn]) for lfn in filesNewlyRegistered))
onlineSE = StorageElement('OnlineRunDB')
# Try to them them all
res = onlineSE.removeFile(registeredFiles)
filesNewlyRemoved = {}
failedRemove = {}
if not res['OK']:
self.log.error("Completely failed to remove successfully registered files.", res['Message'])
failedRemove = dict((lfn, res['Message']) for lfn in registeredFiles)
else:
filesNewlyRemoved = res['Value']['Successful']
failedRemove = res['Value']['Failed']
gMonitor.addMark("ErrorRemove", len(failedRemove))
for lfn, reason in failedRemove.iteritems():
self.log.error("Failed to remove lfn. Setting to Registered", "%s: %s" % (lfn, reason))
res = self.rawIntegrityDB.setFileStatus(lfn, 'Registered')
if not res['OK']:
self.log.error("Error setting file status to Registered", "%s: %s" % (lfn, res['Message']))
now = datetime.datetime.utcnow()
for lfn in filesNewlyRemoved:
self.log.info("Successfully removed %s from the Online storage. Setting it to Done" % lfn)
res = self.rawIntegrityDB.setFileStatus(lfn, 'Done')
if not res['OK']:
self.log.error("Error setting file status to Done", "%s: %s" % (lfn, res['Message']))
else:
# SubmitTime is ALREADY a datetime since it is declared as such in the DB.
submitTime = allUnmigratedFilesMeta[lfn]['SubmitTime']
migrationTime = (now - submitTime).total_seconds()
gMonitor.addMark("MigrationTime", migrationTime)
fileSizeMB = allUnmigratedFilesMeta[lfn]['Size'] / (1024 * 1024.0)
gMonitor.addMark("MigrationRate", fileSizeMB / migrationTime)
return filesNewlyRemoved
def finalize(self):
""" register successfully transferred files """
if self.Status not in FTSJob.FINALSTATES:
return S_OK()
if not len(self):
return S_ERROR("Empty job in finalize")
startTime = time.time()
targetSE = StorageElement(self.TargetSE)
toRegister = [
ftsFile for ftsFile in self if ftsFile.Status == "Finished"
]
toRegisterDict = {}
for ftsFile in toRegister:
pfn = returnSingleResult(
targetSE.getURL(ftsFile.LFN, protocol='srm'))
if pfn["OK"]:
pfn = pfn["Value"]
toRegisterDict[ftsFile.LFN] = {"PFN": pfn, "SE": self.TargetSE}
else:
self._log.error("Error getting SRM URL", pfn['Message'])
if toRegisterDict:
self._regTotal += len(toRegisterDict)
register = self._fc.addReplica(toRegisterDict)
self._regTime += time.time() - startTime
if not register["OK"]:
self._log.error('Error registering replica',
register['Message'])
for ftsFile in toRegister:
ftsFile.Error = "AddCatalogReplicaFailed"
return register
register = register["Value"]
self._regSuccess += len(register.get('Successful', {}))
if self._regSuccess:
self._log.info('Successfully registered %d replicas' %
self._regSuccess)
failedFiles = register.get("Failed", {})
errorReason = {}
for lfn, reason in failedFiles.items():
errorReason.setdefault(str(reason), []).append(lfn)
for reason in errorReason:
self._log.error(
'Error registering %d replicas' % len(errorReason[reason]),
reason)
for ftsFile in toRegister:
if ftsFile.LFN in failedFiles:
ftsFile.Error = "AddCatalogReplicaFailed"
else:
statuses = set([ftsFile.Status for ftsFile in self])
self._log.warn( "No replicas to register for FTSJob (%s) - Files status: '%s'" % \
( self.Status, ','.join( sorted( statuses ) ) ) )
return S_OK()
def setUp(self):
self.numberOfFiles = 1
self.storageElement = StorageElement(storageElementToTest)
self.localSourceFile = fileToTest
self.localFileSize = getSize(self.localSourceFile)
self.destDirectory = lfnDirToTest
# destinationDir = returnSingleResult( self.storageElement.getURL( self.destDirectory ) )['Value']
destinationDir = self.destDirectory
res = self.storageElement.createDirectory(destinationDir)
self.assertTrue(res['OK'])
def sortSEs(ses):
seSvcClass = {}
for se in ses:
if len(se.split(',')) != 1:
return sorted(ses)
if se not in seSvcClass:
seSvcClass[se] = StorageElement(se).getStatus()['Value']['DiskSE']
diskSEs = [se for se in ses if seSvcClass[se]]
tapeSEs = [se for se in ses if se not in diskSEs]
return sorted(diskSEs) + sorted(tapeSEs)
def sortSEs(ses):
""" Returnes an ordered list of SEs, disk first """
seSvcClass = {}
for se in ses:
if len(se.split(',')) != 1:
return sorted(ses)
if se not in seSvcClass:
seSvcClass[se] = StorageElement(se).status()['DiskSE']
diskSEs = [se for se in ses if seSvcClass[se]]
tapeSEs = [se for se in ses if se not in diskSEs]
return sorted(diskSEs) + sorted(tapeSEs)
def doNew(self, masterParams=None):
"""
Gets the total and the free disk space of a DIPS storage element that
is found in the CS and inserts the results in the SpaceTokenOccupancyCache table
of ResourceManagementDB database.
"""
if masterParams is not None:
elementName = masterParams
else:
elementName = self._prepareCommand()
if not elementName['OK']:
return elementName
se = StorageElement(elementName)
elementURL = se.getStorageParameters(protocol="dips")
if elementURL['OK']:
elementURL = se.getStorageParameters(
protocol="dips")['Value']['URLBase']
else:
gLogger.verbose("Not a DIPS storage element, skipping...")
return S_OK()
self.rpc = RPCClient(elementURL, timeout=120)
free = self.rpc.getFreeDiskSpace("/")
if not free['OK']:
return free
free = free['Value']
total = self.rpc.getTotalDiskSpace("/")
if not total['OK']:
return total
total = total['Value']
if free and free < 1:
free = 1
if total and total < 1:
total = 1
result = self.rsClient.addOrModifySpaceTokenOccupancyCache(
endpoint=elementURL,
lastCheckTime=datetime.utcnow(),
free=free,
total=total,
token=elementName)
if not result['OK']:
return result
return S_OK()
def setTargetSE( self, se ):
""" set target SE
:param self: self reference
:param str se: target SE name
"""
if se == self.sourceSE:
return S_ERROR( "TargetSE is SourceSE" )
self.targetSE = se
self.oTargetSE = StorageElement( self.targetSE )
return self.__checkTargetSE()
def setUp(self):
self.numberOfFiles = 1
self.storageElement = StorageElement(storageElementToTest)
self.localSourceFile = "/etc/group"
self.localFileSize = getSize(self.localSourceFile)
self.destDirectory = "/lhcb/test/unit-test/TestStorageElement"
destinationDir = returnSingleResult(
self.storageElement.getPfnForLfn(self.destDirectory))
res = self.storageElement.createDirectory(destinationDir,
singleDirectory=True)
self.assert_(res['OK'])
def _issuePrestageRequests(self, storageElement, seReplicaIDs,
allReplicaInfo):
""" Make the request to the SE and update the DB
"""
# Since we are in a give SE, the lfn is a unique key
lfnRepIDs = {}
for replicaID in seReplicaIDs:
lfn = allReplicaInfo[replicaID]['LFN']
lfnRepIDs[lfn] = replicaID
# Now issue the prestage requests for the remaining replicas
stageRequestMetadata = {}
updatedLfnIDs = []
if lfnRepIDs:
gLogger.info(
"StageRequest._issuePrestageRequests: Submitting %s stage requests for %s."
% (len(lfnRepIDs), storageElement))
res = StorageElement(storageElement).prestageFile(
lfnRepIDs, lifetime=self.pinLifetime)
gLogger.debug(
"StageRequest._issuePrestageRequests: StorageElement.prestageStorageFile: res=",
res)
#Daniela: fishy result from ReplicaManager!!! Should NOT return OK
#res= {'OK': True, 'Value': {'Successful': {}, 'Failed': {'srm://srm-lhcb.cern.ch/castor/cern.ch/grid/lhcb/data/2010/RAW/EXPRESS/LHCb/COLLISION10/71476/071476_0000000241.raw': ' SRM2Storage.__gfal_exec: Failed to perform gfal_prestage.[SE][BringOnline][SRM_INVALID_REQUEST] httpg://srm-lhcb.cern.ch:8443/srm/managerv2: User not able to access specified space token\n'}}}
#res= {'OK': True, 'Value': {'Successful': {'srm://gridka-dCache.fzk.de/pnfs/gridka.de/lhcb/data/2009/RAW/FULL/LHCb/COLLISION09/63495/063495_0000000001.raw': '-2083846379'}, 'Failed': {}}}
if not res['OK']:
gLogger.error(
"StageRequest._issuePrestageRequests: Completely failed to submit stage requests for replicas.",
res['Message'])
else:
for lfn, requestID in res['Value']['Successful'].items():
if not stageRequestMetadata.has_key(requestID):
stageRequestMetadata[requestID] = []
stageRequestMetadata[requestID].append(lfnRepIDs[lfn])
updatedLfnIDs.append(lfnRepIDs[lfn])
if stageRequestMetadata:
gLogger.info(
"StageRequest._issuePrestageRequests: %s stage request metadata to be updated."
% len(stageRequestMetadata))
res = self.stagerClient.insertStageRequest(stageRequestMetadata,
self.pinLifetime)
if not res['OK']:
gLogger.error(
"StageRequest._issuePrestageRequests: Failed to insert stage request metadata.",
res['Message'])
return res
res = self.stagerClient.updateReplicaStatus(
updatedLfnIDs, 'StageSubmitted')
if not res['OK']:
gLogger.error(
"StageRequest._issuePrestageRequests: Failed to insert replica status.",
res['Message'])
return
def test_putFile(setuptest):
""" Copy a file """
# XXX: this is not good !
# The mock I use for S3 seem to have a bug uploading files
# with presigned URL. So for the time being, I upload directly,
# but this should be checked
# https://github.com/adobe/S3Mock/issues/219
se = StorageElement('S3-DIRECT')
res = se.putFile(putFile)
assert res['OK'], res
for lfn in putFile:
assert lfn in res['Value']['Successful']
def getFilesToStage(lfnList):
""" Utility that returns out of a list of LFNs those files that are offline,
and those for which at least one copy is online
"""
onlineLFNs = set()
offlineLFNsDict = {}
if not lfnList:
return S_OK({
'onlineLFNs': list(onlineLFNs),
'offlineLFNs': offlineLFNsDict
})
dm = DataManager()
lfnListReplicas = dm.getActiveReplicas(lfnList)
if not lfnListReplicas['OK']:
return lfnListReplicas
seObjectsDict = dict()
seToLFNs = dict()
if lfnListReplicas['Value']['Failed']:
return S_ERROR("Failures in getting replicas")
for lfn, ld in lfnListReplicas['Value']['Successful'].iteritems():
for se, _ in ld.iteritems():
seObjectsDict.setdefault(se, StorageElement(se))
seToLFNs.setdefault(se, list()).append(lfn)
for se, lfnsInSEList in seToLFNs.iteritems():
fileMetadata = seObjectsDict[se].getFileMetadata(lfnsInSEList)
if not fileMetadata['OK']:
return fileMetadata
if fileMetadata['Value']['Failed']:
return S_ERROR("Failures in getting file metadata")
# is there at least one online?
for lfn, mDict in fileMetadata['Value']['Successful'].iteritems():
if mDict['Cached']:
onlineLFNs.add(lfn)
offlineLFNs = set(lfnList).difference(onlineLFNs)
for offlineLFN in offlineLFNs:
ses = lfnListReplicas['Value']['Successful'][offlineLFN].keys()
random.shuffle(ses)
se = ses[0]
offlineLFNsDict.setdefault(se, list()).append(offlineLFN)
return S_OK({
'onlineLFNs': list(onlineLFNs),
'offlineLFNs': offlineLFNsDict
})
def __uploadFile(self, se, pfn):
res = self.__prepareSecurityDetails()
if not res['OK']:
return res
# Put file to the SE
try:
storageElement = StorageElement(se)
except AttributeError, x:
errStr = "__uploadFile: Exception while instantiating the Storage Element."
gLogger.exception(errStr, se, str(x))
return S_ERROR(errStr)
def _downloadFromSE(self, lfn, seName, reps, guid):
"""Download a local copy from the specified Storage Element."""
if not lfn:
return S_ERROR(
"LFN not specified: assume file is not at this site")
self.log.verbose("Attempting to download file",
"%s from %s:" % (lfn, seName))
downloadDir = self.__getDownloadDir()
fileName = os.path.basename(lfn)
for localFile in (os.path.join(os.getcwd(), fileName),
os.path.join(downloadDir, fileName)):
if os.path.exists(localFile):
self.log.info("File already exists locally",
"%s as %s" % (fileName, localFile))
fileDict = {
"turl": "LocalData",
"protocol": "LocalData",
"se": seName,
"pfn": reps[seName],
"guid": guid,
"path": localFile,
}
return S_OK(fileDict)
localFile = os.path.join(downloadDir, fileName)
result = returnSingleResult(
StorageElement(seName).getFile(lfn, localPath=downloadDir))
if not result["OK"]:
self.log.warn("Problem getting lfn",
"%s from %s:\n%s" % (lfn, seName, result["Message"]))
self.__cleanFailedFile(lfn, downloadDir)
return result
if os.path.exists(localFile):
self.log.verbose("File successfully downloaded locally",
"(%s to %s)" % (lfn, localFile))
fileDict = {
"turl": "Downloaded",
"protocol": "Downloaded",
"se": seName,
"pfn": reps[seName],
"guid": guid,
"path": localFile,
}
return S_OK(fileDict)
else:
self.log.warn(
"File does not exist in local directory after download")
return S_ERROR(
"OK download result but file missing in current directory")
