def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE):
""" Set a request to upload job log files from the output sandbox
Changed to be similar to LHCb createLogUploadRequest
using LHCb LogUpload Request and Removal Request
"""
self.log.info('Setting log upload request for %s at %s' %(targetSE, logFileLFN))
request = self._getRequestContainer()
logUpload = Operation()
logUpload.Type = "LogUpload"
logUpload.TargetSE = targetSE
upFile = File()
upFile.LFN = logFileLFN
logUpload.addFile( upFile )
logRemoval = Operation()
logRemoval.Type = 'RemoveFile'
logRemoval.TargetSE = uploadedSE
logRemoval.addFile( upFile )
request.addOperation ( logUpload )
request.addOperation ( logRemoval )
self.workflow_commons['Request'] = request
return S_OK()
def createRequest(self, requestName, lfnChunk):
"""Create the Request."""
request = Request()
request.RequestName = requestName
replicate = Operation()
replicate.Type = "ReplicateAndRegister"
replicate.TargetSE = self.switches.get("TargetSE")
self.addLFNs(replicate, lfnChunk, addPFN=True)
request.addOperation(replicate)
if self.switches.get("CheckMigration"):
checkMigration = Operation()
checkMigration.Type = "CheckMigration"
checkMigration.TargetSE = self.switches.get("TargetSE")
self.addLFNs(checkMigration, lfnChunk, addPFN=True)
request.addOperation(checkMigration)
removeReplicas = Operation()
removeReplicas.Type = "RemoveReplica"
removeReplicas.TargetSE = ",".join(self.switches.get("SourceSE", []))
self.addLFNs(removeReplicas, lfnChunk)
request.addOperation(removeReplicas)
return request
def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE):
""" Set a request to upload job log files from the output sandbox
Changed to be similar to LHCb createLogUploadRequest
using LHCb LogUpload Request and Removal Request
"""
self.log.info('Setting log upload request for %s at %s' %
(targetSE, logFileLFN))
request = self._getRequestContainer()
logUpload = Operation()
logUpload.Type = "LogUpload"
logUpload.TargetSE = targetSE
upFile = File()
upFile.LFN = logFileLFN
logUpload.addFile(upFile)
logRemoval = Operation()
logRemoval.Type = 'RemoveFile'
logRemoval.TargetSE = uploadedSE
logRemoval.addFile(upFile)
request.addOperation(logUpload)
request.addOperation(logRemoval)
self.workflow_commons['Request'] = request
return S_OK()
def createRequest(self, requestName, lfnChunk):
"""Create the Request."""
request = Request()
request.RequestName = requestName
replicate = Operation()
replicate.Type = 'ReplicateAndRegister'
replicate.TargetSE = self.switches.get('TargetSE')
self.addLFNs(replicate, lfnChunk, addPFN=True)
request.addOperation(replicate)
if self.switches.get('CheckMigration'):
checkMigration = Operation()
checkMigration.Type = 'CheckMigration'
checkMigration.TargetSE = self.switches.get('TargetSE')
self.addLFNs(checkMigration, lfnChunk, addPFN=True)
request.addOperation(checkMigration)
removeReplicas = Operation()
removeReplicas.Type = 'RemoveReplica'
removeReplicas.TargetSE = ','.join(self.switches.get('SourceSE', []))
self.addLFNs(removeReplicas, lfnChunk)
request.addOperation(removeReplicas)
return request
def buildRequest( self, owner, group, sourceSE, targetSE1, targetSE2 ):
files = self.files( owner, group )
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
putFile = File()
putFile.LFN = lfn
putFile.PFN = fname
putFile.Checksum = checksum
putFile.ChecksumType = "adler32"
putFile.Size = size
putFile.GUID = guid
putAndRegister.addFile( putFile )
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = "%s,%s" % ( targetSE1, targetSE2 )
for fname, lfn, size, checksum, guid in files:
repFile = File()
repFile.LFN = lfn
repFile.Size = size
repFile.Checksum = checksum
repFile.ChecksumType = "adler32"
replicateAndRegister.addFile( repFile )
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
removeReplica.addFile( File( {"LFN": lfn } ) )
removeFile = Operation()
removeFile.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFile.addFile( File( {"LFN": lfn } ) )
removeFileInit = Operation()
removeFileInit.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFileInit.addFile( File( {"LFN": lfn } ) )
req = Request()
req.addOperation( removeFileInit )
req.addOperation( putAndRegister )
req.addOperation( replicateAndRegister )
req.addOperation( removeReplica )
req.addOperation( removeFile )
return req
def buildRequest(self, owner, group, sourceSE, targetSE1, targetSE2):
files = self.files(owner, group)
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
putFile = File()
putFile.LFN = lfn
putFile.PFN = fname
putFile.Checksum = checksum
putFile.ChecksumType = "adler32"
putFile.Size = size
putFile.GUID = guid
putAndRegister.addFile(putFile)
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = "%s,%s" % (targetSE1, targetSE2)
for fname, lfn, size, checksum, guid in files:
repFile = File()
repFile.LFN = lfn
repFile.Size = size
repFile.Checksum = checksum
repFile.ChecksumType = "adler32"
replicateAndRegister.addFile(repFile)
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
removeReplica.addFile(File({"LFN": lfn}))
removeFile = Operation()
removeFile.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFile.addFile(File({"LFN": lfn}))
removeFileInit = Operation()
removeFileInit.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFileInit.addFile(File({"LFN": lfn}))
req = Request()
req.addOperation(removeFileInit)
req.addOperation(putAndRegister)
req.addOperation(replicateAndRegister)
req.addOperation(removeReplica)
req.addOperation(removeFile)
return req
def __submitRMSOp(self, target_se, lfns_chunk_dict, whichRMSOp='ReplicateAndRegister' ):
""" target_se : SE name to which to replicate
lfns_chunk_dict : LFNS dict with 100 lfns as key andeEach lfn has 'Size', 'Checksum'
whichRMSOp: Choose from RMP operation - ReplicateAndRegister, ReplicateAndRemove, PutAndRegister
"""
## Setup request
request = Request()
request.RequestName = "DDM_"+ str(target_se) + datetime.datetime.now().strftime("_%Y%m%d_%H%M%S")
myOp = Operation()
myOp.Type = whichRMSOp
myOp.TargetSE = target_se
## Add LFNS to operations
for lfn in lfns_chunk_dict.keys():
opFile = File()
opFile.LFN = lfn
opFile.Size = lfns_chunk_dict[lfn]['Size']
if "Checksum" in lfns_chunk_dict[lfn]:
opFile.Checksum = lfns_chunk_dict[lfn]['Checksum']
opFile.ChecksumType = 'ADLER32'
## Add file to operation
myOp.addFile( opFile )
request.addOperation( myOp )
reqClient = ReqClient()
putRequest = reqClient.putRequest( request )
if not putRequest["OK"]:
gLogger.error( "Unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) )
return S_ERROR("Problem submitting to RMS.")
def __insertRegisterOperation( self, request, operation, toRegister ):
""" add RegisterReplica operation
:param Request request: request instance
:param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
:param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
"""
log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )
byTarget = {}
for ftsFile in toRegister:
if ftsFile.TargetSE not in byTarget:
byTarget.setdefault( ftsFile.TargetSE, [] )
byTarget[ftsFile.TargetSE].append( ftsFile )
log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )
for target, ftsFileList in byTarget.iteritems():
log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
len( ftsFileList ) ) )
registerOperation = Operation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
targetSE = StorageElement( target )
for ftsFile in ftsFileList:
opFile = File()
opFile.LFN = ftsFile.LFN
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
if not pfn["OK"]:
continue
opFile.PFN = pfn["Value"]
registerOperation.addFile( opFile )
request.insertBefore( registerOperation, operation )
return S_OK()
def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ):
if self.getCSOption( "DelayedExternalDeletion", True ):
gLogger.info( "Setting deletion request" )
try:
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() )
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile( fileToRemove )
request.addOperation( physicalRemoval )
return ReqClient().putRequest( request )
except Exception as e:
gLogger.exception( "Exception while setting deletion request" )
return S_ERROR( "Cannot set deletion request: %s" % str( e ) )
else:
gLogger.info( "Deleting external Sandbox" )
try:
return StorageElement( SEName ).removeFile( SEPFN )
except Exception as e:
gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" )
return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def getRegisterOperation(self,
opFile,
targetSE,
type='RegisterFile',
catalog=None):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = StorageElement(targetSE).getPfnForLfn(
opFile.LFN).get('Value', {}).get('Successful', {}).get(opFile.LFN)
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
return registerOperation
def __setFileReplicationRequest( self, lfn, targetSE, fileMetaDict, sourceSE = '' ):
""" Sets a registration request.
"""
self.log.info( 'Setting replication request for %s to %s' % ( lfn, targetSE ) )
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.TargetSE = targetSE
if sourceSE:
transfer.SourceSE = sourceSE
trFile = File()
trFile.LFN = lfn
cksm = fileMetaDict.get( "Checksum", None )
cksmType = fileMetaDict.get( "ChecksumType", None )
if cksm and cksmType:
trFile.Checksum = cksm
trFile.ChecksumType = cksmType
size = fileMetaDict.get( "Size", 0 )
if size:
trFile.Size = size
guid = fileMetaDict.get( "GUID", "" )
if guid:
trFile.GUID = guid
transfer.addFile( trFile )
self.request.addOperation( transfer )
return S_OK()
def __setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE
:param list catalog: list of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get( "Checksum", "" )
regFile.ChecksumType = fileDict.get( "ChecksumType", "" )
regFile.Size = fileDict.get( "Size", 0 )
regFile.GUID = fileDict.get( "GUID", "" )
se = StorageElement( targetSE )
pfn = se.getPfnForLfn( lfn )
if not pfn["OK"]:
self.log.error( "unable to get PFN for LFN: %s" % pfn["Message"] )
return pfn
regFile.PFN = pfn["Value"]
register.addFile( regFile )
self.request.addOperation( register )
return S_OK()
def test02props( self ):
""" test properties """
# # valid values
operation = Operation()
operation.Arguments = "foobar"
self.assertEqual( operation.Arguments, "foobar", "wrong Arguments" )
operation.SourceSE = "CERN-RAW"
self.assertEqual( operation.SourceSE, "CERN-RAW", "wrong SourceSE" )
operation.TargetSE = "CERN-RAW"
self.assertEqual( operation.TargetSE, "CERN-RAW", "wrong TargetSE" )
operation.Catalog = ""
self.assertEqual( operation.Catalog, "", "wrong Catalog" )
operation.Catalog = "BookkeepingDB"
self.assertEqual( operation.Catalog, "BookkeepingDB", "wrong Catalog" )
operation.Error = "error"
self.assertEqual( operation.Error, "error", "wrong Error" )
# # wrong props
try:
operation.RequestID = "foo"
except Exception, error:
self.assertEqual( type( error ), AttributeError, "wrong exc raised" )
self.assertEqual( str( error ), "can't set attribute", "wrong exc reason" )
def __insertRegisterOperation( self, request, operation, toRegister ):
""" add RegisterReplica operation
:param Request request: request instance
:param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
:param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
"""
log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )
byTarget = {}
for ftsFile in toRegister:
if ftsFile.TargetSE not in byTarget:
byTarget.setdefault( ftsFile.TargetSE, [] )
byTarget[ftsFile.TargetSE].append( ftsFile )
log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )
for target, ftsFileList in byTarget.iteritems():
log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
len( ftsFileList ) ) )
registerOperation = Operation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
targetSE = StorageElement( target )
for ftsFile in ftsFileList:
opFile = File()
opFile.LFN = ftsFile.LFN
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
if not pfn["OK"]:
continue
opFile.PFN = pfn["Value"]
registerOperation.addFile( opFile )
request.insertBefore( registerOperation, operation )
return S_OK()
def _setFileReplicationRequest(self,
lfn,
targetSE,
fileMetaDict,
sourceSE=""):
"""Sets a registration request."""
self.log.info("Setting ReplicateAndRegister request",
"for %s to %s" % (lfn, targetSE))
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.TargetSE = targetSE
if sourceSE:
transfer.SourceSE = sourceSE
trFile = File()
trFile.LFN = lfn
cksm = fileMetaDict.get("Checksum", None)
cksmType = fileMetaDict.get("ChecksumType", self.defaultChecksumType)
if cksm and cksmType:
trFile.Checksum = cksm
trFile.ChecksumType = cksmType
size = fileMetaDict.get("Size", 0)
if size:
trFile.Size = size
guid = fileMetaDict.get("GUID", "")
if guid:
trFile.GUID = guid
transfer.addFile(trFile)
self.request.addOperation(transfer)
return S_OK()
def test02props(self):
""" test properties """
# # valid values
operation = Operation()
operation.OperationID = 1
self.assertEqual(operation.OperationID, 1, "wrong OperationID")
operation.OperationID = "1"
self.assertEqual(operation.OperationID, 1, "wrong OperationID")
operation.Arguments = "foobar"
self.assertEqual(operation.Arguments, "foobar", "wrong Arguments")
operation.SourceSE = "CERN-RAW"
self.assertEqual(operation.SourceSE, "CERN-RAW", "wrong SourceSE")
operation.TargetSE = "CERN-RAW"
self.assertEqual(operation.TargetSE, "CERN-RAW", "wrong TargetSE")
operation.Catalog = ""
self.assertEqual(operation.Catalog, "", "wrong Catalog")
operation.Catalog = "BookkeepingDB"
self.assertEqual(operation.Catalog, "BookkeepingDB", "wrong Catalog")
operation.Error = "error"
self.assertEqual(operation.Error, "error", "wrong Error")
# # wrong props
try:
operation.RequestID = "foo"
except Exception, error:
self.assertEqual(type(error), AttributeError, "wrong exc raised")
self.assertEqual(str(error), "can't set attribute",
"wrong exc reason")
def getRegisterOperation(self, opFile, targetSE, type="RegisterFile", catalog=None):
"""add RegisterReplica operation for file
:param ~DIRAC.RequestManagementSystem.Client.File.File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = (
StorageElement(targetSE)
.getURL(opFile.LFN, protocol=self.registrationProtocols)
.get("Value", {})
.get("Successful", {})
.get(opFile.LFN)
)
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
return registerOperation
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
if self.getCSOption("DelayedExternalDeletion", True):
gLogger.info("Setting deletion request")
try:
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % (
SEName, SEPFN, time.time())
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile(fileToRemove)
request.addOperation(physicalRemoval)
return ReqClient().putRequest(request)
except Exception as e:
gLogger.exception("Exception while setting deletion request")
return S_ERROR("Cannot set deletion request: %s" % str(e))
else:
gLogger.info("Deleting external Sandbox")
try:
return StorageElement(SEName).removeFile(SEPFN)
except Exception as e:
gLogger.exception(
"RM raised an exception while trying to delete a remote sandbox"
)
return S_ERROR(
"RM raised an exception while trying to delete a remote sandbox"
)
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def _replicateSourceFiles(self, request, lfns):
"""Create the replicateAndRegisterRequest.
:param request: The request to add the operation to
:param lfns: list of LFNs
"""
registerSource = Operation()
registerSource.Type = "ReplicateAndRegister"
registerSource.TargetSE = self.sourceSEs[0]
self.addLFNs(registerSource, lfns, addPFN=True)
request.addOperation(registerSource)
def __createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE):
""" Set a request to upload job log files from the output sandbox
"""
self.log.info('Setting log upload request for %s at %s' % (logFileLFN, targetSE))
logUpload = Operation()
logUpload.Type = 'LogUpload'
logUpload.TargetSE = targetSE
logFile = File()
logFile.LFN = logFileLFN
logUpload.addFile(logFile)
self.request.addOperation(logUpload)
logRemoval = Operation()
logRemoval.Type = 'RemoveFile'
logRemoval.TargetSE = uploadedSE
logRemoval.addFile(logFile)
self.request.addOperation(logRemoval)
def _setFileRemovalRequest(self, lfn, se="", pfn=""):
"""Sets a removal request for a file including all replicas."""
remove = Operation()
remove.Type = "RemoveFile"
if se:
remove.TargetSE = se
rmFile = File()
rmFile.LFN = lfn
if pfn:
rmFile.PFN = pfn
remove.addFile(rmFile)
self.request.addOperation(remove)
return S_OK()
def __setFileRemovalRequest( self, lfn, se = '', pfn = '' ):
""" Sets a removal request for a file including all replicas.
"""
remove = Operation()
remove.Type = "RemoveFile"
if se:
remove.TargetSE = se
rmFile = File()
rmFile.LFN = lfn
if pfn:
rmFile.PFN = pfn
remove.addFile( rmFile )
self.request.addOperation( remove )
return S_OK()
def checkRequestAndOp(listOfLFNs):
req = Request()
req.RequestName = "MyRequest"
op = Operation()
op.Type = "CheckMigration"
op.TargetSE = "Foo-SE"
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = "01130a%0d" % index
oFile.ChecksumType = "adler32"
op.addFile(oFile)
req.addOperation(op)
return req, op
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
"""deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = "ReplicateAndRegister"
if transBody:
try:
_requestType, requestOperation = transBody.split(";")
except AttributeError:
pass
failedTasks = []
# Do not remove sorted, we might pop elements in the loop
for taskID, task in taskDict.items():
transID = task["TransformationID"]
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = task["TargetSE"]
# If there are input files
if task.get("InputData"):
if isinstance(task["InputData"], list):
files = task["InputData"]
elif isinstance(task["InputData"], six.string_types):
files = task["InputData"].split(";")
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
result = self._assignRequestToTask(oRequest, taskDict, transID,
taskID, ownerDN, ownerGroup)
if not result["OK"]:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split(';')
except AttributeError:
pass
failedTasks = []
# Do not remove sorted, we might pop elements in the loop
for taskID, task in taskDict.iteritems():
transID = task['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = task['TargetSE']
# If there are input files
if task.get('InputData'):
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
if self.getCSOption("DelayedExternalDeletion", True):
gLogger.info("Setting deletion request")
try:
# We need the hostDN used in order to pass these credentials to the
# SandboxStoreDB..
hostCertLocation, _ = Locations.getHostCertificateAndKeyLocation(
)
hostCert = X509Certificate.X509Certificate()
hostCert.loadFromFile(hostCertLocation)
hostDN = hostCert.getSubjectDN().get("Value")
# use the host authentication to fetch the data
result = self.sandboxDB.getSandboxOwner(
SEName, SEPFN, hostDN, "hosts")
if not result["OK"]:
return result
_owner, ownerDN, ownerGroup = result["Value"]
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % (
SEName, SEPFN, time.time())
request.OwnerDN = ownerDN
request.OwnerGroup = ownerGroup
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile(fileToRemove)
request.addOperation(physicalRemoval)
return ReqClient().putRequest(request)
except Exception as e:
gLogger.exception("Exception while setting deletion request")
return S_ERROR(f"Cannot set deletion request: {e}")
else:
gLogger.info("Deleting external Sandbox")
try:
return StorageElement(SEName).removeFile(SEPFN)
except Exception:
gLogger.exception(
"RM raised an exception while trying to delete a remote sandbox"
)
return S_ERROR(
"RM raised an exception while trying to delete a remote sandbox"
)
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info('Setting registration request for %s at %s.' %
(lfn, targetSE))
if not type(catalog) == type([]):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get("Checksum", "")
regFile.ChecksumType = fileDict.get("ChecksumType",
self.defaultChecksumType)
regFile.Size = fileDict.get("Size", 0)
regFile.GUID = fileDict.get("GUID", "")
se = StorageElement(targetSE)
pfn = se.getPfnForLfn(lfn)
if not pfn["OK"] or lfn not in pfn["Value"]['Successful']:
self.log.error(
"unable to get PFN for LFN: %s" %
pfn.get('Message',
pfn.get('Value', {}).get('Failed', {}).get(lfn)))
return pfn
regFile.PFN = pfn["Value"]['Successful'][lfn]
register.addFile(regFile)
self.request.addOperation(register)
return S_OK()
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup ):
""" deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
# Do not remove sorted, we might pop elements in the loop
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
# If there are input files
if paramDict.get('InputData'):
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
self._assignRequestToTask( oRequest, taskDict, transID, taskID, ownerDN, ownerGroup )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if type( paramDict['InputData'] ) == type( [] ):
files = paramDict['InputData']
elif type( paramDict['InputData'] ) == type( '' ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
oRequest.OwnerDN = owner
oRequest.OwnerGroup = ownerGroup
isValid = gRequestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog):
"""Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info("Setting registration request",
"for %s at %s." % (lfn, targetSE))
if not isinstance(catalog, list):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get("Checksum", "")
regFile.ChecksumType = fileDict.get("ChecksumType",
self.defaultChecksumType)
regFile.Size = fileDict.get("Size", 0)
regFile.GUID = fileDict.get("GUID", "")
se = StorageElement(targetSE)
res = returnSingleResult(se.getURL(lfn,
self.registrationProtocols))
if not res["OK"]:
self.log.error("Unable to get PFN for LFN", res["Message"])
return res
regFile.PFN = res["Value"]
register.addFile(regFile)
self.request.addOperation(register)
return S_OK()
def __setReplicaRemovalRequest( self, lfn, se ):
""" Sets a removal request for a replica.
:param str lfn: LFN
:param se:
"""
if type( se ) == str:
se = ",".join( [ se.strip() for se in se.split( "," ) if se.strip() ] )
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = se
replicaToRemove = File()
replicaToRemove.LFN = lfn
removeReplica.addFile( replicaToRemove )
self.request.addOperation( removeReplica )
return S_OK()
def getRegisterOperation(self, opFile, targetSE):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = "RegisterFile"
registerOperation.TargetSE = targetSE
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = opFile.PFN
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
return registerOperation
def _setReplicaRemovalRequest(self, lfn, se):
"""Sets a removal request for a replica.
:param str lfn: LFN
:param se:
"""
if isinstance(se, str):
se = ",".join([se.strip() for se in se.split(",") if se.strip()])
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = se
replicaToRemove = File()
replicaToRemove.LFN = lfn
removeReplica.addFile(replicaToRemove)
self.request.addOperation(removeReplica)
return S_OK()
def getRegisterOperation( self, opFile, targetSE ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = "RegisterFile"
registerOperation.TargetSE = targetSE
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = opFile.PFN
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
return registerOperation
def addRegisterReplica( self, opFile, targetSE ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = "RegisterFile"
registerOperation.TargetSE = targetSE
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = opFile.PFN
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
self.request.insertAfter( registerOperation, self.operation )
return S_OK()
def addRegisterReplica(self, opFile, targetSE):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = "RegisterFile"
registerOperation.TargetSE = targetSE
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = opFile.PFN
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
self.request.insertAfter(registerOperation, self.operation)
return S_OK()
def _setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )
if not isinstance( catalog, list ):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get( "Checksum", "" )
regFile.ChecksumType = fileDict.get( "ChecksumType", self.defaultChecksumType )
regFile.Size = fileDict.get( "Size", 0 )
regFile.GUID = fileDict.get( "GUID", "" )
se = StorageElement( targetSE )
pfn = se.getURL( lfn, self.registrationProtocols )
if not pfn["OK"] or lfn not in pfn["Value"]['Successful']:
self.log.error( "Unable to get PFN for LFN", "%s" % pfn.get( 'Message', pfn.get( 'Value', {} ).get( 'Failed', {} ).get( lfn ) ) )
return pfn
regFile.PFN = pfn["Value"]['Successful'][lfn]
register.addFile( regFile )
self.request.addOperation( register )
return S_OK()
def getRegisterOperation( self, opFile, targetSE, type = 'RegisterFile', catalog = None ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = StorageElement( targetSE ).getURL( opFile.LFN, protocol = self.registrationProtocols ).get( 'Value', {} ).get( 'Successful', {} ).get( opFile.LFN )
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
return registerOperation
def test_valid_properties():
operation = Operation()
operation.Arguments = "foobar"
assert operation.Arguments == b"foobar", "wrong Arguments"
operation.SourceSE = "CERN-RAW"
assert operation.SourceSE == "CERN-RAW", "wrong SourceSE"
operation.TargetSE = "CERN-RAW"
assert operation.TargetSE == "CERN-RAW", "wrong TargetSE"
operation.Catalog = ""
assert operation.Catalog == "", "wrong Catalog"
operation.Catalog = "BookkeepingDB"
assert operation.Catalog == "BookkeepingDB", "wrong Catalog"
operation.Error = "error"
assert operation.Error == "error", "wrong Error"
toJSON = operation.toJSON()
assert toJSON["OK"]
gLogger.error( "LFN list is empty!!!" )
error = -1
continue
if len( lfnChunk ) > Operation.MAX_FILES:
gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES )
error = -1
continue
count += 1
request = Request()
request.RequestName = requestName if not multiRequests else '%s_%d' % ( requestName, count )
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = ",".join( targetSEs )
if catalog is not None:
replicateAndRegister.Catalog = catalog
for lfn in lfnChunk:
metaDict = metaDatas["Successful"][lfn]
opFile = File()
opFile.LFN = lfn
opFile.Size = metaDict["Size"]
if "Checksum" in metaDict:
# # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
opFile.Checksum = metaDict["Checksum"]
opFile.ChecksumType = "ADLER32"
replicateAndRegister.addFile( opFile )
gLogger.error("Location not known, upload to location yourself, and publish in CS manually")
return S_ERROR()
else:
lfnpath = "%s%s" % (path, os.path.basename(appTar))
res = datMan.putAndRegister(lfnpath, appTar, ops.getValue('Software/BaseStorageElement', "CERN-SRM"))
if not res['OK']:
return res
request = Request()
requestClient = ReqClient()
request.RequestName = 'copy_%s' % os.path.basename(appTar).replace(".tgz", "").replace(".tar.gz", "")
request.SourceComponent = 'ReplicateILCSoft'
copies_at = ops.getValue('Software/CopiesAt', [])
for copies in copies_at:
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.TargetSE = copies
trFile = File()
trFile.LFN = lfnpath
trFile.GUID = ""
transfer.addFile(trFile)
request.addOperation(transfer)
res = RequestValidator().validate(request)
if not res['OK']:
return res
if copies_at:
res = requestClient.putRequest(request)
if not res['OK']:
gLogger.error('Could not set replication request', res['Message'])
return S_OK('Application uploaded')
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
reqClient = ReqClient()
fc = FileCatalog()
for lfnList in breakListIntoChunks( lfns, 100 ):
oRequest = Request()
oRequest.RequestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] )
replicateAndRegister = Operation()
replicateAndRegister.Type = 'ReplicateAndRegister'
replicateAndRegister.TargetSE = targetSE
res = fc.getFileMetadata( lfnList )
if not res['OK']:
print "Can't get file metadata: %s" % res['Message']
DIRAC.exit( 1 )
if res['Value']['Failed']:
print "Could not get the file metadata of the following, so skipping them:"
for fFile in res['Value']['Failed']:
print fFile
lfnMetadata = res['Value']['Successful']
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
gLogger.error( "%s does not exist" % PFN )
DIRAC.exit( -1 )
if not os.path.isfile( PFN ):
gLogger.error( "%s is not a file" % PFN )
DIRAC.exit( -1 )
PFN = os.path.abspath( PFN )
size = os.path.getsize( PFN )
adler32 = fileAdler( PFN )
request = Request()
request.RequestName = requestName
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = targetSE
opFile = File()
opFile.LFN = LFN
opFile.PFN = PFN
opFile.Size = size
opFile.Checksum = adler32
opFile.ChecksumType = "ADLER32"
putAndRegister.addFile( opFile )
request.addOperation( putAndRegister )
reqClient = ReqClient()
putRequest = reqClient.putRequest( request )
if not putRequest["OK"]:
gLogger.error( "unable to put request '%s': %s" % ( requestName, putRequest["Message"] ) )
DIRAC.exit( -1 )
gLogger.always( "Request '%s' has been put to ReqDB for execution." % requestName )
continue
if len( lfnChunk ) > Operation.MAX_FILES:
gLogger.error( "too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES )
error = -1
continue
count += 1
request = Request()
request.RequestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] )
moveReplica = Operation()
moveReplica.Type = 'MoveReplica'
moveReplica.SourceSE = sourceSE
moveReplica.TargetSE = ",".join( targetSEs )
for lfn in lfnChunk:
metaDict = metaDatas["Successful"][lfn]
opFile = File()
opFile.LFN = lfn
opFile.Size = metaDict["Size"]
if "Checksum" in metaDict:
# # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
opFile.Checksum = metaDict["Checksum"]
opFile.ChecksumType = "ADLER32"
moveReplica.addFile( opFile )
request.addOperation( moveReplica )
def test05FTS( self ):
""" FTS state machine """
req = Request()
req.RequestName = "FTSTest"
ftsTransfer = Operation()
ftsTransfer.Type = "ReplicateAndRegister"
ftsTransfer.TargetSE = "CERN-USER"
ftsFile = File()
ftsFile.LFN = "/a/b/c"
ftsFile.Checksum = "123456"
ftsFile.ChecksumType = "Adler32"
ftsTransfer.addFile( ftsFile )
req.addOperation( ftsTransfer )
self.assertEqual( req.Status, "Waiting", "1. wrong request status: %s" % req.Status )
self.assertEqual( ftsTransfer.Status, "Waiting", "1. wrong ftsStatus status: %s" % ftsTransfer.Status )
# # scheduled
ftsFile.Status = "Scheduled"
self.assertEqual( ftsTransfer.Status, "Scheduled", "2. wrong status for ftsTransfer: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Scheduled", "2. wrong status for request: %s" % req.Status )
# # add new operation before FTS
insertBefore = Operation()
insertBefore.Type = "RegisterReplica"
insertBefore.TargetSE = "CERN-USER"
insertFile = File()
insertFile.LFN = "/a/b/c"
insertFile.PFN = "http://foo/bar"
insertBefore.addFile( insertFile )
req.insertBefore( insertBefore, ftsTransfer )
self.assertEqual( insertBefore.Status, "Waiting", "3. wrong status for insertBefore: %s" % insertBefore.Status )
self.assertEqual( ftsTransfer.Status, "Scheduled", "3. wrong status for ftsStatus: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Waiting", "3. wrong status for request: %s" % req.Status )
# # prev done
insertFile.Status = "Done"
self.assertEqual( insertBefore.Status, "Done", "4. wrong status for insertBefore: %s" % insertBefore.Status )
self.assertEqual( ftsTransfer.Status, "Scheduled", "4. wrong status for ftsStatus: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Scheduled", "4. wrong status for request: %s" % req.Status )
# # reschedule
ftsFile.Status = "Waiting"
self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status )
self.assertEqual( ftsTransfer.Status, "Waiting", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Waiting", "5. wrong status for request: %s" % req.Status )
# # fts done
ftsFile.Status = "Done"
self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status )
self.assertEqual( ftsTransfer.Status, "Done", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Done", "5. wrong status for request: %s" % req.Status )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if not taskDict:
return S_OK({})
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
# Do not remove sorted, we might pop elements in the loop
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
# If there are input files
if paramDict['InputData']:
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
self.log.error( "Error creating request for task", "%s %s" % ( taskID, isValid ) )
# This works because we loop over a copy of the keys !
taskDict.pop( taskID )
continue
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
reqClient = ReqClient()
fc = FileCatalog()
requestOperation = 'RemoveReplica'
if targetSE == 'All':
requestOperation = 'RemoveFile'
for lfnList in breakListIntoChunks( lfns, 100 ):
oRequest = Request()
requestName = "%s_%s" % ( md5( repr( time.time() ) ).hexdigest()[:16], md5( repr( time.time() ) ).hexdigest()[:16] )
oRequest.RequestName = requestName
oOperation = Operation()
oOperation.Type = requestOperation
oOperation.TargetSE = targetSE
res = fc.getFileMetadata( lfnList )
if not res['OK']:
print "Can't get file metadata: %s" % res['Message']
DIRAC.exit( 1 )
if res['Value']['Failed']:
print "Could not get the file metadata of the following, so skipping them:"
for fFile in res['Value']['Failed']:
print fFile
lfnMetadata = res['Value']['Successful']
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
def createRequest(self, requestName, archiveLFN, lfnChunk):
"""Create the Request."""
request = Request()
request.RequestName = requestName
self._checkReplicaSites(request, lfnChunk)
archiveFiles = Operation()
archiveFiles.Type = "ArchiveFiles"
archiveFiles.Arguments = DEncode.encode({
"SourceSE":
self.sourceSEs[0],
"TarballSE":
self.switches["TarballSE"],
"RegisterDescendent":
self.switches["RegisterDescendent"],
"ArchiveLFN":
archiveLFN,
})
self.addLFNs(archiveFiles, lfnChunk)
request.addOperation(archiveFiles)
# Replicate the Tarball, ArchiveFiles will upload it
if self.switches.get("ReplicateTarball"):
replicateAndRegisterTarBall = Operation()
replicateAndRegisterTarBall.Type = "ReplicateAndRegister"
replicateAndRegisterTarBall.TargetSE = self.targetSE
opFile = File()
opFile.LFN = archiveLFN
replicateAndRegisterTarBall.addFile(opFile)
request.addOperation(replicateAndRegisterTarBall)
if self.switches.get("CheckMigration"):
checkMigrationTarBall = Operation()
checkMigrationTarBall.Type = "CheckMigration"
migrationTarget = self.targetSE if self.switches.get(
"ReplicateTarball") else self.switches["TarballSE"]
checkMigrationTarBall.TargetSE = migrationTarget
opFile = File()
opFile.LFN = archiveLFN
checkMigrationTarBall.addFile(opFile)
request.addOperation(checkMigrationTarBall)
# Register Archive Replica for LFNs
if self.switches.get("ArchiveSE"):
registerArchived = Operation()
registerArchived.Type = "RegisterReplica"
registerArchived.TargetSE = self.switches.get("ArchiveSE")
self.addLFNs(registerArchived, lfnChunk, addPFN=True)
request.addOperation(registerArchived)
# Remove all Other Replicas for LFNs
if self.switches.get("RemoveReplicas"):
removeArchiveReplicas = Operation()
removeArchiveReplicas.Type = "RemoveReplica"
removeArchiveReplicas.TargetSE = ",".join(self.replicaSEs)
self.addLFNs(removeArchiveReplicas, lfnChunk)
request.addOperation(removeArchiveReplicas)
# Remove all Replicas for LFNs
if self.switches.get("RemoveFiles"):
removeArchiveFiles = Operation()
removeArchiveFiles.Type = "RemoveFile"
self.addLFNs(removeArchiveFiles, lfnChunk)
request.addOperation(removeArchiveFiles)
# Remove Original tarball replica
if self.switches.get("ReplicateTarball"):
removeTarballOrg = Operation()
removeTarballOrg.Type = "RemoveReplica"
removeTarballOrg.TargetSE = self.sourceSEs[0]
opFile = File()
opFile.LFN = archiveLFN
removeTarballOrg.addFile(opFile)
request.addOperation(removeTarballOrg)
return request
def createRequest( reqType ):
r = Request()
# Simple failover
op1 = Operation()
f = File()
f.LFN = '/This/is/an/LFN'
op1.addFile( f )
op1.Type = 'ReplicateAndRegister'
op1.SourceSE = 'CERN-FAILOVER'
op1.TargetSE = 'CERN-BUFFER'
r.addOperation( op1 )
op2 = Operation()
op2.addFile( f )
op2.Type = 'RemoveReplica'
op2.TargetSE = 'CERN-FAILOVER'
r.addOperation( op2 )
if reqType == 0:
return r
# two files for Failover
f1 = File()
f1.LFN = '/This/is/a/second/LFN'
op3 = Operation()
op3.addFile( f1 )
op3.Type = 'ReplicateAndRegister'
op3.SourceSE = 'CERN-FAILOVER'
op3.TargetSE = 'CERN-BUFFER'
r.addOperation( op3 )
op3 = Operation()
op3.addFile( f1 )
op3.Type = 'RemoveReplica'
op3.TargetSE = 'CERN-FAILOVER'
r.addOperation( op3 )
if reqType == 1:
return r
op = Operation()
op.Type = 'ForwardDiset'
if reqType == 2:
r.addOperation( op )
return r
r.insertBefore( op, r[0] )
if reqType == 3:
return r
op4 = Operation()
op4.Type = 'ForwardDiset'
r.addOperation( op4 )
if reqType == 4:
return r
# 2 different FAILOVER SEs: removal not optimized
r[1].SourceSE = 'RAL-FAILOVER'
r[2].SourceSE = 'RAL-FAILOVER'
if reqType == 5:
return r
# 2 different destinations, same FAILOVER: replication not optimized
r[3].SourceSE = 'RAL-FAILOVER'
r[4].SourceSE = 'RAL-FAILOVER'
r[3].TargetSE = 'RAL-BUFFER'
if reqType == 6:
return r
print 'This should not happen, reqType =', reqType
if len(lfnChunk) > Operation.MAX_FILES:
gLogger.error(
"too many LFNs, max number of files per operation is %s" %
Operation.MAX_FILES)
error = -1
continue
count += 1
request = Request()
request.RequestName = requestName if not multiRequests else '%s_%d' % (
requestName, count)
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = ",".join(targetSEs)
if catalog is not None:
replicateAndRegister.Catalog = catalog
for lfn in lfnChunk:
metaDict = metaDatas["Successful"][lfn]
opFile = File()
opFile.LFN = lfn
opFile.Size = metaDict["Size"]
if "Checksum" in metaDict:
# # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
opFile.Checksum = metaDict["Checksum"]
opFile.ChecksumType = "ADLER32"
replicateAndRegister.addFile(opFile)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(" SE: StorageElement|All")
Script.registerArgument(["LFN: LFN or file containing a List of LFNs"])
Script.parseCommandLine(ignoreErrors=False)
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
args = Script.getPositionalArgs()
targetSE = args.pop(0)
lfns = []
for inputFileName in args:
if os.path.exists(inputFileName):
with open(inputFileName, "r") as inputFile:
string = inputFile.read()
lfns.extend([lfn.strip() for lfn in string.splitlines()])
else:
lfns.append(inputFileName)
from DIRAC.Resources.Storage.StorageElement import StorageElement
import DIRAC
# Check is provided SE is OK
if targetSE != "All":
se = StorageElement(targetSE)
if not se.valid:
print(se.errorReason)
print()
Script.showHelp()
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
reqClient = ReqClient()
fc = FileCatalog()
requestOperation = "RemoveReplica"
if targetSE == "All":
requestOperation = "RemoveFile"
for lfnList in breakListIntoChunks(lfns, 100):
oRequest = Request()
requestName = "%s_%s" % (
md5(repr(time.time()).encode()).hexdigest()[:16],
md5(repr(time.time()).encode()).hexdigest()[:16],
)
oRequest.RequestName = requestName
oOperation = Operation()
oOperation.Type = requestOperation
oOperation.TargetSE = targetSE
res = fc.getFileMetadata(lfnList)
if not res["OK"]:
print("Can't get file metadata: %s" % res["Message"])
DIRAC.exit(1)
if res["Value"]["Failed"]:
print(
"Could not get the file metadata of the following, so skipping them:"
)
for fFile in res["Value"]["Failed"]:
print(fFile)
lfnMetadata = res["Value"]["Successful"]
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
rarFile.Size = lfnMetadata[lfn]["Size"]
rarFile.Checksum = lfnMetadata[lfn]["Checksum"]
rarFile.GUID = lfnMetadata[lfn]["GUID"]
rarFile.ChecksumType = "ADLER32"
oOperation.addFile(rarFile)
oRequest.addOperation(oOperation)
isValid = RequestValidator().validate(oRequest)
if not isValid["OK"]:
print("Request is not valid: ", isValid["Message"])
DIRAC.exit(1)
result = reqClient.putRequest(oRequest)
if result["OK"]:
print("Request %d Submitted" % result["Value"])
else:
print("Failed to submit Request: ", result["Message"])
def createRequest(self, requestName, archiveLFN, lfnChunk):
"""Create the Request."""
request = Request()
request.RequestName = requestName
self._checkReplicaSites(request, lfnChunk)
archiveFiles = Operation()
archiveFiles.Type = 'ArchiveFiles'
archiveFiles.Arguments = DEncode.encode({'SourceSE': self.sourceSEs[0],
'TarballSE': self.switches['TarballSE'],
'RegisterDescendent': self.switches['RegisterDescendent'],
'ArchiveLFN': archiveLFN})
self.addLFNs(archiveFiles, lfnChunk)
request.addOperation(archiveFiles)
# Replicate the Tarball, ArchiveFiles will upload it
if self.switches.get('ReplicateTarball'):
replicateAndRegisterTarBall = Operation()
replicateAndRegisterTarBall.Type = 'ReplicateAndRegister'
replicateAndRegisterTarBall.TargetSE = self.targetSE
opFile = File()
opFile.LFN = archiveLFN
replicateAndRegisterTarBall.addFile(opFile)
request.addOperation(replicateAndRegisterTarBall)
if self.switches.get('CheckMigration'):
checkMigrationTarBall = Operation()
checkMigrationTarBall.Type = 'CheckMigration'
migrationTarget = self.targetSE if self.switches.get('ReplicateTarball') else self.switches['TarballSE']
checkMigrationTarBall.TargetSE = migrationTarget
opFile = File()
opFile.LFN = archiveLFN
checkMigrationTarBall.addFile(opFile)
request.addOperation(checkMigrationTarBall)
# Register Archive Replica for LFNs
if self.switches.get('ArchiveSE'):
registerArchived = Operation()
registerArchived.Type = 'RegisterReplica'
registerArchived.TargetSE = self.switches.get('ArchiveSE')
self.addLFNs(registerArchived, lfnChunk, addPFN=True)
request.addOperation(registerArchived)
# Remove all Other Replicas for LFNs
if self.switches.get('RemoveReplicas'):
removeArchiveReplicas = Operation()
removeArchiveReplicas.Type = 'RemoveReplica'
removeArchiveReplicas.TargetSE = ','.join(self.replicaSEs)
self.addLFNs(removeArchiveReplicas, lfnChunk)
request.addOperation(removeArchiveReplicas)
# Remove all Replicas for LFNs
if self.switches.get('RemoveFiles'):
removeArchiveFiles = Operation()
removeArchiveFiles.Type = 'RemoveFile'
self.addLFNs(removeArchiveFiles, lfnChunk)
request.addOperation(removeArchiveFiles)
# Remove Original tarball replica
if self.switches.get('ReplicateTarball'):
removeTarballOrg = Operation()
removeTarballOrg.Type = 'RemoveReplica'
removeTarballOrg.TargetSE = self.sourceSEs[0]
opFile = File()
opFile.LFN = archiveLFN
removeTarballOrg.addFile(opFile)
request.addOperation(removeTarballOrg)
return request
def createRequest(reqType):
r = Request()
# Simple failover
op1 = Operation()
f = File()
f.LFN = '/This/is/an/LFN'
op1.addFile(f)
op1.Type = 'ReplicateAndRegister'
op1.SourceSE = 'CERN-FAILOVER'
op1.TargetSE = 'CERN-BUFFER'
r.addOperation(op1)
op2 = Operation()
op2.addFile(f)
op2.Type = 'RemoveReplica'
op2.TargetSE = 'CERN-FAILOVER'
r.addOperation(op2)
if reqType == 0:
return r
# two files for Failover
f1 = File()
f1.LFN = '/This/is/a/second/LFN'
op3 = Operation()
op3.addFile(f1)
op3.Type = 'ReplicateAndRegister'
op3.SourceSE = 'CERN-FAILOVER'
op3.TargetSE = 'CERN-BUFFER'
r.addOperation(op3)
op3 = Operation()
op3.addFile(f1)
op3.Type = 'RemoveReplica'
op3.TargetSE = 'CERN-FAILOVER'
r.addOperation(op3)
if reqType == 1:
return r
op = Operation()
op.Type = 'ForwardDiset'
if reqType == 2:
r.addOperation(op)
return r
r.insertBefore(op, r[0])
if reqType == 3:
return r
op4 = Operation()
op4.Type = 'ForwardDiset'
r.addOperation(op4)
if reqType == 4:
return r
# 2 different FAILOVER SEs: removal not optimized
r[1].SourceSE = 'RAL-FAILOVER'
r[2].SourceSE = 'RAL-FAILOVER'
if reqType == 5:
return r
# 2 different destinations, same FAILOVER: replication not optimized
r[3].SourceSE = 'RAL-FAILOVER'
r[4].SourceSE = 'RAL-FAILOVER'
r[3].TargetSE = 'RAL-BUFFER'
if reqType == 6:
return r
print 'This should not happen, reqType =', reqType
