def buildRequest(self, owner, group, sourceSE, targetSE1, targetSE2):
files = self.files(owner, group)
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
putFile = File()
putFile.LFN = lfn
putFile.PFN = fname
putFile.Checksum = checksum
putFile.ChecksumType = "adler32"
putFile.Size = size
putFile.GUID = guid
putAndRegister.addFile(putFile)
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = "%s,%s" % (targetSE1, targetSE2)
for fname, lfn, size, checksum, guid in files:
repFile = File()
repFile.LFN = lfn
repFile.Size = size
repFile.Checksum = checksum
repFile.ChecksumType = "adler32"
replicateAndRegister.addFile(repFile)
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
removeReplica.addFile(File({"LFN": lfn}))
removeFile = Operation()
removeFile.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFile.addFile(File({"LFN": lfn}))
removeFileInit = Operation()
removeFileInit.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFileInit.addFile(File({"LFN": lfn}))
req = Request()
req.addOperation(removeFileInit)
req.addOperation(putAndRegister)
req.addOperation(replicateAndRegister)
req.addOperation(removeReplica)
req.addOperation(removeFile)
return req
def buildRequest( self, owner, group, sourceSE, targetSE1, targetSE2 ):
files = self.files( owner, group )
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
putFile = File()
putFile.LFN = lfn
putFile.PFN = fname
putFile.Checksum = checksum
putFile.ChecksumType = "adler32"
putFile.Size = size
putFile.GUID = guid
putAndRegister.addFile( putFile )
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = "%s,%s" % ( targetSE1, targetSE2 )
for fname, lfn, size, checksum, guid in files:
repFile = File()
repFile.LFN = lfn
repFile.Size = size
repFile.Checksum = checksum
repFile.ChecksumType = "adler32"
replicateAndRegister.addFile( repFile )
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
removeReplica.addFile( File( {"LFN": lfn } ) )
removeFile = Operation()
removeFile.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFile.addFile( File( {"LFN": lfn } ) )
removeFileInit = Operation()
removeFileInit.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFileInit.addFile( File( {"LFN": lfn } ) )
req = Request()
req.addOperation( removeFileInit )
req.addOperation( putAndRegister )
req.addOperation( replicateAndRegister )
req.addOperation( removeReplica )
req.addOperation( removeFile )
return req
def test__addMetadataToFiles(self):
resMeta = {
"OK": True,
"Value": {
"Failed": {},
"Successful": {
"/lhcb/1.dst": {
"ChecksumType": "AD",
"Checksum": "123456",
"CreationDate": datetime.datetime(2013, 12, 11, 20, 20, 21),
"GUID": "92F9CE97-7A62-E311-8401-0025907FD430",
"Mode": 436,
"ModificationDate": datetime.datetime(2013, 12, 11, 20, 20, 21),
"NumberOfLinks": 1,
"Size": 5846023777,
"Status": "-",
},
"/lhcb/2.dst": {
"ChecksumType": "AD",
"Checksum": "987654",
"CreationDate": datetime.datetime(2013, 12, 12, 6, 26, 52),
"GUID": "DAE4933A-C162-E311-8A6B-003048FEAF04",
"Mode": 436,
"ModificationDate": datetime.datetime(2013, 12, 12, 6, 26, 52),
"NumberOfLinks": 1,
"Size": 5893396937,
"Status": "-",
},
},
},
}
self.rr.fc.getFileMetadata.return_value = resMeta
file1 = File()
file1.LFN = "/lhcb/1.dst"
file2 = File()
file2.LFN = "/lhcb/2.dst"
toSchedule = {"/lhcb/1.dst": [file1, ["SE1"], ["SE2", "SE3"]], "/lhcb/2.dst": [file2, ["SE4"], ["SE5", "SE6"]]}
res = self.rr._addMetadataToFiles(toSchedule)
self.assertTrue(res["OK"])
for lfn in toSchedule:
self.assertEqual(res["Value"][lfn].LFN, lfn)
for attr in ("GUID", "Size", "Checksum"):
self.assertEqual(getattr(res["Value"][lfn], attr), resMeta["Value"]["Successful"][lfn][attr])
# AD should be transformed into Adler32
self.assertEqual(res["Value"][lfn].ChecksumType, "ADLER32")
def __insertRegisterOperation( self, request, operation, toRegister ):
""" add RegisterReplica operation
:param Request request: request instance
:param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
:param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
"""
log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )
byTarget = {}
for ftsFile in toRegister:
if ftsFile.TargetSE not in byTarget:
byTarget.setdefault( ftsFile.TargetSE, [] )
byTarget[ftsFile.TargetSE].append( ftsFile )
log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )
for target, ftsFileList in byTarget.iteritems():
log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
len( ftsFileList ) ) )
registerOperation = Operation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
targetSE = StorageElement( target )
for ftsFile in ftsFileList:
opFile = File()
opFile.LFN = ftsFile.LFN
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
if not pfn["OK"]:
continue
opFile.PFN = pfn["Value"]
registerOperation.addFile( opFile )
request.insertBefore( registerOperation, operation )
return S_OK()
def _setFileReplicationRequest(self,
lfn,
targetSE,
fileMetaDict,
sourceSE=""):
"""Sets a registration request."""
self.log.info("Setting ReplicateAndRegister request",
"for %s to %s" % (lfn, targetSE))
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.TargetSE = targetSE
if sourceSE:
transfer.SourceSE = sourceSE
trFile = File()
trFile.LFN = lfn
cksm = fileMetaDict.get("Checksum", None)
cksmType = fileMetaDict.get("ChecksumType", self.defaultChecksumType)
if cksm and cksmType:
trFile.Checksum = cksm
trFile.ChecksumType = cksmType
size = fileMetaDict.get("Size", 0)
if size:
trFile.Size = size
guid = fileMetaDict.get("GUID", "")
if guid:
trFile.GUID = guid
transfer.addFile(trFile)
self.request.addOperation(transfer)
return S_OK()
def myRequest():
"""Create a request and put it to the db"""
request = Request()
request.RequestName = 'myAwesomeRemovalRequest.xml'
request.JobID = 0
request.SourceComponent = "myScript"
remove = Operation()
remove.Type = "RemoveFile"
lfn = "/ilc/user/s/sailer/test.txt"
rmFile = File()
rmFile.LFN = lfn
remove.addFile( rmFile )
request.addOperation( remove )
isValid = RequestValidator().validate( request )
if not isValid['OK']:
raise RuntimeError( "Failover request is not valid: %s" % isValid['Message'] )
else:
print("It is a GOGOGO")
requestClient = ReqClient()
result = requestClient.putRequest( request )
print(result)
def __setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE
:param list catalog: list of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get( "Checksum", "" )
regFile.ChecksumType = fileDict.get( "ChecksumType", "" )
regFile.Size = fileDict.get( "Size", 0 )
regFile.GUID = fileDict.get( "GUID", "" )
se = StorageElement( targetSE )
pfn = se.getPfnForLfn( lfn )
if not pfn["OK"]:
self.log.error( "unable to get PFN for LFN: %s" % pfn["Message"] )
return pfn
regFile.PFN = pfn["Value"]
register.addFile( regFile )
self.request.addOperation( register )
return S_OK()
def setBKRegistrationRequest(self, lfn, error='',
metaData={'Checksum': 'justSomething',
'ChecksumType': 'ADLER32',
'GUID': 'aGUID'}):
""" Set a BK registration request for changing the replica flag.
Uses the global request object (self.request).
"""
if error:
self.log.info('BK registration for %s failed with message: "%s" setting failover request' % (lfn, error))
else:
self.log.info('Setting BK registration request for %s' % (lfn))
regFile = Operation()
regFile.Type = 'RegisterFile'
regFile.Catalog = 'BookkeepingDB'
bkFile = File()
bkFile.LFN = lfn
# this should NOT be needed... but RMS complains!
bkFile.PFN = lfn
bkFile.GUID = metaData['GUID']
bkFile.Checksum = metaData['Checksum']
bkFile.ChecksumType = metaData['ChecksumType']
regFile.addFile(bkFile)
res = self.request.addOperation(regFile)
if not res['OK']:
raise RuntimeError(res['Message'])
def myRequest():
"""Create a request and put it to the db"""
request = Request()
request.RequestName = 'myAwesomeRemovalRequest.xml'
request.JobID = 0
request.SourceComponent = "myScript"
remove = Operation()
remove.Type = "RemoveFile"
lfn = "/ilc/user/s/sailer/test.txt"
rmFile = File()
rmFile.LFN = lfn
remove.addFile(rmFile)
request.addOperation(remove)
isValid = RequestValidator().validate(request)
if not isValid['OK']:
raise RuntimeError("Failover request is not valid: %s" %
isValid['Message'])
else:
print "It is a GOGOGO"
requestClient = ReqClient()
result = requestClient.putRequest(request)
print result
def __insertRegisterOperation( self, request, operation, toRegister ):
""" add RegisterReplica operation
:param Request request: request instance
:param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
:param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
"""
log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )
byTarget = {}
for ftsFile in toRegister:
if ftsFile.TargetSE not in byTarget:
byTarget.setdefault( ftsFile.TargetSE, [] )
byTarget[ftsFile.TargetSE].append( ftsFile )
log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )
for target, ftsFileList in byTarget.iteritems():
log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
len( ftsFileList ) ) )
registerOperation = Operation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
targetSE = StorageElement( target )
for ftsFile in ftsFileList:
opFile = File()
opFile.LFN = ftsFile.LFN
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
if not pfn["OK"]:
continue
opFile.PFN = pfn["Value"]
registerOperation.addFile( opFile )
request.insertBefore( registerOperation, operation )
return S_OK()
def __setFileReplicationRequest( self, lfn, targetSE, fileMetaDict, sourceSE = '' ):
""" Sets a registration request.
"""
self.log.info( 'Setting replication request for %s to %s' % ( lfn, targetSE ) )
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.TargetSE = targetSE
if sourceSE:
transfer.SourceSE = sourceSE
trFile = File()
trFile.LFN = lfn
cksm = fileMetaDict.get( "Checksum", None )
cksmType = fileMetaDict.get( "ChecksumType", None )
if cksm and cksmType:
trFile.Checksum = cksm
trFile.ChecksumType = cksmType
size = fileMetaDict.get( "Size", 0 )
if size:
trFile.Size = size
guid = fileMetaDict.get( "GUID", "" )
if guid:
trFile.GUID = guid
transfer.addFile( trFile )
self.request.addOperation( transfer )
return S_OK()
def archiveRequestAndOp(listOfLFNs):
"""Return a tuple of the request and operation."""
req = Request()
req.RequestName = "MyRequest"
op = Operation()
switches = {}
archiveLFN = "/vo/tars/myTar.tar"
op.Arguments = DEncode.encode({
"SourceSE":
switches.get("SourceSE", "SOURCE-SE"),
"TarballSE":
switches.get("TarballSE", "TARBALL-SE"),
"RegisterDescendent":
False,
"ArchiveLFN":
archiveLFN,
})
op.Type = "ArchiveFiles"
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = "01130a%0d" % index
oFile.ChecksumType = "adler32"
op.addFile(oFile)
req.addOperation(op)
return req, op
def getRegisterOperation(self,
opFile,
targetSE,
type='RegisterFile',
catalog=None):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = StorageElement(targetSE).getPfnForLfn(
opFile.LFN).get('Value', {}).get('Successful', {}).get(opFile.LFN)
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
return registerOperation
def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE):
""" Set a request to upload job log files from the output sandbox
Changed to be similar to LHCb createLogUploadRequest
using LHCb LogUpload Request and Removal Request
"""
self.log.info('Setting log upload request for %s at %s' %
(targetSE, logFileLFN))
request = self._getRequestContainer()
logUpload = Operation()
logUpload.Type = "LogUpload"
logUpload.TargetSE = targetSE
upFile = File()
upFile.LFN = logFileLFN
logUpload.addFile(upFile)
logRemoval = Operation()
logRemoval.Type = 'RemoveFile'
logRemoval.TargetSE = uploadedSE
logRemoval.addFile(upFile)
request.addOperation(logUpload)
request.addOperation(logRemoval)
self.workflow_commons['Request'] = request
return S_OK()
def __submitRMSOp(self, target_se, lfns_chunk_dict, whichRMSOp='ReplicateAndRegister' ):
""" target_se : SE name to which to replicate
lfns_chunk_dict : LFNS dict with 100 lfns as key andeEach lfn has 'Size', 'Checksum'
whichRMSOp: Choose from RMP operation - ReplicateAndRegister, ReplicateAndRemove, PutAndRegister
"""
## Setup request
request = Request()
request.RequestName = "DDM_"+ str(target_se) + datetime.datetime.now().strftime("_%Y%m%d_%H%M%S")
myOp = Operation()
myOp.Type = whichRMSOp
myOp.TargetSE = target_se
## Add LFNS to operations
for lfn in lfns_chunk_dict.keys():
opFile = File()
opFile.LFN = lfn
opFile.Size = lfns_chunk_dict[lfn]['Size']
if "Checksum" in lfns_chunk_dict[lfn]:
opFile.Checksum = lfns_chunk_dict[lfn]['Checksum']
opFile.ChecksumType = 'ADLER32'
## Add file to operation
myOp.addFile( opFile )
request.addOperation( myOp )
reqClient = ReqClient()
putRequest = reqClient.putRequest( request )
if not putRequest["OK"]:
gLogger.error( "Unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) )
return S_ERROR("Problem submitting to RMS.")
def getRegisterOperation(self, opFile, targetSE, type="RegisterFile", catalog=None):
"""add RegisterReplica operation for file
:param ~DIRAC.RequestManagementSystem.Client.File.File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = (
StorageElement(targetSE)
.getURL(opFile.LFN, protocol=self.registrationProtocols)
.get("Value", {})
.get("Successful", {})
.get(opFile.LFN)
)
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
return registerOperation
def archiveRequestAndOp(listOfLFNs):
"""Return a tuple of the request and operation."""
req = Request()
req.RequestName = 'MyRequest'
op = Operation()
switches = {}
archiveLFN = '/vo/tars/myTar.tar'
op.Arguments = DEncode.encode({
'SourceSE':
switches.get('SourceSE', 'SOURCE-SE'),
'TarballSE':
switches.get('TarballSE', 'TARBALL-SE'),
'RegisterDescendent':
False,
'ArchiveLFN':
archiveLFN
})
op.Type = 'ArchiveFiles'
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = '01130a%0d' % index
oFile.ChecksumType = 'adler32'
op.addFile(oFile)
req.addOperation(op)
return req, op
def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE):
""" Set a request to upload job log files from the output sandbox
Changed to be similar to LHCb createLogUploadRequest
using LHCb LogUpload Request and Removal Request
"""
self.log.info('Setting log upload request for %s at %s' %(targetSE, logFileLFN))
request = self._getRequestContainer()
logUpload = Operation()
logUpload.Type = "LogUpload"
logUpload.TargetSE = targetSE
upFile = File()
upFile.LFN = logFileLFN
logUpload.addFile( upFile )
logRemoval = Operation()
logRemoval.Type = 'RemoveFile'
logRemoval.TargetSE = uploadedSE
logRemoval.addFile( upFile )
request.addOperation ( logUpload )
request.addOperation ( logRemoval )
self.workflow_commons['Request'] = request
return S_OK()
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def __setFileRemovalRequest(self, lfn):
""" Sets a removal request for a file including all replicas.
"""
self.log.info('Setting file removal request for %s' % lfn)
removeFile = Operation()
removeFile.Type = 'RemoveFile'
rmFile = File()
rmFile.LFN = lfn
removeFile.addFile(rmFile)
self.request.addOperation(removeFile)
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has multiple operations
:param transJson: list of lists of string and dictionaries, e.g.:
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
failedTasks = []
for taskID, task in taskDict.items():
transID = task['TransformationID']
if not task.get('InputData'):
self._logError("Error creating request for task",
"%s, No input data" % taskID,
transID=transID)
taskDict.pop(taskID)
continue
files = []
oRequest = Request()
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
# create the operations from the json structure
for operationTuple in transJson:
op = Operation()
op.Type = operationTuple[0]
for parameter, value in operationTuple[1].iteritems():
setattr(op, parameter, value)
for lfn in files:
opFile = File()
opFile.LFN = lfn
op.addFile(opFile)
oRequest.addOperation(op)
result = self._assignRequestToTask(oRequest, taskDict, transID,
taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def test__addMetadataToFiles( self ):
resMeta = {'OK': True,
'Value': {'Failed': {},
'Successful': {'/lhcb/1.dst': {'ChecksumType': 'AD',
'Checksum': '123456',
'CreationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ),
'GUID': '92F9CE97-7A62-E311-8401-0025907FD430',
'Mode': 436,
'ModificationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ),
'NumberOfLinks': 1,
'Size': 5846023777,
'Status': '-'},
'/lhcb/2.dst': {'ChecksumType': 'AD',
'Checksum': '987654',
'CreationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ),
'GUID': 'DAE4933A-C162-E311-8A6B-003048FEAF04',
'Mode': 436,
'ModificationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ),
'NumberOfLinks': 1,
'Size': 5893396937,
'Status': '-'}}}}
self.rr.fc.getFileMetadata.return_value = resMeta
file1 = File()
file1.LFN = '/lhcb/1.dst'
file2 = File()
file2.LFN = '/lhcb/2.dst'
toSchedule = {'/lhcb/1.dst': [file1, ['SE1'], ['SE2', 'SE3']],
'/lhcb/2.dst': [file2, ['SE4'], ['SE5', 'SE6']]}
res = self.rr._addMetadataToFiles( toSchedule )
self.assertTrue(res['OK'])
for lfn in toSchedule:
self.assertEqual( res['Value'][lfn].LFN, lfn )
for attr in ('GUID', 'Size', 'Checksum'):
self.assertEqual( getattr(res['Value'][lfn],attr), resMeta['Value']['Successful'][lfn][attr] )
# AD should be transformed into Adler32
self.assertEqual( res['Value'][lfn].ChecksumType, "ADLER32" )
def test__addMetadataToFiles( self ):
resMeta = {'OK': True,
'Value': {'Failed': {},
'Successful': {'/lhcb/1.dst': {'ChecksumType': 'AD',
'Checksum': '123456',
'CreationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ),
'GUID': '92F9CE97-7A62-E311-8401-0025907FD430',
'Mode': 436,
'ModificationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ),
'NumberOfLinks': 1,
'Size': 5846023777,
'Status': '-'},
'/lhcb/2.dst': {'ChecksumType': 'AD',
'Checksum': '987654',
'CreationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ),
'GUID': 'DAE4933A-C162-E311-8A6B-003048FEAF04',
'Mode': 436,
'ModificationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ),
'NumberOfLinks': 1,
'Size': 5893396937,
'Status': '-'}}}}
self.rr.fc.getFileMetadata.return_value = resMeta
file1 = File()
file1.LFN = '/lhcb/1.dst'
file2 = File()
file2.LFN = '/lhcb/2.dst'
toSchedule = {'/lhcb/1.dst': [file1, ['SE1'], ['SE2', 'SE3']],
'/lhcb/2.dst': [file2, ['SE4'], ['SE5', 'SE6']]}
res = self.rr._addMetadataToFiles( toSchedule )
self.assertTrue(res['OK'])
for lfn in toSchedule:
self.assertEqual( res['Value'][lfn].LFN, lfn )
for attr in ('GUID', 'Size', 'Checksum'):
self.assertEqual( getattr(res['Value'][lfn],attr), resMeta['Value']['Successful'][lfn][attr] )
# AD should be transformed into Adler32
self.assertEqual( res['Value'][lfn].ChecksumType, "ADLER32" )
def test__addMetadataToFiles( self ):
resMeta = {'OK': True,
'Value': {'Failed': {},
'Successful': {'/lhcb/1.dst': {'ChecksumType': 'AD',
'Checksum': '123456',
'CreationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ),
'GUID': '92F9CE97-7A62-E311-8401-0025907FD430',
'Mode': 436,
'ModificationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ),
'NumberOfLinks': 1,
'Size': 5846023777,
'Status': '-'},
'/lhcb/2.dst': {'ChecksumType': 'AD',
'Checksum': '987654',
'CreationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ),
'GUID': 'DAE4933A-C162-E311-8A6B-003048FEAF04',
'Mode': 436,
'ModificationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ),
'NumberOfLinks': 1,
'Size': 5893396937,
'Status': '-'}}}}
self.rr.fc.getFileMetadata.return_value = resMeta
file1 = File()
file1.LFN = '/lhcb/1.dst'
file2 = File()
file2.LFN = '/lhcb/2.dst'
toSchedule = {'/lhcb/1.dst': [file1, ['SE1'], ['SE2', 'SE3']],
'/lhcb/2.dst': [file2, ['SE4'], ['SE5', 'SE6']]}
res = self.rr._addMetadataToFiles( toSchedule )
self.assert_( res['OK'] )
self.assertEqual( json.loads( res['Value'][0][0] )['LFN'], resMeta['Value']['Successful'].keys()[0] )
self.assertEqual( json.loads( res['Value'][0][0] )['Size'], resMeta['Value']['Successful'].values()[0]['Size'] )
self.assertEqual( json.loads( res['Value'][1][0] )['LFN'], resMeta['Value']['Successful'].keys()[1] )
self.assertEqual( json.loads( res['Value'][1][0] )['Size'], resMeta['Value']['Successful'].values()[1]['Size'] )
def test__addMetadataToFiles( self ):
resMeta = {'OK': True,
'Value': {'Failed': {},
'Successful': {'/lhcb/1.dst': {'ChecksumType': 'AD',
'Checksum': '123456',
'CreationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ),
'GUID': '92F9CE97-7A62-E311-8401-0025907FD430',
'Mode': 436,
'ModificationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ),
'NumberOfLinks': 1,
'Size': 5846023777,
'Status': '-'},
'/lhcb/2.dst': {'ChecksumType': 'AD',
'Checksum': '987654',
'CreationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ),
'GUID': 'DAE4933A-C162-E311-8A6B-003048FEAF04',
'Mode': 436,
'ModificationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ),
'NumberOfLinks': 1,
'Size': 5893396937,
'Status': '-'}}}}
self.rr.fc.getFileMetadata.return_value = resMeta
file1 = File()
file1.LFN = '/lhcb/1.dst'
file2 = File()
file2.LFN = '/lhcb/2.dst'
toSchedule = {'/lhcb/1.dst': [file1, ['SE1'], ['SE2', 'SE3']],
'/lhcb/2.dst': [file2, ['SE4'], ['SE5', 'SE6']]}
res = self.rr._addMetadataToFiles( toSchedule )
self.assert_( res['OK'] )
self.assertEqual( json.loads( res['Value'][0][0] )['LFN'], resMeta['Value']['Successful'].keys()[0] )
self.assertEqual( json.loads( res['Value'][0][0] )['Size'], resMeta['Value']['Successful'].values()[0]['Size'] )
self.assertEqual( json.loads( res['Value'][1][0] )['LFN'], resMeta['Value']['Successful'].keys()[1] )
self.assertEqual( json.loads( res['Value'][1][0] )['Size'], resMeta['Value']['Successful'].values()[1]['Size'] )
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has multiple operations
:param transJson: list of lists of string and dictionaries, e.g.:
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
failedTasks = []
for taskID, task in taskDict.items():
transID = task['TransformationID']
if not task.get('InputData'):
self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID)
taskDict.pop(taskID)
continue
files = []
oRequest = Request()
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
# create the operations from the json structure
for operationTuple in transJson:
op = Operation()
op.Type = operationTuple[0]
for parameter, value in operationTuple[1].iteritems():
setattr(op, parameter, value)
for lfn in files:
opFile = File()
opFile.LFN = lfn
op.addFile(opFile)
oRequest.addOperation(op)
result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def addRemovalRequests(self, lfnList):
"""Create removalRequests for lfns in lfnList and add it to the common request"""
request = self._getRequestContainer()
remove = Operation()
remove.Type = "RemoveFile"
for lfn in lfnList:
rmFile = File()
rmFile.LFN = lfn
remove.addFile( rmFile )
request.addOperation( remove )
self.workflow_commons['Request'] = request
def addRemovalRequests(self, lfnList):
"""Create removalRequests for lfns in lfnList and add it to the common request"""
request = self._getRequestContainer()
remove = Operation()
remove.Type = "RemoveFile"
for lfn in lfnList:
rmFile = File()
rmFile.LFN = lfn
remove.addFile( rmFile )
request.addOperation( remove )
self.workflow_commons['Request'] = request
def _setFileRemovalRequest(self, lfn, se="", pfn=""):
"""Sets a removal request for a file including all replicas."""
remove = Operation()
remove.Type = "RemoveFile"
if se:
remove.TargetSE = se
rmFile = File()
rmFile.LFN = lfn
if pfn:
rmFile.PFN = pfn
remove.addFile(rmFile)
self.request.addOperation(remove)
return S_OK()
def checkRequestAndOp(listOfLFNs):
req = Request()
req.RequestName = 'MyRequest'
op = Operation()
op.Type = 'CheckMigration'
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = '01130a%0d' % index
oFile.ChecksumType = 'adler32'
op.addFile(oFile)
req.addOperation(op)
return req, op
def __setFileRemovalRequest( self, lfn, se = '', pfn = '' ):
""" Sets a removal request for a file including all replicas.
"""
remove = Operation()
remove.Type = "RemoveFile"
if se:
remove.TargetSE = se
rmFile = File()
rmFile.LFN = lfn
if pfn:
rmFile.PFN = pfn
remove.addFile( rmFile )
self.request.addOperation( remove )
return S_OK()
def test__getLFNsForBKRegistration(mocker):
mocker.patch("LHCbDIRAC.Workflow.Modules.ModuleBase.RequestValidator",
side_effect=MagicMock())
f1 = File()
f1.LFN = '/a/1.txt'
f2 = File()
f2.LFN = '/a/2.txt'
f3 = File()
f3.LFN = '/a/3.txt'
o1 = Operation()
o1.Type = 'RegisterFile'
o1.addFile(f1)
o2 = Operation()
o2.Type = 'RegisterFile'
o2.addFile(f2)
o3 = Operation()
o3.Type = 'ForwardDISET'
o4 = Operation()
o4.Type = 'RegisterFile'
o4.addFile(f1)
o4.addFile(f3)
r = Request()
r.addOperation(o4)
r.addOperation(o1)
r.addOperation(o2)
r.addOperation(o3)
uod = UploadOutputData(bkClient=bkc_mock, dm=dm_mock)
uod.request = r
lfns = set(['/a/1.txt', '/a/5.txt', '/a/6.txt', '/a/3.txt'])
lfnsForBkReg = uod._getLFNsForBKRegistration(lfns)
assert sorted(lfnsForBkReg) == sorted(['/a/5.txt', '/a/6.txt'])
def checkRequestAndOp(listOfLFNs):
req = Request()
req.RequestName = "MyRequest"
op = Operation()
op.Type = "CheckMigration"
op.TargetSE = "Foo-SE"
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = "01130a%0d" % index
oFile.ChecksumType = "adler32"
op.addFile(oFile)
req.addOperation(op)
return req, op
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
"""deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = "ReplicateAndRegister"
if transBody:
try:
_requestType, requestOperation = transBody.split(";")
except AttributeError:
pass
failedTasks = []
# Do not remove sorted, we might pop elements in the loop
for taskID, task in taskDict.items():
transID = task["TransformationID"]
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = task["TargetSE"]
# If there are input files
if task.get("InputData"):
if isinstance(task["InputData"], list):
files = task["InputData"]
elif isinstance(task["InputData"], six.string_types):
files = task["InputData"].split(";")
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
result = self._assignRequestToTask(oRequest, taskDict, transID,
taskID, ownerDN, ownerGroup)
if not result["OK"]:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split(';')
except AttributeError:
pass
failedTasks = []
# Do not remove sorted, we might pop elements in the loop
for taskID, task in taskDict.iteritems():
transID = task['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = task['TargetSE']
# If there are input files
if task.get('InputData'):
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
""" Set removal request with the given credentials
"""
oRequest = Request()
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml'
oRequest.SourceComponent = 'JobCleaningAgent'
removeFile = Operation()
removeFile.Type = 'RemoveFile'
removedFile = File()
removedFile.LFN = lfn
removeFile.addFile( removedFile )
oRequest.addOperation( removeFile )
return ReqClient().putRequest( oRequest )
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
""" Set removal request with the given credentials
"""
oRequest = Request()
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml'
oRequest.SourceComponent = 'JobCleaningAgent'
removeFile = Operation()
removeFile.Type = 'RemoveFile'
removedFile = File()
removedFile.LFN = lfn
removeFile.addFile( removedFile )
oRequest.addOperation( removeFile )
return ReqClient().putRequest( oRequest )
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info('Setting registration request for %s at %s.' %
(lfn, targetSE))
if not type(catalog) == type([]):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get("Checksum", "")
regFile.ChecksumType = fileDict.get("ChecksumType",
self.defaultChecksumType)
regFile.Size = fileDict.get("Size", 0)
regFile.GUID = fileDict.get("GUID", "")
se = StorageElement(targetSE)
pfn = se.getPfnForLfn(lfn)
if not pfn["OK"] or lfn not in pfn["Value"]['Successful']:
self.log.error(
"unable to get PFN for LFN: %s" %
pfn.get('Message',
pfn.get('Value', {}).get('Failed', {}).get(lfn)))
return pfn
regFile.PFN = pfn["Value"]['Successful'][lfn]
register.addFile(regFile)
self.request.addOperation(register)
return S_OK()
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if type( paramDict['InputData'] ) == type( [] ):
files = paramDict['InputData']
elif type( paramDict['InputData'] ) == type( '' ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
oRequest.OwnerDN = owner
oRequest.OwnerGroup = ownerGroup
isValid = gRequestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup ):
""" deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
# Do not remove sorted, we might pop elements in the loop
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
# If there are input files
if paramDict.get('InputData'):
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
self._assignRequestToTask( oRequest, taskDict, transID, taskID, ownerDN, ownerGroup )
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog):
"""Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info("Setting registration request",
"for %s at %s." % (lfn, targetSE))
if not isinstance(catalog, list):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get("Checksum", "")
regFile.ChecksumType = fileDict.get("ChecksumType",
self.defaultChecksumType)
regFile.Size = fileDict.get("Size", 0)
regFile.GUID = fileDict.get("GUID", "")
se = StorageElement(targetSE)
res = returnSingleResult(se.getURL(lfn,
self.registrationProtocols))
if not res["OK"]:
self.log.error("Unable to get PFN for LFN", res["Message"])
return res
regFile.PFN = res["Value"]
register.addFile(regFile)
self.request.addOperation(register)
return S_OK()
def test_valid_properties():
theFile = File()
theFile.FileID = 1
assert theFile.FileID == 1
theFile.Status = "Done"
assert theFile.Status == "Done"
theFile.LFN = "/some/path/somewhere"
assert theFile.LFN == "/some/path/somewhere"
theFile.PFN = "/some/path/somewhere"
assert theFile.PFN == "/some/path/somewhere"
theFile.Attempt = 1
assert theFile.Attempt == 1
theFile.Size = 1
assert theFile.Size == 1
theFile.GUID = "2bbabe80-e2f1-11e1-9b23-0800200c9a66"
assert theFile.GUID == "2bbabe80-e2f1-11e1-9b23-0800200c9a66"
theFile.ChecksumType = "adler32"
assert theFile.ChecksumType == "ADLER32"
theFile.Checksum = "123456"
assert theFile.Checksum == "123456"
def _setReplicaRemovalRequest(self, lfn, se):
"""Sets a removal request for a replica.
:param str lfn: LFN
:param se:
"""
if isinstance(se, str):
se = ",".join([se.strip() for se in se.split(",") if se.strip()])
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = se
replicaToRemove = File()
replicaToRemove.LFN = lfn
removeReplica.addFile(replicaToRemove)
self.request.addOperation(removeReplica)
return S_OK()
def __setReplicaRemovalRequest( self, lfn, se ):
""" Sets a removal request for a replica.
:param str lfn: LFN
:param se:
"""
if type( se ) == str:
se = ",".join( [ se.strip() for se in se.split( "," ) if se.strip() ] )
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = se
replicaToRemove = File()
replicaToRemove.LFN = lfn
removeReplica.addFile( replicaToRemove )
self.request.addOperation( removeReplica )
return S_OK()
def __createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE):
""" Set a request to upload job log files from the output sandbox
"""
self.log.info('Setting log upload request for %s at %s' % (logFileLFN, targetSE))
logUpload = Operation()
logUpload.Type = 'LogUpload'
logUpload.TargetSE = targetSE
logFile = File()
logFile.LFN = logFileLFN
logUpload.addFile(logFile)
self.request.addOperation(logUpload)
logRemoval = Operation()
logRemoval.Type = 'RemoveFile'
logRemoval.TargetSE = uploadedSE
logRemoval.addFile(logFile)
self.request.addOperation(logRemoval)
def getRegisterOperation( self, opFile, targetSE ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = "RegisterFile"
registerOperation.TargetSE = targetSE
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = opFile.PFN
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
return registerOperation
def addRegisterReplica( self, opFile, targetSE ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = "RegisterFile"
registerOperation.TargetSE = targetSE
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = opFile.PFN
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
self.request.insertAfter( registerOperation, self.operation )
return S_OK()
def _setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )
if not isinstance( catalog, list ):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get( "Checksum", "" )
regFile.ChecksumType = fileDict.get( "ChecksumType", self.defaultChecksumType )
regFile.Size = fileDict.get( "Size", 0 )
regFile.GUID = fileDict.get( "GUID", "" )
se = StorageElement( targetSE )
pfn = se.getURL( lfn, self.registrationProtocols )
if not pfn["OK"] or lfn not in pfn["Value"]['Successful']:
self.log.error( "Unable to get PFN for LFN", "%s" % pfn.get( 'Message', pfn.get( 'Value', {} ).get( 'Failed', {} ).get( lfn ) ) )
return pfn
regFile.PFN = pfn["Value"]['Successful'][lfn]
register.addFile( regFile )
self.request.addOperation( register )
return S_OK()
def getRegisterOperation( self, opFile, targetSE, type = 'RegisterFile', catalog = None ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = StorageElement( targetSE ).getURL( opFile.LFN, protocol = self.registrationProtocols ).get( 'Value', {} ).get( 'Successful', {} ).get( opFile.LFN )
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
return registerOperation
def test02props( self ):
""" test props and attributes """
theFile = File()
# valid props
theFile.FileID = 1
self.assertEqual( theFile.FileID, 1 )
theFile.Status = "Done"
self.assertEqual( theFile.Status, "Done" )
theFile.LFN = "/some/path/somewhere"
self.assertEqual( theFile.LFN, "/some/path/somewhere" )
theFile.PFN = "/some/path/somewhere"
self.assertEqual( theFile.PFN, "/some/path/somewhere" )
theFile.Attempt = 1
self.assertEqual( theFile.Attempt, 1 )
theFile.Size = 1
self.assertEqual( theFile.Size, 1 )
theFile.GUID = "2bbabe80-e2f1-11e1-9b23-0800200c9a66"
self.assertEqual( theFile.GUID, "2bbabe80-e2f1-11e1-9b23-0800200c9a66" )
theFile.ChecksumType = "adler32"
self.assertEqual( theFile.ChecksumType, "ADLER32" )
theFile.Checksum = "123456"
self.assertEqual( theFile.Checksum, "123456" )
# #
theFile.Checksum = None
theFile.ChecksumType = None
self.assertEqual( theFile.Checksum, "" )
self.assertEqual( theFile.ChecksumType, "" )
# # invalid props
# FileID
try:
theFile.FileID = "foo"
except Exception, error:
self.assertEqual( isinstance( error, ValueError ), True )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if not taskDict:
return S_OK({})
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
# Do not remove sorted, we might pop elements in the loop
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
# If there are input files
if paramDict['InputData']:
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
self.log.error( "Error creating request for task", "%s %s" % ( taskID, isValid ) )
# This works because we loop over a copy of the keys !
taskDict.pop( taskID )
continue
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
replicateAndRegister.TargetSE = targetSE
res = fc.getFileMetadata( lfnList )
if not res['OK']:
print "Can't get file metadata: %s" % res['Message']
DIRAC.exit( 1 )
if res['Value']['Failed']:
print "Could not get the file metadata of the following, so skipping them:"
for fFile in res['Value']['Failed']:
print fFile
lfnMetadata = res['Value']['Successful']
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
rarFile.Size = lfnMetadata[lfn]['Size']
rarFile.Checksum = lfnMetadata[lfn]['Checksum']
rarFile.GUID = lfnMetadata[lfn]['GUID']
rarFile.ChecksumType = 'ADLER32'
replicateAndRegister.addFile( rarFile )
oRequest.addOperation( replicateAndRegister )
isValid = RequestValidator().validate( oRequest )
if not isValid['OK']:
print "Request is not valid: ", isValid['Message']
DIRAC.exit( 1 )
result = reqClient.putRequest( oRequest )
if result['OK']:
print "Request %d submitted successfully" % result['Value']
if not os.path.isfile( PFN ):
gLogger.error( "%s is not a file" % PFN )
DIRAC.exit( -1 )
PFN = os.path.abspath( PFN )
size = os.path.getsize( PFN )
adler32 = fileAdler( PFN )
request = Request()
request.RequestName = requestName
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = targetSE
opFile = File()
opFile.LFN = LFN
opFile.PFN = PFN
opFile.Size = size
opFile.Checksum = adler32
opFile.ChecksumType = "ADLER32"
putAndRegister.addFile( opFile )
request.addOperation( putAndRegister )
reqClient = ReqClient()
putRequest = reqClient.putRequest( request )
if not putRequest["OK"]:
gLogger.error( "unable to put request '%s': %s" % ( requestName, putRequest["Message"] ) )
DIRAC.exit( -1 )
gLogger.always( "Request '%s' has been put to ReqDB for execution." % requestName )
gLogger.always( "You can monitor its status using command: 'dirac-rms-show-request %s'" % requestName )
DIRAC.exit( 0 )
def test05FTS( self ):
""" FTS state machine """
req = Request()
req.RequestName = "FTSTest"
ftsTransfer = Operation()
ftsTransfer.Type = "ReplicateAndRegister"
ftsTransfer.TargetSE = "CERN-USER"
ftsFile = File()
ftsFile.LFN = "/a/b/c"
ftsFile.Checksum = "123456"
ftsFile.ChecksumType = "Adler32"
ftsTransfer.addFile( ftsFile )
req.addOperation( ftsTransfer )
self.assertEqual( req.Status, "Waiting", "1. wrong request status: %s" % req.Status )
self.assertEqual( ftsTransfer.Status, "Waiting", "1. wrong ftsStatus status: %s" % ftsTransfer.Status )
# # scheduled
ftsFile.Status = "Scheduled"
self.assertEqual( ftsTransfer.Status, "Scheduled", "2. wrong status for ftsTransfer: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Scheduled", "2. wrong status for request: %s" % req.Status )
# # add new operation before FTS
insertBefore = Operation()
insertBefore.Type = "RegisterReplica"
insertBefore.TargetSE = "CERN-USER"
insertFile = File()
insertFile.LFN = "/a/b/c"
insertFile.PFN = "http://foo/bar"
insertBefore.addFile( insertFile )
req.insertBefore( insertBefore, ftsTransfer )
self.assertEqual( insertBefore.Status, "Waiting", "3. wrong status for insertBefore: %s" % insertBefore.Status )
self.assertEqual( ftsTransfer.Status, "Scheduled", "3. wrong status for ftsStatus: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Waiting", "3. wrong status for request: %s" % req.Status )
# # prev done
insertFile.Status = "Done"
self.assertEqual( insertBefore.Status, "Done", "4. wrong status for insertBefore: %s" % insertBefore.Status )
self.assertEqual( ftsTransfer.Status, "Scheduled", "4. wrong status for ftsStatus: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Scheduled", "4. wrong status for request: %s" % req.Status )
# # reschedule
ftsFile.Status = "Waiting"
self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status )
self.assertEqual( ftsTransfer.Status, "Waiting", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Waiting", "5. wrong status for request: %s" % req.Status )
# # fts done
ftsFile.Status = "Done"
self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status )
self.assertEqual( ftsTransfer.Status, "Done", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Done", "5. wrong status for request: %s" % req.Status )
def createRequest( reqType ):
r = Request()
# Simple failover
op1 = Operation()
f = File()
f.LFN = '/This/is/an/LFN'
op1.addFile( f )
op1.Type = 'ReplicateAndRegister'
op1.SourceSE = 'CERN-FAILOVER'
op1.TargetSE = 'CERN-BUFFER'
r.addOperation( op1 )
op2 = Operation()
op2.addFile( f )
op2.Type = 'RemoveReplica'
op2.TargetSE = 'CERN-FAILOVER'
r.addOperation( op2 )
if reqType == 0:
return r
# two files for Failover
f1 = File()
f1.LFN = '/This/is/a/second/LFN'
op3 = Operation()
op3.addFile( f1 )
op3.Type = 'ReplicateAndRegister'
op3.SourceSE = 'CERN-FAILOVER'
op3.TargetSE = 'CERN-BUFFER'
r.addOperation( op3 )
op3 = Operation()
op3.addFile( f1 )
op3.Type = 'RemoveReplica'
op3.TargetSE = 'CERN-FAILOVER'
r.addOperation( op3 )
if reqType == 1:
return r
op = Operation()
op.Type = 'ForwardDiset'
if reqType == 2:
r.addOperation( op )
return r
r.insertBefore( op, r[0] )
if reqType == 3:
return r
op4 = Operation()
op4.Type = 'ForwardDiset'
r.addOperation( op4 )
if reqType == 4:
return r
# 2 different FAILOVER SEs: removal not optimized
r[1].SourceSE = 'RAL-FAILOVER'
r[2].SourceSE = 'RAL-FAILOVER'
if reqType == 5:
return r
# 2 different destinations, same FAILOVER: replication not optimized
r[3].SourceSE = 'RAL-FAILOVER'
r[4].SourceSE = 'RAL-FAILOVER'
r[3].TargetSE = 'RAL-BUFFER'
if reqType == 6:
return r
print 'This should not happen, reqType =', reqType
continue
count += 1
request = Request()
request.RequestName = requestName if not multiRequests else '%s_%d' % ( requestName, count )
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = ",".join( targetSEs )
if catalog is not None:
replicateAndRegister.Catalog = catalog
for lfn in lfnChunk:
metaDict = metaDatas["Successful"][lfn]
opFile = File()
opFile.LFN = lfn
opFile.Size = metaDict["Size"]
if "Checksum" in metaDict:
# # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
opFile.Checksum = metaDict["Checksum"]
opFile.ChecksumType = "ADLER32"
replicateAndRegister.addFile( opFile )
request.addOperation( replicateAndRegister )
putRequest = reqClient.putRequest( request )
if not putRequest["OK"]:
gLogger.error( "unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) )
error = -1
continue
