def setBKRegistrationRequest(self, lfn, error='',
metaData={'Checksum': 'justSomething',
'ChecksumType': 'ADLER32',
'GUID': 'aGUID'}):
""" Set a BK registration request for changing the replica flag.
Uses the global request object (self.request).
"""
if error:
self.log.info('BK registration for %s failed with message: "%s" setting failover request' % (lfn, error))
else:
self.log.info('Setting BK registration request for %s' % (lfn))
regFile = Operation()
regFile.Type = 'RegisterFile'
regFile.Catalog = 'BookkeepingDB'
bkFile = File()
bkFile.LFN = lfn
# this should NOT be needed... but RMS complains!
bkFile.PFN = lfn
bkFile.GUID = metaData['GUID']
bkFile.Checksum = metaData['Checksum']
bkFile.ChecksumType = metaData['ChecksumType']
regFile.addFile(bkFile)
res = self.request.addOperation(regFile)
if not res['OK']:
raise RuntimeError(res['Message'])
def myRequest():
"""Create a request and put it to the db"""
request = Request()
request.RequestName = 'myAwesomeRemovalRequest.xml'
request.JobID = 0
request.SourceComponent = "myScript"
remove = Operation()
remove.Type = "RemoveFile"
lfn = "/ilc/user/s/sailer/test.txt"
rmFile = File()
rmFile.LFN = lfn
remove.addFile(rmFile)
request.addOperation(remove)
isValid = RequestValidator().validate(request)
if not isValid['OK']:
raise RuntimeError("Failover request is not valid: %s" %
isValid['Message'])
else:
print "It is a GOGOGO"
requestClient = ReqClient()
result = requestClient.putRequest(request)
print result
def __setFileReplicationRequest( self, lfn, targetSE, fileMetaDict, sourceSE = '' ):
""" Sets a registration request.
"""
self.log.info( 'Setting replication request for %s to %s' % ( lfn, targetSE ) )
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.TargetSE = targetSE
if sourceSE:
transfer.SourceSE = sourceSE
trFile = File()
trFile.LFN = lfn
cksm = fileMetaDict.get( "Checksum", None )
cksmType = fileMetaDict.get( "ChecksumType", None )
if cksm and cksmType:
trFile.Checksum = cksm
trFile.ChecksumType = cksmType
size = fileMetaDict.get( "Size", 0 )
if size:
trFile.Size = size
guid = fileMetaDict.get( "GUID", "" )
if guid:
trFile.GUID = guid
transfer.addFile( trFile )
self.request.addOperation( transfer )
return S_OK()
def __setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE
:param list catalog: list of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get( "Checksum", "" )
regFile.ChecksumType = fileDict.get( "ChecksumType", "" )
regFile.Size = fileDict.get( "Size", 0 )
regFile.GUID = fileDict.get( "GUID", "" )
se = StorageElement( targetSE )
pfn = se.getPfnForLfn( lfn )
if not pfn["OK"]:
self.log.error( "unable to get PFN for LFN: %s" % pfn["Message"] )
return pfn
regFile.PFN = pfn["Value"]
register.addFile( regFile )
self.request.addOperation( register )
return S_OK()
def getRegisterOperation(self,
opFile,
targetSE,
type='RegisterFile',
catalog=None):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = StorageElement(targetSE).getPfnForLfn(
opFile.LFN).get('Value', {}).get('Successful', {}).get(opFile.LFN)
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
return registerOperation
def test01ctor(self):
""" test constructors and (de)serialisation """
# # empty ctor
self.assertEqual(isinstance(Operation(), Operation), True,
"empty ctor failed")
# # using fromDict
operation = Operation(self.fromDict)
self.assertEqual(isinstance(operation, Operation), True,
"fromDict ctor failed")
for key, value in self.fromDict.items():
self.assertEqual(
getattr(operation, key), value, "wrong attr value %s (%s) %s" %
(key, getattr(operation, key), value))
# # same with file
operation = Operation(self.fromDict)
operation.addFile(self.subFile)
for key, value in self.fromDict.items():
self.assertEqual(
getattr(operation, key), value, "wrong attr value %s (%s) %s" %
(key, getattr(operation, key), value))
toJSON = operation.toJSON()
self.assertEqual(toJSON["OK"], True, "JSON serialization failed")
def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE):
""" Set a request to upload job log files from the output sandbox
Changed to be similar to LHCb createLogUploadRequest
using LHCb LogUpload Request and Removal Request
"""
self.log.info('Setting log upload request for %s at %s' %
(targetSE, logFileLFN))
request = self._getRequestContainer()
logUpload = Operation()
logUpload.Type = "LogUpload"
logUpload.TargetSE = targetSE
upFile = File()
upFile.LFN = logFileLFN
logUpload.addFile(upFile)
logRemoval = Operation()
logRemoval.Type = 'RemoveFile'
logRemoval.TargetSE = uploadedSE
logRemoval.addFile(upFile)
request.addOperation(logUpload)
request.addOperation(logRemoval)
self.workflow_commons['Request'] = request
return S_OK()
def test_ctor():
"""test constructors and (de)serialisation"""
assert isinstance(Operation(), Operation), "empty ctor failed"
# # using fromDict
fromDict = {
"Type": "replicateAndRegister",
"TargetSE": "CERN-USER,PIC-USER",
"SourceSE": None,
}
operation = Operation(fromDict)
assert isinstance(operation, Operation), "fromDict ctor failed"
for key, value in fromDict.items():
assert getattr(operation,
key) == value, "wrong attr value %s (%s) %s" % (
key, getattr(operation, key), value)
# # same with file
operation = Operation(fromDict)
operation.addFile(
File({
"LFN": "/lhcb/user/c/cibak/testFile",
"Checksum": "1234567",
"ChecksumType": "ADLER32",
"Size": 1024,
"Status": "Waiting",
}))
for key, value in fromDict.items():
assert getattr(operation,
key) == value, "wrong attr value %s (%s) %s" % (
key, getattr(operation, key), value)
toJSON = operation.toJSON()
assert toJSON["OK"], "JSON serialization failed"
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
if self.getCSOption("DelayedExternalDeletion", True):
gLogger.info("Setting deletion request")
try:
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % (
SEName, SEPFN, time.time())
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile(fileToRemove)
request.addOperation(physicalRemoval)
return ReqClient().putRequest(request)
except Exception as e:
gLogger.exception("Exception while setting deletion request")
return S_ERROR("Cannot set deletion request: %s" % str(e))
else:
gLogger.info("Deleting external Sandbox")
try:
return StorageElement(SEName).removeFile(SEPFN)
except Exception as e:
gLogger.exception(
"RM raised an exception while trying to delete a remote sandbox"
)
return S_ERROR(
"RM raised an exception while trying to delete a remote sandbox"
)
def _setFileReplicationRequest(self,
lfn,
targetSE,
fileMetaDict,
sourceSE=""):
"""Sets a registration request."""
self.log.info("Setting ReplicateAndRegister request",
"for %s to %s" % (lfn, targetSE))
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.TargetSE = targetSE
if sourceSE:
transfer.SourceSE = sourceSE
trFile = File()
trFile.LFN = lfn
cksm = fileMetaDict.get("Checksum", None)
cksmType = fileMetaDict.get("ChecksumType", self.defaultChecksumType)
if cksm and cksmType:
trFile.Checksum = cksm
trFile.ChecksumType = cksmType
size = fileMetaDict.get("Size", 0)
if size:
trFile.Size = size
guid = fileMetaDict.get("GUID", "")
if guid:
trFile.GUID = guid
transfer.addFile(trFile)
self.request.addOperation(transfer)
return S_OK()
def __submitRMSOp(self, target_se, lfns_chunk_dict, whichRMSOp='ReplicateAndRegister' ):
""" target_se : SE name to which to replicate
lfns_chunk_dict : LFNS dict with 100 lfns as key andeEach lfn has 'Size', 'Checksum'
whichRMSOp: Choose from RMP operation - ReplicateAndRegister, ReplicateAndRemove, PutAndRegister
"""
## Setup request
request = Request()
request.RequestName = "DDM_"+ str(target_se) + datetime.datetime.now().strftime("_%Y%m%d_%H%M%S")
myOp = Operation()
myOp.Type = whichRMSOp
myOp.TargetSE = target_se
## Add LFNS to operations
for lfn in lfns_chunk_dict.keys():
opFile = File()
opFile.LFN = lfn
opFile.Size = lfns_chunk_dict[lfn]['Size']
if "Checksum" in lfns_chunk_dict[lfn]:
opFile.Checksum = lfns_chunk_dict[lfn]['Checksum']
opFile.ChecksumType = 'ADLER32'
## Add file to operation
myOp.addFile( opFile )
request.addOperation( myOp )
reqClient = ReqClient()
putRequest = reqClient.putRequest( request )
if not putRequest["OK"]:
gLogger.error( "Unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) )
return S_ERROR("Problem submitting to RMS.")
def __insertRegisterOperation( self, request, operation, toRegister ):
""" add RegisterReplica operation
:param Request request: request instance
:param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
:param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
"""
log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )
byTarget = {}
for ftsFile in toRegister:
if ftsFile.TargetSE not in byTarget:
byTarget.setdefault( ftsFile.TargetSE, [] )
byTarget[ftsFile.TargetSE].append( ftsFile )
log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )
for target, ftsFileList in byTarget.iteritems():
log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
len( ftsFileList ) ) )
registerOperation = Operation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
targetSE = StorageElement( target )
for ftsFile in ftsFileList:
opFile = File()
opFile.LFN = ftsFile.LFN
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
if not pfn["OK"]:
continue
opFile.PFN = pfn["Value"]
registerOperation.addFile( opFile )
request.insertBefore( registerOperation, operation )
return S_OK()
def test01ctor( self ):
""" test constructors and (de)serialisation """
# # empty ctor
self.assertEqual( isinstance( Operation(), Operation ), True, "empty ctor failed" )
# # using fromDict
operation = Operation( self.fromDict )
self.assertEqual( isinstance( operation, Operation ), True, "fromDict ctor failed" )
for key, value in self.fromDict.items():
self.assertEqual( getattr( operation, key ), value, "wrong attr value %s (%s) %s" % ( key,
getattr( operation, key ),
value ) )
# # same with file
operation = Operation( self.fromDict )
operation.addFile( self.subFile )
for key, value in self.fromDict.items():
self.assertEqual( getattr( operation, key ), value, "wrong attr value %s (%s) %s" % ( key,
getattr( operation, key ),
value ) )
toJSON = operation.toJSON()
self.assertEqual( toJSON["OK"], True, "JSON serialization failed" )
def getRegisterOperation(self, opFile, targetSE, type="RegisterFile", catalog=None):
"""add RegisterReplica operation for file
:param ~DIRAC.RequestManagementSystem.Client.File.File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = (
StorageElement(targetSE)
.getURL(opFile.LFN, protocol=self.registrationProtocols)
.get("Value", {})
.get("Successful", {})
.get(opFile.LFN)
)
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
return registerOperation
def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ):
if self.getCSOption( "DelayedExternalDeletion", True ):
gLogger.info( "Setting deletion request" )
try:
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() )
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile( fileToRemove )
request.addOperation( physicalRemoval )
return ReqClient().putRequest( request )
except Exception as e:
gLogger.exception( "Exception while setting deletion request" )
return S_ERROR( "Cannot set deletion request: %s" % str( e ) )
else:
gLogger.info( "Deleting external Sandbox" )
try:
return StorageElement( SEName ).removeFile( SEPFN )
except Exception as e:
gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" )
return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def myRequest():
"""Create a request and put it to the db"""
request = Request()
request.RequestName = 'myAwesomeRemovalRequest.xml'
request.JobID = 0
request.SourceComponent = "myScript"
remove = Operation()
remove.Type = "RemoveFile"
lfn = "/ilc/user/s/sailer/test.txt"
rmFile = File()
rmFile.LFN = lfn
remove.addFile( rmFile )
request.addOperation( remove )
isValid = RequestValidator().validate( request )
if not isValid['OK']:
raise RuntimeError( "Failover request is not valid: %s" % isValid['Message'] )
else:
print("It is a GOGOGO")
requestClient = ReqClient()
result = requestClient.putRequest( request )
print(result)
def archiveRequestAndOp(listOfLFNs):
"""Return a tuple of the request and operation."""
req = Request()
req.RequestName = 'MyRequest'
op = Operation()
switches = {}
archiveLFN = '/vo/tars/myTar.tar'
op.Arguments = DEncode.encode({
'SourceSE':
switches.get('SourceSE', 'SOURCE-SE'),
'TarballSE':
switches.get('TarballSE', 'TARBALL-SE'),
'RegisterDescendent':
False,
'ArchiveLFN':
archiveLFN
})
op.Type = 'ArchiveFiles'
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = '01130a%0d' % index
oFile.ChecksumType = 'adler32'
op.addFile(oFile)
req.addOperation(op)
return req, op
def __insertRegisterOperation( self, request, operation, toRegister ):
""" add RegisterReplica operation
:param Request request: request instance
:param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
:param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
"""
log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )
byTarget = {}
for ftsFile in toRegister:
if ftsFile.TargetSE not in byTarget:
byTarget.setdefault( ftsFile.TargetSE, [] )
byTarget[ftsFile.TargetSE].append( ftsFile )
log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )
for target, ftsFileList in byTarget.iteritems():
log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
len( ftsFileList ) ) )
registerOperation = Operation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
targetSE = StorageElement( target )
for ftsFile in ftsFileList:
opFile = File()
opFile.LFN = ftsFile.LFN
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
if not pfn["OK"]:
continue
opFile.PFN = pfn["Value"]
registerOperation.addFile( opFile )
request.insertBefore( registerOperation, operation )
return S_OK()
def archiveRequestAndOp(listOfLFNs):
"""Return a tuple of the request and operation."""
req = Request()
req.RequestName = "MyRequest"
op = Operation()
switches = {}
archiveLFN = "/vo/tars/myTar.tar"
op.Arguments = DEncode.encode({
"SourceSE":
switches.get("SourceSE", "SOURCE-SE"),
"TarballSE":
switches.get("TarballSE", "TARBALL-SE"),
"RegisterDescendent":
False,
"ArchiveLFN":
archiveLFN,
})
op.Type = "ArchiveFiles"
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = "01130a%0d" % index
oFile.ChecksumType = "adler32"
op.addFile(oFile)
req.addOperation(op)
return req, op
def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE):
""" Set a request to upload job log files from the output sandbox
Changed to be similar to LHCb createLogUploadRequest
using LHCb LogUpload Request and Removal Request
"""
self.log.info('Setting log upload request for %s at %s' %(targetSE, logFileLFN))
request = self._getRequestContainer()
logUpload = Operation()
logUpload.Type = "LogUpload"
logUpload.TargetSE = targetSE
upFile = File()
upFile.LFN = logFileLFN
logUpload.addFile( upFile )
logRemoval = Operation()
logRemoval.Type = 'RemoveFile'
logRemoval.TargetSE = uploadedSE
logRemoval.addFile( upFile )
request.addOperation ( logUpload )
request.addOperation ( logRemoval )
self.workflow_commons['Request'] = request
return S_OK()
class MoveReplicaSuccess( ReqOpsTestCase ):
def setUp(self):
self.op = Operation()
self.op.Type = "MoveFile"
self.op.SourceSE = "%s,%s" % ( "sourceSE1", "sourceSE2" )
self.op.TargetSE = "%s,%s" % ( "targetSE1", "targetSE2" )
self.File = File()
self.File.LFN = '/cta/file1'
self.File.Size = 2L
self.File.Checksum = '011300a2'
self.File.ChecksumType = "adler32"
self.op.addFile( self.File )
self.req = Request()
self.req.addOperation( self.op )
self.mr = MoveReplica( self.op )
self.mr.dm = MagicMock()
self.mr.fc = MagicMock()
# This test needs to be fixed. It currently fails because StorageElement is not mocked
'''def test__dmTransfer( self ):
successful = {}
for sourceSE in self.op.sourceSEList:
successful[sourceSE] = 'dips://' + sourceSE.lower() + ':9148/DataManagement/StorageElement' + self.File.LFN
res = {'OK': True, 'Value': {'Successful': {self.File.LFN : successful}, 'Failed': {}}}
self.mr.dm.getActiveReplicas.return_value = res
res = {'OK': True, 'Value': {'Successful': {self.File.LFN : {'register': 0.1228799819946289, 'replicate': 9.872732877731323}}, 'Failed': {}}}
self.mr.dm.replicateAndRegister.return_value = res
res = self.mr.dmTransfer( self.File )
self.assertTrue( res['OK'] )
self.assertEqual( self.mr.operation.__files__[0].Status, 'Waiting' )
self.assertEqual( self.mr.operation.Status, 'Waiting' )
self.assertEqual( self.mr.request.Status, 'Waiting' )'''
def test__dmRemoval( self ):
res = {'OK': True, 'Value': {'Successful': { self.File.LFN : {'DIRACFileCatalog': True}}, 'Failed': {}}}
self.mr.dm.removeReplica.return_value = res
toRemoveDict = {self.File.LFN: self.File}
targetSEs = self.op.sourceSEList
res = self.mr.dmRemoval( toRemoveDict, targetSEs )
self.assertTrue( res['OK'] )
resvalue = dict( [ ( targetSE, '' ) for targetSE in targetSEs ] )
self.assertEqual( res['Value'], {self.File.LFN: resvalue} )
self.assertEqual( self.mr.operation.__files__[0].Status, 'Done' )
self.assertEqual( self.mr.operation.Status, 'Done' )
self.assertEqual( self.mr.request.Status, 'Done' )
class MoveReplicaSuccess( ReqOpsTestCase ):
def setUp(self):
self.op = Operation()
self.op.Type = "MoveFile"
self.op.SourceSE = "%s,%s" % ( "sourceSE1", "sourceSE2" )
self.op.TargetSE = "%s,%s" % ( "targetSE1", "targetSE2" )
self.File = File()
self.File.LFN = '/cta/file1'
self.File.Size = 2L
self.File.Checksum = '011300a2'
self.File.ChecksumType = "adler32"
self.op.addFile( self.File )
self.req = Request()
self.req.addOperation( self.op )
self.mr = MoveReplica( self.op )
self.mr.dm = MagicMock()
self.mr.fc = MagicMock()
# This test needs to be fixed. It currently fails because StorageElement is not mocked
'''def test__dmTransfer( self ):
successful = {}
for sourceSE in self.op.sourceSEList:
successful[sourceSE] = 'dips://' + sourceSE.lower() + ':9148/DataManagement/StorageElement' + self.File.LFN
res = {'OK': True, 'Value': {'Successful': {self.File.LFN : successful}, 'Failed': {}}}
self.mr.dm.getActiveReplicas.return_value = res
res = {'OK': True, 'Value': {'Successful': {self.File.LFN : {'register': 0.1228799819946289, 'replicate': 9.872732877731323}}, 'Failed': {}}}
self.mr.dm.replicateAndRegister.return_value = res
res = self.mr.dmTransfer( self.File )
self.assertTrue( res['OK'] )
self.assertEqual( self.mr.operation.__files__[0].Status, 'Waiting' )
self.assertEqual( self.mr.operation.Status, 'Waiting' )
self.assertEqual( self.mr.request.Status, 'Waiting' )'''
def test__dmRemoval( self ):
res = {'OK': True, 'Value': {'Successful': { self.File.LFN : {'DIRACFileCatalog': True}}, 'Failed': {}}}
self.mr.dm.removeReplica.return_value = res
toRemoveDict = {self.File.LFN: self.File}
targetSEs = self.op.sourceSEList
res = self.mr.dmRemoval( toRemoveDict, targetSEs )
self.assertTrue( res['OK'] )
resvalue = dict( [ ( targetSE, '' ) for targetSE in targetSEs ] )
self.assertEqual( res['Value'], {self.File.LFN: resvalue} )
self.assertEqual( self.mr.operation.__files__[0].Status, 'Done' )
self.assertEqual( self.mr.operation.Status, 'Done' )
self.assertEqual( self.mr.request.Status, 'Done' )
def createRequest(requestID, opType, opStatus, fileStatus, lfnError=" ",
lfn="/ilc/fake/lfn"):
"""Create a request."""
req = Request({"RequestID": requestID})
op = Operation({"Type": opType, "Status": opStatus})
op.addFile(File({"LFN": lfn, "Status": fileStatus, "Error": lfnError}))
req.addOperation(op)
return req
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def __setFileRemovalRequest(self, lfn):
""" Sets a removal request for a file including all replicas.
"""
self.log.info('Setting file removal request for %s' % lfn)
removeFile = Operation()
removeFile.Type = 'RemoveFile'
rmFile = File()
rmFile.LFN = lfn
removeFile.addFile(rmFile)
self.request.addOperation(removeFile)
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has multiple operations
:param transJson: list of lists of string and dictionaries, e.g.:
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
failedTasks = []
for taskID, task in taskDict.items():
transID = task['TransformationID']
if not task.get('InputData'):
self._logError("Error creating request for task",
"%s, No input data" % taskID,
transID=transID)
taskDict.pop(taskID)
continue
files = []
oRequest = Request()
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
# create the operations from the json structure
for operationTuple in transJson:
op = Operation()
op.Type = operationTuple[0]
for parameter, value in operationTuple[1].iteritems():
setattr(op, parameter, value)
for lfn in files:
opFile = File()
opFile.LFN = lfn
op.addFile(opFile)
oRequest.addOperation(op)
result = self._assignRequestToTask(oRequest, taskDict, transID,
taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has multiple operations
:param transJson: list of lists of string and dictionaries, e.g.:
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
failedTasks = []
for taskID, task in taskDict.items():
transID = task['TransformationID']
if not task.get('InputData'):
self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID)
taskDict.pop(taskID)
continue
files = []
oRequest = Request()
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
# create the operations from the json structure
for operationTuple in transJson:
op = Operation()
op.Type = operationTuple[0]
for parameter, value in operationTuple[1].iteritems():
setattr(op, parameter, value)
for lfn in files:
opFile = File()
opFile.LFN = lfn
op.addFile(opFile)
oRequest.addOperation(op)
result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def addRemovalRequests(self, lfnList):
"""Create removalRequests for lfns in lfnList and add it to the common request"""
request = self._getRequestContainer()
remove = Operation()
remove.Type = "RemoveFile"
for lfn in lfnList:
rmFile = File()
rmFile.LFN = lfn
remove.addFile( rmFile )
request.addOperation( remove )
self.workflow_commons['Request'] = request
def addRemovalRequests(self, lfnList):
"""Create removalRequests for lfns in lfnList and add it to the common request"""
request = self._getRequestContainer()
remove = Operation()
remove.Type = "RemoveFile"
for lfn in lfnList:
rmFile = File()
rmFile.LFN = lfn
remove.addFile( rmFile )
request.addOperation( remove )
self.workflow_commons['Request'] = request
def _setFileRemovalRequest(self, lfn, se="", pfn=""):
"""Sets a removal request for a file including all replicas."""
remove = Operation()
remove.Type = "RemoveFile"
if se:
remove.TargetSE = se
rmFile = File()
rmFile.LFN = lfn
if pfn:
rmFile.PFN = pfn
remove.addFile(rmFile)
self.request.addOperation(remove)
return S_OK()
def buildRequest( self, owner, group, sourceSE, targetSE1, targetSE2 ):
files = self.files( owner, group )
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
putFile = File()
putFile.LFN = lfn
putFile.PFN = fname
putFile.Checksum = checksum
putFile.ChecksumType = "adler32"
putFile.Size = size
putFile.GUID = guid
putAndRegister.addFile( putFile )
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = "%s,%s" % ( targetSE1, targetSE2 )
for fname, lfn, size, checksum, guid in files:
repFile = File()
repFile.LFN = lfn
repFile.Size = size
repFile.Checksum = checksum
repFile.ChecksumType = "adler32"
replicateAndRegister.addFile( repFile )
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
removeReplica.addFile( File( {"LFN": lfn } ) )
removeFile = Operation()
removeFile.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFile.addFile( File( {"LFN": lfn } ) )
removeFileInit = Operation()
removeFileInit.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFileInit.addFile( File( {"LFN": lfn } ) )
req = Request()
req.addOperation( removeFileInit )
req.addOperation( putAndRegister )
req.addOperation( replicateAndRegister )
req.addOperation( removeReplica )
req.addOperation( removeFile )
return req
def buildRequest(self, owner, group, sourceSE, targetSE1, targetSE2):
files = self.files(owner, group)
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
putFile = File()
putFile.LFN = lfn
putFile.PFN = fname
putFile.Checksum = checksum
putFile.ChecksumType = "adler32"
putFile.Size = size
putFile.GUID = guid
putAndRegister.addFile(putFile)
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = "%s,%s" % (targetSE1, targetSE2)
for fname, lfn, size, checksum, guid in files:
repFile = File()
repFile.LFN = lfn
repFile.Size = size
repFile.Checksum = checksum
repFile.ChecksumType = "adler32"
replicateAndRegister.addFile(repFile)
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
removeReplica.addFile(File({"LFN": lfn}))
removeFile = Operation()
removeFile.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFile.addFile(File({"LFN": lfn}))
removeFileInit = Operation()
removeFileInit.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFileInit.addFile(File({"LFN": lfn}))
req = Request()
req.addOperation(removeFileInit)
req.addOperation(putAndRegister)
req.addOperation(replicateAndRegister)
req.addOperation(removeReplica)
req.addOperation(removeFile)
return req
class RequestManagerHandlerTests( unittest.TestCase ):
"""
.. class:: RequestManagerHandlerTests
"""
def setUp( self ):
""" test setup
:param self: self reference
"""
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
self.request.OwnerGroup = "dirac_user"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.request.addOperation( self.operation )
self.operation.addFile( self.file )
# # xml representation of a whole request
self.xmlStr = self.request.toXML( True )["Value"]
# # request client
self.requestClient = RequestClient()
def tearDown( self ):
""" test case tear down """
del self.request
del self.operation
del self.file
del self.xmlStr
def test01PutRequest( self ):
""" test set request """
put = self.requestClient.putRequest( self.request )
self.assertEqual( put["OK"], True, "put failed" )
def test02GetRequest( self ):
""" test get request """
get = self.requestClient.getRequest( self.request.RequestName )
self.assertEqual( get["OK"], True, "get failed" )
def test03DeleteRequest( self ):
""" test delete request """
delete = self.requestClient.deleteRequest( "test" )
self.assertEqual( delete["OK"], True, "delete failed" )
def test_StateMachine():
"""state machine"""
op = Operation()
assert op.Status == "Queued", "1. wrong status %s" % op.Status
op.addFile(File({"Status": "Waiting"}))
assert op.Status == "Queued", "2. wrong status %s" % op.Status
op.addFile(File({"Status": "Scheduled"}))
assert op.Status == "Scheduled", "3. wrong status %s" % op.Status
op.addFile(File({"Status": "Done"}))
assert op.Status == "Scheduled", "4. wrong status %s" % op.Status
op.addFile(File({"Status": "Failed"}))
assert op.Status == "Scheduled", "5. wrong status %s" % op.Status
op[3].Status = "Scheduled"
assert op.Status == "Scheduled", "6. wrong status %s" % op.Status
op[0].Status = "Scheduled"
assert op.Status == "Scheduled", "7. wrong status %s" % op.Status
op[0].Status = "Waiting"
assert op.Status == "Scheduled", "8. wrong status %s" % op.Status
for f in op:
f.Status = "Done"
assert op.Status == "Done", "9. wrong status %s" % op.Status
for f in op:
f.Status = "Failed"
assert op.Status == "Failed", "9. wrong status %s" % op.Status
def test04StateMachine(self):
""" state machine """
op = Operation()
self.assertEqual(op.Status, "Queued", "1. wrong status %s" % op.Status)
op.addFile(File({"Status": "Waiting"}))
self.assertEqual(op.Status, "Queued", "2. wrong status %s" % op.Status)
op.addFile(File({"Status": "Scheduled"}))
self.assertEqual(op.Status, "Queued", "3. wrong status %s" % op.Status)
op.addFile(File({"Status": "Done"}))
self.assertEqual(op.Status, "Queued", "4. wrong status %s" % op.Status)
op.addFile(File({"Status": "Failed"}))
self.assertEqual(op.Status, "Failed", "5. wrong status %s" % op.Status)
op[3].Status = "Scheduled"
self.assertEqual(op.Status, "Queued", "6. wrong status %s" % op.Status)
op[0].Status = "Scheduled"
self.assertEqual(op.Status, "Scheduled",
"7. wrong status %s" % op.Status)
op[0].Status = "Waiting"
self.assertEqual(op.Status, "Queued", "8. wrong status %s" % op.Status)
for f in op:
f.Status = "Done"
self.assertEqual(op.Status, "Done", "9. wrong status %s" % op.Status)
def test04StateMachine( self ):
""" state machine """
op = Operation()
self.assertEqual( op.Status, "Queued", "1. wrong status %s" % op.Status )
op.addFile( File( {"Status": "Waiting"} ) )
self.assertEqual( op.Status, "Queued", "2. wrong status %s" % op.Status )
op.addFile( File( {"Status": "Scheduled" } ) )
self.assertEqual( op.Status, "Scheduled", "3. wrong status %s" % op.Status )
op.addFile( File( {"Status": "Done" } ) )
self.assertEqual( op.Status, "Scheduled", "4. wrong status %s" % op.Status )
op.addFile( File( { "Status": "Failed" } ) )
self.assertEqual( op.Status, "Scheduled", "5. wrong status %s" % op.Status )
op[3].Status = "Scheduled"
self.assertEqual( op.Status, "Scheduled", "6. wrong status %s" % op.Status )
op[0].Status = "Scheduled"
self.assertEqual( op.Status, "Scheduled", "7. wrong status %s" % op.Status )
op[0].Status = "Waiting"
self.assertEqual( op.Status, "Scheduled", "8. wrong status %s" % op.Status )
for f in op:
f.Status = "Done"
self.assertEqual( op.Status, "Done", "9. wrong status %s" % op.Status )
for f in op:
f.Status = "Failed"
self.assertEqual( op.Status, "Failed", "9. wrong status %s" % op.Status )
def checkRequestAndOp(listOfLFNs):
req = Request()
req.RequestName = 'MyRequest'
op = Operation()
op.Type = 'CheckMigration'
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = '01130a%0d' % index
oFile.ChecksumType = 'adler32'
op.addFile(oFile)
req.addOperation(op)
return req, op
def __setFileRemovalRequest( self, lfn, se = '', pfn = '' ):
""" Sets a removal request for a file including all replicas.
"""
remove = Operation()
remove.Type = "RemoveFile"
if se:
remove.TargetSE = se
rmFile = File()
rmFile.LFN = lfn
if pfn:
rmFile.PFN = pfn
remove.addFile( rmFile )
self.request.addOperation( remove )
return S_OK()
class ReqClientTestCase( unittest.TestCase ):
"""
.. class:: ReqClientTestCase
"""
def setUp( self ):
""" test case set up """
gLogger.setLevel( 'INFO' )
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.file2 = File()
self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
self.file2.Checksum = "654321"
self.file2.ChecksumType = "ADLER32"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.operation.addFile( self.file )
self.operation.addFile( self.file2 )
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
self.request.OwnerGroup = "dirac_user"
self.request.JobID = 123
self.request.addOperation( self.operation )
# # JSON representation of a whole request
self.jsonStr = self.request.toJSON()['Value']
# # request client
self.requestClient = ReqClient()
self.stressRequests = 1000
self.bulkRequest = 1000
def tearDown( self ):
""" clean up """
del self.request
del self.operation
del self.file
del self.jsonStr
class ReqClientTestCase(unittest.TestCase):
"""
.. class:: ReqClientTestCase
"""
def setUp(self):
""" test case set up """
gLogger.setLevel('INFO')
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.file2 = File()
self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
self.file2.Checksum = "654321"
self.file2.ChecksumType = "ADLER32"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.operation.addFile(self.file)
self.operation.addFile(self.file2)
proxyInfo = getProxyInfo()['Value']
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = proxyInfo['identity']
self.request.OwnerGroup = proxyInfo['group']
self.request.JobID = 123
self.request.addOperation(self.operation)
# # JSON representation of a whole request
self.jsonStr = self.request.toJSON()['Value']
# # request client
self.requestClient = ReqClient()
self.stressRequests = 1000
self.bulkRequest = 1000
def tearDown(self):
""" clean up """
del self.request
del self.operation
del self.file
del self.jsonStr
class ReqClientTestCase( unittest.TestCase ):
"""
.. class:: ReqClientTestCase
"""
def setUp( self ):
""" test case set up """
gLogger.setLevel( 'NOTICE' )
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.file2 = File()
self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
self.file2.Checksum = "654321"
self.file2.ChecksumType = "ADLER32"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.operation.addFile( self.file )
self.operation.addFile( self.file2 )
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
self.request.OwnerGroup = "dirac_user"
self.request.JobID = 123
self.request.addOperation( self.operation )
# # JSON representation of a whole request
self.jsonStr = self.request.toJSON()['Value']
# # request client
self.requestClient = ReqClient()
self.stressRequests = 1000
self.bulkRequest = 1000
def tearDown( self ):
""" clean up """
del self.request
del self.operation
del self.file
del self.jsonStr
def checkRequestAndOp(listOfLFNs):
req = Request()
req.RequestName = "MyRequest"
op = Operation()
op.Type = "CheckMigration"
op.TargetSE = "Foo-SE"
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = "01130a%0d" % index
oFile.ChecksumType = "adler32"
op.addFile(oFile)
req.addOperation(op)
return req, op
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
"""deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = "ReplicateAndRegister"
if transBody:
try:
_requestType, requestOperation = transBody.split(";")
except AttributeError:
pass
failedTasks = []
# Do not remove sorted, we might pop elements in the loop
for taskID, task in taskDict.items():
transID = task["TransformationID"]
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = task["TargetSE"]
# If there are input files
if task.get("InputData"):
if isinstance(task["InputData"], list):
files = task["InputData"]
elif isinstance(task["InputData"], six.string_types):
files = task["InputData"].split(";")
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
result = self._assignRequestToTask(oRequest, taskDict, transID,
taskID, ownerDN, ownerGroup)
if not result["OK"]:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split(';')
except AttributeError:
pass
failedTasks = []
# Do not remove sorted, we might pop elements in the loop
for taskID, task in taskDict.iteritems():
transID = task['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = task['TargetSE']
# If there are input files
if task.get('InputData'):
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
if self.getCSOption("DelayedExternalDeletion", True):
gLogger.info("Setting deletion request")
try:
# We need the hostDN used in order to pass these credentials to the
# SandboxStoreDB..
hostCertLocation, _ = Locations.getHostCertificateAndKeyLocation(
)
hostCert = X509Certificate.X509Certificate()
hostCert.loadFromFile(hostCertLocation)
hostDN = hostCert.getSubjectDN().get("Value")
# use the host authentication to fetch the data
result = self.sandboxDB.getSandboxOwner(
SEName, SEPFN, hostDN, "hosts")
if not result["OK"]:
return result
_owner, ownerDN, ownerGroup = result["Value"]
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % (
SEName, SEPFN, time.time())
request.OwnerDN = ownerDN
request.OwnerGroup = ownerGroup
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile(fileToRemove)
request.addOperation(physicalRemoval)
return ReqClient().putRequest(request)
except Exception as e:
gLogger.exception("Exception while setting deletion request")
return S_ERROR(f"Cannot set deletion request: {e}")
else:
gLogger.info("Deleting external Sandbox")
try:
return StorageElement(SEName).removeFile(SEPFN)
except Exception:
gLogger.exception(
"RM raised an exception while trying to delete a remote sandbox"
)
return S_ERROR(
"RM raised an exception while trying to delete a remote sandbox"
)
class ReqClientTestCase(unittest.TestCase):
"""
.. class:: ReqClientTestCase
"""
def setUp(self):
""" test case set up """
gLogger.setLevel('INFO')
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.file2 = File()
self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
self.file2.Checksum = "654321"
self.file2.ChecksumType = "ADLER32"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.operation.addFile(self.file)
self.operation.addFile(self.file2)
proxyInfo = getProxyInfo()['Value']
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = proxyInfo['identity']
self.request.OwnerGroup = proxyInfo['group']
self.request.JobID = 123
self.request.addOperation(self.operation)
# # JSON representation of a whole request
self.jsonStr = self.request.toJSON()['Value']
# # request client
self.requestClient = ReqClient()
def tearDown(self):
""" clean up """
del self.request
del self.operation
del self.file
del self.jsonStr
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
""" Set removal request with the given credentials
"""
oRequest = Request()
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml'
oRequest.SourceComponent = 'JobCleaningAgent'
removeFile = Operation()
removeFile.Type = 'RemoveFile'
removedFile = File()
removedFile.LFN = lfn
removeFile.addFile( removedFile )
oRequest.addOperation( removeFile )
return ReqClient().putRequest( oRequest )
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
""" Set removal request with the given credentials
"""
oRequest = Request()
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml'
oRequest.SourceComponent = 'JobCleaningAgent'
removeFile = Operation()
removeFile.Type = 'RemoveFile'
removedFile = File()
removedFile.LFN = lfn
removeFile.addFile( removedFile )
oRequest.addOperation( removeFile )
return ReqClient().putRequest( oRequest )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if type( paramDict['InputData'] ) == type( [] ):
files = paramDict['InputData']
elif type( paramDict['InputData'] ) == type( '' ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
oRequest.OwnerDN = owner
oRequest.OwnerGroup = ownerGroup
isValid = gRequestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def getRegisterOperation( self, opFile, targetSE ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = "RegisterFile"
registerOperation.TargetSE = targetSE
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = opFile.PFN
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
return registerOperation
def __setReplicaRemovalRequest( self, lfn, se ):
""" Sets a removal request for a replica.
:param str lfn: LFN
:param se:
"""
if type( se ) == str:
se = ",".join( [ se.strip() for se in se.split( "," ) if se.strip() ] )
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = se
replicaToRemove = File()
replicaToRemove.LFN = lfn
removeReplica.addFile( replicaToRemove )
self.request.addOperation( removeReplica )
return S_OK()
def addRegisterReplica( self, opFile, targetSE ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = "RegisterFile"
registerOperation.TargetSE = targetSE
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = opFile.PFN
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
self.request.insertAfter( registerOperation, self.operation )
return S_OK()
def _setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )
if not isinstance( catalog, list ):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get( "Checksum", "" )
regFile.ChecksumType = fileDict.get( "ChecksumType", self.defaultChecksumType )
regFile.Size = fileDict.get( "Size", 0 )
regFile.GUID = fileDict.get( "GUID", "" )
se = StorageElement( targetSE )
pfn = se.getURL( lfn, self.registrationProtocols )
if not pfn["OK"] or lfn not in pfn["Value"]['Successful']:
self.log.error( "Unable to get PFN for LFN", "%s" % pfn.get( 'Message', pfn.get( 'Value', {} ).get( 'Failed', {} ).get( lfn ) ) )
return pfn
regFile.PFN = pfn["Value"]['Successful'][lfn]
register.addFile( regFile )
self.request.addOperation( register )
return S_OK()
def getRegisterOperation( self, opFile, targetSE, type = 'RegisterFile', catalog = None ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = StorageElement( targetSE ).getURL( opFile.LFN, protocol = self.registrationProtocols ).get( 'Value', {} ).get( 'Successful', {} ).get( opFile.LFN )
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
return registerOperation
print "Can't get file metadata: %s" % res['Message']
DIRAC.exit( 1 )
if res['Value']['Failed']:
print "Could not get the file metadata of the following, so skipping them:"
for fFile in res['Value']['Failed']:
print fFile
lfnMetadata = res['Value']['Successful']
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
rarFile.Size = lfnMetadata[lfn]['Size']
rarFile.Checksum = lfnMetadata[lfn]['Checksum']
rarFile.GUID = lfnMetadata[lfn]['GUID']
rarFile.ChecksumType = 'ADLER32'
replicateAndRegister.addFile( rarFile )
oRequest.addOperation( replicateAndRegister )
isValid = RequestValidator().validate( oRequest )
if not isValid['OK']:
print "Request is not valid: ", isValid['Message']
DIRAC.exit( 1 )
result = reqClient.putRequest( oRequest )
if result['OK']:
print "Request %d submitted successfully" % result['Value']
else:
print "Failed to submit Request: ", result['Message']
def getRequest( self, requestName = '', assigned = True ):
""" read request for execution
:param str requestName: request's name (default None)
"""
requestID = None
if requestName:
self.log.info( "getRequest: selecting request '%s'" % requestName )
reqIDQuery = "SELECT `RequestID`, `Status` FROM `Request` WHERE `RequestName` = '%s';" % str( requestName )
reqID = self._transaction( reqIDQuery )
if not reqID["OK"]:
self.log.error( "getRequest: %s" % reqID["Message"] )
return reqID
requestID = reqID["Value"][reqIDQuery][0]["RequestID"] if "RequestID" in reqID["Value"][reqIDQuery][0] else None
status = reqID["Value"][reqIDQuery][0]["Status"] if "Status" in reqID["Value"][reqIDQuery][0] else None
if not all( ( requestID, status ) ):
return S_ERROR( "getRequest: request '%s' not exists" % requestName )
if requestID and status and status == "Assigned" and assigned:
return S_ERROR( "getRequest: status of request '%s' is 'Assigned', request cannot be selected" % requestName )
else:
reqIDsQuery = "SELECT `RequestID` FROM `Request` WHERE `Status` = 'Waiting' ORDER BY `LastUpdate` ASC LIMIT 100;"
reqIDs = self._transaction( reqIDsQuery )
if not reqIDs["OK"]:
self.log.error( "getRequest: %s" % reqIDs["Message"] )
return reqIDs
reqIDs = reqIDs["Value"][reqIDsQuery]
reqIDs = [ reqID["RequestID"] for reqID in reqIDs ]
if not reqIDs:
return S_OK()
random.shuffle( reqIDs )
requestID = reqIDs[0]
selectQuery = [ "SELECT * FROM `Request` WHERE `RequestID` = %s;" % requestID,
"SELECT * FROM `Operation` WHERE `RequestID` = %s;" % requestID ]
selectReq = self._transaction( selectQuery )
if not selectReq["OK"]:
self.log.error( "getRequest: %s" % selectReq["Message"] )
return S_ERROR( selectReq["Message"] )
selectReq = selectReq["Value"]
request = Request( selectReq[selectQuery[0]][0] )
for records in sorted( selectReq[selectQuery[1]], key = lambda k: k["Order"] ):
# # order is ro, remove
del records["Order"]
operation = Operation( records )
getFilesQuery = "SELECT * FROM `File` WHERE `OperationID` = %s;" % operation.OperationID
getFiles = self._transaction( getFilesQuery )
if not getFiles["OK"]:
self.log.error( "getRequest: %s" % getFiles["Message"] )
return getFiles
getFiles = getFiles["Value"][getFilesQuery]
for getFile in getFiles:
getFileDict = dict( [ ( key, value ) for key, value in getFile.items() if value != None ] )
operation.addFile( File( getFileDict ) )
request.addOperation( operation )
if assigned:
setAssigned = self._transaction( "UPDATE `Request` SET `Status` = 'Assigned' WHERE RequestID = %s;" % requestID )
if not setAssigned["OK"]:
self.log.error( "getRequest: %s" % setAssigned["Message"] )
return setAssigned
return S_OK( request )
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = ",".join( targetSEs )
if catalog is not None:
replicateAndRegister.Catalog = catalog
for lfn in lfnChunk:
metaDict = metaDatas["Successful"][lfn]
opFile = File()
opFile.LFN = lfn
opFile.Size = metaDict["Size"]
if "Checksum" in metaDict:
# # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
opFile.Checksum = metaDict["Checksum"]
opFile.ChecksumType = "ADLER32"
replicateAndRegister.addFile( opFile )
request.addOperation( replicateAndRegister )
putRequest = reqClient.putRequest( request )
if not putRequest["OK"]:
gLogger.error( "unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) )
error = -1
continue
requestIDs.append( str( putRequest["Value"] ) )
if not multiRequests:
gLogger.always( "Request '%s' has been put to ReqDB for execution." % request.RequestName )
if multiRequests:
gLogger.always( "%d requests have been put to ReqDB for execution, with name %s_<num>" % ( count, requestName ) )
if requestIDs:
class RequestDBTests(unittest.TestCase):
"""
.. class:: RequestDBTests
unittest for RequestDB
"""
def setUp(self):
""" test case setup """
self.request = Request({"RequestName": "test1", "JobID": 1})
self.operation1 = Operation({"Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"})
self.file = File({"LFN": "/a/b/c", "ChecksumType": "ADLER32", "Checksum": "123456"})
self.request.addOperation(self.operation1)
self.operation1.addFile(self.file)
self.operation2 = Operation()
self.operation2.Type = "RemoveFile"
self.operation2.addFile(File({"LFN": "/c/d/e"}))
self.request.addOperation(self.operation2)
# ## set some defaults
gConfig.setOptionValue("DIRAC/Setup", "Test")
gConfig.setOptionValue("/DIRAC/Setups/Test/RequestManagement", "Test")
gConfig.setOptionValue("/Systems/RequestManagement/Test/Databases/ReqDB/Host", "localhost")
gConfig.setOptionValue("/Systems/RequestManagement/Test/Databases/ReqDB/DBName", "ReqDB")
gConfig.setOptionValue("/Systems/RequestManagement/Test/Databases/ReqDB/User", "Dirac")
self.i = 1000
def tearDown(self):
""" test case tear down """
del self.file
del self.operation1
del self.operation2
del self.request
def test01TableDesc(self):
""" table description """
tableDict = RequestDB.getTableMeta()
self.assertEqual("Request" in tableDict, True)
self.assertEqual("Operation" in tableDict, True)
self.assertEqual("File" in tableDict, True)
self.assertEqual(tableDict["Request"], Request.tableDesc())
self.assertEqual(tableDict["Operation"], Operation.tableDesc())
self.assertEqual(tableDict["File"], File.tableDesc())
def test03RequestRW(self):
""" db r/w requests """
db = RequestDB()
db._checkTables(True)
# # empty DB at that stage
ret = db.getDBSummary()
self.assertEqual(ret, {"OK": True, "Value": {"Operation": {}, "Request": {}, "File": {}}})
# # insert
ret = db.putRequest(self.request)
self.assertEqual(ret, {"OK": True, "Value": ""})
# # get digest -> JSON
ret = db.getDigest(self.request.RequestName)
self.assertEqual(ret["OK"], True)
self.assertEqual(bool(ret["Value"]), True)
# # db summary
ret = db.getDBSummary()
self.assertEqual(
ret,
{
"OK": True,
"Value": {
"Operation": {"RemoveFile": {"Queued": 1L}, "ReplicateAndRegister": {"Waiting": 1L}},
"Request": {"Waiting": 1L},
"File": {"Waiting": 2L},
},
},
)
# # get request for jobs
ret = db.getRequestNamesForJobs([1])
self.assertEqual(ret["OK"], True)
self.assertEqual(ret["Value"], {1: "test1"})
# # read requests
ret = db.readRequestsForJobs([1])
self.assertEqual(ret["OK"], True)
self.assertEqual(ret["Value"][1]["OK"], True)
# # select
ret = db.getRequest()
self.assertEqual(ret["OK"], True)
request = ret["Value"]
self.assertEqual(isinstance(request, Request), True)
# # summary
ret = db.getDBSummary()
self.assertEqual(
ret,
{
"OK": True,
"Value": {
"Operation": {"RemoveFile": {"Queued": 1L}, "ReplicateAndRegister": {"Waiting": 1L}},
"Request": {"Assigned": 1L},
"File": {"Waiting": 2L},
},
},
)
# # update
ret = db.putRequest(request)
self.assertEqual(ret, {"OK": True, "Value": ""})
# # get summary again
ret = db.getDBSummary()
self.assertEqual(
ret,
{
"OK": True,
"Value": {
"Operation": {"RemoveFile": {"Queued": 1L}, "ReplicateAndRegister": {"Waiting": 1L}},
"Request": {"Waiting": 1L},
"File": {"Waiting": 2L},
},
},
)
# # delete
ret = db.deleteRequest(self.request.RequestName)
self.assertEqual(ret, {"OK": True, "Value": ""})
# # should be empty now
ret = db.getDBSummary()
self.assertEqual(ret, {"OK": True, "Value": {"Operation": {}, "Request": {}, "File": {}}})
def test04Stress(self):
""" stress test """
db = RequestDB()
for i in range(self.i):
request = Request({"RequestName": "test-%d" % i})
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
put = db.putRequest(request)
self.assertEqual(put["OK"], True, "put failed")
for i in range(self.i):
get = db.getRequest("test-%s" % i, False)
if "Message" in get:
print get["Message"]
self.assertEqual(get["OK"], True, "get failed")
for i in range(self.i):
delete = db.deleteRequest("test-%s" % i)
self.assertEqual(delete["OK"], True, "delete failed")
def test05Scheduled(self):
""" scheduled request r/w """
db = RequestDB()
req = Request({"RequestName": "FTSTest"})
op = Operation({"Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"})
op += File({"LFN": "/a/b/c", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32"})
req += op
put = db.putRequest(req)
self.assertEqual(put["OK"], True, "putRequest failed")
peek = db.peekRequest(req.RequestName)
self.assertEqual(peek["OK"], True, "peek failed ")
peek = peek["Value"]
for op in peek:
opId = op.OperationID
getFTS = db.getScheduledRequest(opId)
self.assertEqual(getFTS["OK"], True, "getScheduled failed")
self.assertEqual(getFTS["Value"].RequestName, "FTSTest", "wrong request selected")
def test06Dirty(self):
""" dirty records """
db = RequestDB()
r = Request()
r.RequestName = "dirty"
op1 = Operation({"Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"})
op1 += File({"LFN": "/a/b/c/1", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32"})
op2 = Operation({"Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"})
op2 += File({"LFN": "/a/b/c/2", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32"})
op3 = Operation({"Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"})
op3 += File({"LFN": "/a/b/c/3", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32"})
r += op1
r += op2
r += op3
put = db.putRequest(r)
self.assertEqual(put["OK"], True, "1. putRequest failed: %s" % put.get("Message", ""))
r = db.getRequest("dirty")
self.assertEqual(r["OK"], True, "1. getRequest failed: %s" % r.get("Message", ""))
r = r["Value"]
del r[0]
self.assertEqual(len(r), 2, "1. len wrong")
put = db.putRequest(r)
self.assertEqual(put["OK"], True, "2. putRequest failed: %s" % put.get("Message", ""))
r = db.getRequest("dirty")
self.assertEqual(r["OK"], True, "2. getRequest failed: %s" % r.get("Message", ""))
r = r["Value"]
self.assertEqual(len(r), 2, "2. len wrong")
op4 = Operation({"Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"})
op4 += File({"LFN": "/a/b/c/4", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32"})
r[0] = op4
put = db.putRequest(r)
self.assertEqual(put["OK"], True, "3. putRequest failed: %s" % put.get("Message", ""))
r = db.getRequest("dirty")
self.assertEqual(r["OK"], True, "3. getRequest failed: %s" % r.get("Message", ""))
r = r["Value"]
self.assertEqual(len(r), 2, "3. len wrong")