def myRequest():
"""Create a request and put it to the db"""
request = Request()
request.RequestName = 'myAwesomeRemovalRequest.xml'
request.JobID = 0
request.SourceComponent = "myScript"
remove = Operation()
remove.Type = "RemoveFile"
lfn = "/ilc/user/s/sailer/test.txt"
rmFile = File()
rmFile.LFN = lfn
remove.addFile( rmFile )
request.addOperation( remove )
isValid = RequestValidator().validate( request )
if not isValid['OK']:
raise RuntimeError( "Failover request is not valid: %s" % isValid['Message'] )
else:
print("It is a GOGOGO")
requestClient = ReqClient()
result = requestClient.putRequest( request )
print(result)
def __setFileReplicationRequest( self, lfn, se, fileMetaDict ):
""" Sets a registration request.
"""
self.log.info( 'Setting replication request for %s to %s' % ( lfn, se ) )
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.TargetSE = se
trFile = File()
trFile.LFN = lfn
cksm = fileMetaDict.get( "Checksum", None )
cksmType = fileMetaDict.get( "ChecksumType", None )
if cksm and cksmType:
trFile.Checksum = cksm
trFile.ChecksumType = cksmType
size = fileMetaDict.get( "Size", 0 )
if size:
trFile.Size = size
guid = fileMetaDict.get( "GUID", "" )
if guid:
trFile.GUID = guid
transfer.addFile( trFile )
self.request.addOperation( transfer )
return S_OK()
def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE):
""" Set a request to upload job log files from the output sandbox
Changed to be similar to LHCb createLogUploadRequest
using LHCb LogUpload Request and Removal Request
"""
self.log.info('Setting log upload request for %s at %s' %(targetSE, logFileLFN))
request = self._getRequestContainer()
logUpload = Operation()
logUpload.Type = "LogUpload"
logUpload.TargetSE = targetSE
upFile = File()
upFile.LFN = logFileLFN
logUpload.addFile( upFile )
logRemoval = Operation()
logRemoval.Type = 'RemoveFile'
logRemoval.TargetSE = uploadedSE
logRemoval.addFile( upFile )
request.addOperation ( logUpload )
request.addOperation ( logRemoval )
self.workflow_commons['Request'] = request
return S_OK()
def test01ctor( self ):
""" test constructors and (de)serialisation """
# # empty ctor
self.assertEqual( isinstance( Operation(), Operation ), True, "empty ctor failed" )
# # using fromDict
operation = Operation( self.fromDict )
self.assertEqual( isinstance( operation, Operation ), True, "fromDict ctor failed" )
for key, value in self.fromDict.items():
self.assertEqual( getattr( operation, key ), value, "wrong attr value %s (%s) %s" % ( key,
getattr( operation, key ),
value ) )
# # same with file
operation = Operation( self.fromDict )
operation.addFile( self.subFile )
for key, value in self.fromDict.items():
self.assertEqual( getattr( operation, key ), value, "wrong attr value %s (%s) %s" % ( key,
getattr( operation, key ),
value ) )
toJSON = operation.toJSON()
self.assertEqual( toJSON["OK"], True, "JSON serialization failed" )
def test02props( self ):
""" test properties """
# # valid values
operation = Operation()
operation.OperationID = 1
self.assertEqual( operation.OperationID, 1, "wrong OperationID" )
operation.OperationID = "1"
self.assertEqual( operation.OperationID, 1, "wrong OperationID" )
operation.Arguments = "foobar"
self.assertEqual( operation.Arguments, "foobar", "wrong Arguments" )
operation.SourceSE = "CERN-RAW"
self.assertEqual( operation.SourceSE, "CERN-RAW", "wrong SourceSE" )
operation.TargetSE = "CERN-RAW"
self.assertEqual( operation.TargetSE, "CERN-RAW", "wrong TargetSE" )
operation.Catalog = ""
self.assertEqual( operation.Catalog, "", "wrong Catalog" )
operation.Catalog = "BookkeepingDB"
self.assertEqual( operation.Catalog, "BookkeepingDB", "wrong Catalog" )
operation.Error = "error"
self.assertEqual( operation.Error, "error", "wrong Error" )
# # wrong props
try:
operation.RequestID = "foo"
except Exception, error:
self.assertEqual( type( error ), AttributeError, "wrong exc raised" )
self.assertEqual( str( error ), "can't set attribute", "wrong exc reason" )
def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ):
if self.getCSOption( "DelayedExternalDeletion", True ):
gLogger.info( "Setting deletion request" )
try:
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() )
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile( fileToRemove )
request.addOperation( physicalRemoval )
return ReqClient().putRequest( request )
except Exception as e:
gLogger.exception( "Exception while setting deletion request" )
return S_ERROR( "Cannot set deletion request: %s" % str( e ) )
else:
gLogger.info( "Deleting external Sandbox" )
try:
return StorageElement( SEName ).removeFile( SEPFN )
except Exception as e:
gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" )
return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
class MoveReplicaSuccess( ReqOpsTestCase ):
def setUp(self):
self.op = Operation()
self.op.Type = "MoveFile"
self.op.SourceSE = "%s,%s" % ( "sourceSE1", "sourceSE2" )
self.op.TargetSE = "%s,%s" % ( "targetSE1", "targetSE2" )
self.File = File()
self.File.LFN = '/cta/file1'
self.File.Size = 2L
self.File.Checksum = '011300a2'
self.File.ChecksumType = "adler32"
self.op.addFile( self.File )
self.req = Request()
self.req.addOperation( self.op )
self.mr = MoveReplica( self.op )
self.mr.dm = MagicMock()
self.mr.fc = MagicMock()
# This test needs to be fixed. It currently fails because StorageElement is not mocked
'''def test__dmTransfer( self ):
successful = {}
for sourceSE in self.op.sourceSEList:
successful[sourceSE] = 'dips://' + sourceSE.lower() + ':9148/DataManagement/StorageElement' + self.File.LFN
res = {'OK': True, 'Value': {'Successful': {self.File.LFN : successful}, 'Failed': {}}}
self.mr.dm.getActiveReplicas.return_value = res
res = {'OK': True, 'Value': {'Successful': {self.File.LFN : {'register': 0.1228799819946289, 'replicate': 9.872732877731323}}, 'Failed': {}}}
self.mr.dm.replicateAndRegister.return_value = res
res = self.mr.dmTransfer( self.File )
self.assertTrue( res['OK'] )
self.assertEqual( self.mr.operation.__files__[0].Status, 'Waiting' )
self.assertEqual( self.mr.operation.Status, 'Waiting' )
self.assertEqual( self.mr.request.Status, 'Waiting' )'''
def test__dmRemoval( self ):
res = {'OK': True, 'Value': {'Successful': { self.File.LFN : {'DIRACFileCatalog': True}}, 'Failed': {}}}
self.mr.dm.removeReplica.return_value = res
toRemoveDict = {self.File.LFN: self.File}
targetSEs = self.op.sourceSEList
res = self.mr.dmRemoval( toRemoveDict, targetSEs )
self.assertTrue( res['OK'] )
resvalue = dict( [ ( targetSE, '' ) for targetSE in targetSEs ] )
self.assertEqual( res['Value'], {self.File.LFN: resvalue} )
self.assertEqual( self.mr.operation.__files__[0].Status, 'Done' )
self.assertEqual( self.mr.operation.Status, 'Done' )
self.assertEqual( self.mr.request.Status, 'Done' )
def createRequest(requestID, opType, opStatus, fileStatus, lfnError=" ",
lfn="/ilc/fake/lfn"):
"""Create a request."""
req = Request({"RequestID": requestID})
op = Operation({"Type": opType, "Status": opStatus})
op.addFile(File({"LFN": lfn, "Status": fileStatus, "Error": lfnError}))
req.addOperation(op)
return req
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def test04StateMachine( self ):
""" state machine """
op = Operation()
self.assertEqual( op.Status, "Queued", "1. wrong status %s" % op.Status )
op.addFile( File( {"Status": "Waiting"} ) )
self.assertEqual( op.Status, "Queued", "2. wrong status %s" % op.Status )
op.addFile( File( {"Status": "Scheduled" } ) )
self.assertEqual( op.Status, "Scheduled", "3. wrong status %s" % op.Status )
op.addFile( File( {"Status": "Done" } ) )
self.assertEqual( op.Status, "Scheduled", "4. wrong status %s" % op.Status )
op.addFile( File( { "Status": "Failed" } ) )
self.assertEqual( op.Status, "Scheduled", "5. wrong status %s" % op.Status )
op[3].Status = "Scheduled"
self.assertEqual( op.Status, "Scheduled", "6. wrong status %s" % op.Status )
op[0].Status = "Scheduled"
self.assertEqual( op.Status, "Scheduled", "7. wrong status %s" % op.Status )
op[0].Status = "Waiting"
self.assertEqual( op.Status, "Scheduled", "8. wrong status %s" % op.Status )
for f in op:
f.Status = "Done"
self.assertEqual( op.Status, "Done", "9. wrong status %s" % op.Status )
for f in op:
f.Status = "Failed"
self.assertEqual( op.Status, "Failed", "9. wrong status %s" % op.Status )
def _sendToFailover( rpcStub ): """ Create a ForwardDISET operation for failover """ request = Request() request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ) forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode( rpcStub ) request.addOperation( forwardDISETOp ) return ReqClient().putRequest( request )
def generateForwardDISET( self ):
""" Commit the accumulated records and generate request eventually """
result = self.commit()
commitOp = None
if not result['OK']:
# Generate Request
commitOp = Operation()
commitOp.Type = 'SetFileStatus'
commitOp.Arguments = DEncode.encode( {'transformation':self.transformation, 'statusDict':self.statusDict, 'force':self.force} )
return S_OK( commitOp )
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has multiple operations
:param transJson: list of lists of string and dictionaries, e.g.:
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
failedTasks = []
for taskID, task in taskDict.items():
transID = task['TransformationID']
if not task.get('InputData'):
self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID)
taskDict.pop(taskID)
continue
files = []
oRequest = Request()
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
# create the operations from the json structure
for operationTuple in transJson:
op = Operation()
op.Type = operationTuple[0]
for parameter, value in operationTuple[1].iteritems():
setattr(op, parameter, value)
for lfn in files:
opFile = File()
opFile.LFN = lfn
op.addFile(opFile)
oRequest.addOperation(op)
result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def addRemovalRequests(self, lfnList):
"""Create removalRequests for lfns in lfnList and add it to the common request"""
request = self._getRequestContainer()
remove = Operation()
remove.Type = "RemoveFile"
for lfn in lfnList:
rmFile = File()
rmFile.LFN = lfn
remove.addFile( rmFile )
request.addOperation( remove )
self.workflow_commons['Request'] = request
class RequestManagerHandlerTests( unittest.TestCase ):
"""
.. class:: RequestManagerHandlerTests
"""
def setUp( self ):
""" test setup
:param self: self reference
"""
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
self.request.OwnerGroup = "dirac_user"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.request.addOperation( self.operation )
self.operation.addFile( self.file )
# # xml representation of a whole request
self.xmlStr = self.request.toXML( True )["Value"]
# # request client
self.requestClient = RequestClient()
def tearDown( self ):
""" test case tear down """
del self.request
del self.operation
del self.file
del self.xmlStr
def test01PutRequest( self ):
""" test set request """
put = self.requestClient.putRequest( self.request )
self.assertEqual( put["OK"], True, "put failed" )
def test02GetRequest( self ):
""" test get request """
get = self.requestClient.getRequest( self.request.RequestName )
self.assertEqual( get["OK"], True, "get failed" )
def test03DeleteRequest( self ):
""" test delete request """
delete = self.requestClient.deleteRequest( "test" )
self.assertEqual( delete["OK"], True, "delete failed" )
def __setFileRemovalRequest( self, lfn, se = '', pfn = '' ):
""" Sets a removal request for a file including all replicas.
"""
remove = Operation()
remove.Type = "RemoveFile"
if se:
remove.TargetSE = se
rmFile = File()
rmFile.LFN = lfn
if pfn:
rmFile.PFN = pfn
remove.addFile( rmFile )
self.request.addOperation( remove )
return S_OK()
def generateForwardDISET( self ):
""" Commit the accumulated records and generate request eventually """
result = self.commit()
forwardDISETOp = None
if not result['OK']:
# Generate Request
if "FailedResults" in result:
for res in result['FailedResults']:
if 'rpcStub' in res:
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode( res['rpcStub'] )
return S_OK( forwardDISETOp )
class ReqClientTestCase( unittest.TestCase ):
"""
.. class:: ReqClientTestCase
"""
def setUp( self ):
""" test case set up """
gLogger.setLevel( 'NOTICE' )
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.file2 = File()
self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
self.file2.Checksum = "654321"
self.file2.ChecksumType = "ADLER32"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.operation.addFile( self.file )
self.operation.addFile( self.file2 )
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
self.request.OwnerGroup = "dirac_user"
self.request.JobID = 123
self.request.addOperation( self.operation )
# # JSON representation of a whole request
self.jsonStr = self.request.toJSON()['Value']
# # request client
self.requestClient = ReqClient()
self.stressRequests = 1000
self.bulkRequest = 1000
def tearDown( self ):
""" clean up """
del self.request
del self.operation
del self.file
del self.jsonStr
class ReqClientTestCase(unittest.TestCase):
"""
.. class:: ReqClientTestCase
"""
def setUp(self):
""" test case set up """
gLogger.setLevel('INFO')
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.file2 = File()
self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
self.file2.Checksum = "654321"
self.file2.ChecksumType = "ADLER32"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.operation.addFile(self.file)
self.operation.addFile(self.file2)
proxyInfo = getProxyInfo()['Value']
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = proxyInfo['identity']
self.request.OwnerGroup = proxyInfo['group']
self.request.JobID = 123
self.request.addOperation(self.operation)
# # JSON representation of a whole request
self.jsonStr = self.request.toJSON()['Value']
# # request client
self.requestClient = ReqClient()
self.stressRequests = 1000
self.bulkRequest = 1000
def tearDown(self):
""" clean up """
del self.request
del self.operation
del self.file
del self.jsonStr
def test03sql( self ):
""" sql insert or update """
operation = Operation()
operation.Type = "ReplicateAndRegister"
request = Request()
request.RequestName = "testRequest"
request.RequestID = 1
# # no parent request set
try:
operation.toSQL()
except Exception, error:
self.assertEqual( isinstance( error, AttributeError ), True, "wrong exc raised" )
self.assertEqual( str( error ), "RequestID not set", "wrong exc reason" )
def generateForwardDISET( self ):
""" Commit the accumulated records and generate request eventually """
result = self.commit()
forwardDISETOp = None
if not result['OK']:
# Generate Request
if result.has_key( 'rpcStub' ):
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode( result['rpcStub'] )
else:
return S_ERROR( 'Could not create ForwardDISET operation' )
return S_OK( forwardDISETOp )
def _sendToFailover( rpcStub ):
""" Create a ForwardDISET operation for failover
"""
try:
request = Request()
request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() )
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode( rpcStub )
request.addOperation( forwardDISETOp )
return ReqClient().putRequest( request )
# We catch all the exceptions, because it should never crash
except Exception as e: # pylint: disable=broad-except
return S_ERROR( ERMSUKN, "Exception sending accounting failover request: %s" % repr( e ) )
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split(';')
except AttributeError:
pass
failedTasks = []
# Do not remove sorted, we might pop elements in the loop
for taskID, task in taskDict.iteritems():
transID = task['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = task['TargetSE']
# If there are input files
if task.get('InputData'):
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def test01TableDesc(self):
""" table description """
tableDict = RequestDB.getTableMeta()
self.assertEqual("Request" in tableDict, True)
self.assertEqual("Operation" in tableDict, True)
self.assertEqual("File" in tableDict, True)
self.assertEqual(tableDict["Request"], Request.tableDesc())
self.assertEqual(tableDict["Operation"], Operation.tableDesc())
self.assertEqual(tableDict["File"], File.tableDesc())
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
""" Set removal request with the given credentials
"""
oRequest = Request()
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml'
oRequest.SourceComponent = 'JobCleaningAgent'
removeFile = Operation()
removeFile.Type = 'RemoveFile'
removedFile = File()
removedFile.LFN = lfn
removeFile.addFile( removedFile )
oRequest.addOperation( removeFile )
return ReqClient().putRequest( oRequest )
def generateForwardDISET( self ):
""" Generate and return failover requests for the operations in the internal cache
"""
fowardDISETOp = None
result = self.sendStoredStatusInfo()
if not result['OK']:
if 'rpcStub' in result:
rpcStub = result['rpcStub']
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode( rpcStub )
else:
return S_ERROR( 'Could not create ForwardDISET operation' )
return S_OK( fowardDISETOp )
def generateForwardDISET(self):
""" Generate and return failover requests for the operations in the internal cache
"""
fowardDISETOp = None
result = self.sendStoredStatusInfo()
if not result["OK"]:
if "rpcStub" in result:
rpcStub = result["rpcStub"]
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode(rpcStub)
else:
return S_ERROR("Could not create job parameters sub-request")
return S_OK(fowardDISETOp)
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if type( paramDict['InputData'] ) == type( [] ):
files = paramDict['InputData']
elif type( paramDict['InputData'] ) == type( '' ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
oRequest.OwnerDN = owner
oRequest.OwnerGroup = ownerGroup
isValid = gRequestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def setUp(self):
""" test case setup """
self.request = Request({"RequestName": "test1", "JobID": 1})
self.operation1 = Operation({"Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"})
self.file = File({"LFN": "/a/b/c", "ChecksumType": "ADLER32", "Checksum": "123456"})
self.request.addOperation(self.operation1)
self.operation1.addFile(self.file)
self.operation2 = Operation()
self.operation2.Type = "RemoveFile"
self.operation2.addFile(File({"LFN": "/c/d/e"}))
self.request.addOperation(self.operation2)
# ## set some defaults
gConfig.setOptionValue("DIRAC/Setup", "Test")
gConfig.setOptionValue("/DIRAC/Setups/Test/RequestManagement", "Test")
gConfig.setOptionValue("/Systems/RequestManagement/Test/Databases/ReqDB/Host", "localhost")
gConfig.setOptionValue("/Systems/RequestManagement/Test/Databases/ReqDB/DBName", "ReqDB")
gConfig.setOptionValue("/Systems/RequestManagement/Test/Databases/ReqDB/User", "Dirac")
self.i = 1000
def test07Authorization(self):
""" Test whether request sets on behalf of others are rejected"""
request = Request({"RequestName": "unauthorized"})
request.OwnerDN = 'NotMe'
request.OwnerDN = 'AnotherGroup'
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
res = self.requestClient.putRequest(request)
self.assertFalse(res['OK'], res)
def __setReplicaRemovalRequest( self, lfn, se ):
""" Sets a removal request for a replica.
:param str lfn: LFN
:param se:
"""
if type( se ) == str:
se = ",".join( [ se.strip() for se in se.split( "," ) if se.strip() ] )
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = se
replicaToRemove = File()
replicaToRemove.LFN = lfn
removeReplica.addFile( replicaToRemove )
self.request.addOperation( removeReplica )
return S_OK()
def generateForwardDISET(self):
""" Generate and return failover requests for the operations in the internal cache
"""
forwardDISETOp = None
result = self.sendStoredStatusInfo()
if not result['OK']:
gLogger.error("Error while sending the job status",
result['Message'])
if 'rpcStub' in result:
rpcStub = result['rpcStub']
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode(rpcStub)
else:
return S_ERROR('Could not create ForwardDISET operation')
return S_OK(forwardDISETOp)
def setBKRegistrationRequest(self, lfn, error='',
metaData={'Checksum': 'justSomething',
'ChecksumType': 'ADLER32',
'GUID': 'aGUID'}):
""" Set a BK registration request for changing the replica flag.
Uses the global request object (self.request).
"""
if error:
self.log.info('BK registration for %s failed with message: "%s" setting failover request' % (lfn, error))
else:
self.log.info('Setting BK registration request for %s' % (lfn))
regFile = Operation()
regFile.Type = 'RegisterFile'
regFile.Catalog = 'BookkeepingDB'
bkFile = File()
bkFile.LFN = lfn
# this should NOT be needed... but RMS complains!
bkFile.PFN = lfn
bkFile.GUID = metaData['GUID']
bkFile.Checksum = metaData['Checksum']
bkFile.ChecksumType = metaData['ChecksumType']
regFile.addFile(bkFile)
res = self.request.addOperation(regFile)
if not res['OK']:
raise RuntimeError(res['Message'])
def __submitRMSOp(self, target_se, lfns_chunk_dict, whichRMSOp='ReplicateAndRegister' ):
""" target_se : SE name to which to replicate
lfns_chunk_dict : LFNS dict with 100 lfns as key andeEach lfn has 'Size', 'Checksum'
whichRMSOp: Choose from RMP operation - ReplicateAndRegister, ReplicateAndRemove, PutAndRegister
"""
## Setup request
request = Request()
request.RequestName = "DDM_"+ str(target_se) + datetime.datetime.now().strftime("_%Y%m%d_%H%M%S")
myOp = Operation()
myOp.Type = whichRMSOp
myOp.TargetSE = target_se
## Add LFNS to operations
for lfn in lfns_chunk_dict.keys():
opFile = File()
opFile.LFN = lfn
opFile.Size = lfns_chunk_dict[lfn]['Size']
if "Checksum" in lfns_chunk_dict[lfn]:
opFile.Checksum = lfns_chunk_dict[lfn]['Checksum']
opFile.ChecksumType = 'ADLER32'
## Add file to operation
myOp.addFile( opFile )
request.addOperation( myOp )
reqClient = ReqClient()
putRequest = reqClient.putRequest( request )
if not putRequest["OK"]:
gLogger.error( "Unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) )
return S_ERROR("Problem submitting to RMS.")
def archiveRequestAndOp(listOfLFNs):
"""Return a tuple of the request and operation."""
req = Request()
req.RequestName = "MyRequest"
op = Operation()
switches = {}
archiveLFN = "/vo/tars/myTar.tar"
op.Arguments = DEncode.encode({
"SourceSE":
switches.get("SourceSE", "SOURCE-SE"),
"TarballSE":
switches.get("TarballSE", "TARBALL-SE"),
"RegisterDescendent":
False,
"ArchiveLFN":
archiveLFN,
})
op.Type = "ArchiveFiles"
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = "01130a%0d" % index
oFile.ChecksumType = "adler32"
op.addFile(oFile)
req.addOperation(op)
return req, op
def archiveRequestAndOp(listOfLFNs):
"""Return a tuple of the request and operation."""
req = Request()
req.RequestName = 'MyRequest'
op = Operation()
switches = {}
archiveLFN = '/vo/tars/myTar.tar'
op.Arguments = DEncode.encode({
'SourceSE':
switches.get('SourceSE', 'SOURCE-SE'),
'TarballSE':
switches.get('TarballSE', 'TARBALL-SE'),
'RegisterDescendent':
False,
'ArchiveLFN':
archiveLFN
})
op.Type = 'ArchiveFiles'
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = '01130a%0d' % index
oFile.ChecksumType = 'adler32'
op.addFile(oFile)
req.addOperation(op)
return req, op
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
if self.getCSOption("DelayedExternalDeletion", True):
gLogger.info("Setting deletion request")
try:
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % (
SEName, SEPFN, time.time())
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile(fileToRemove)
request.addOperation(physicalRemoval)
return ReqClient().putRequest(request)
except Exception as e:
gLogger.exception("Exception while setting deletion request")
return S_ERROR("Cannot set deletion request: %s" % str(e))
else:
gLogger.info("Deleting external Sandbox")
try:
return StorageElement(SEName).removeFile(SEPFN)
except Exception as e:
gLogger.exception(
"RM raised an exception while trying to delete a remote sandbox"
)
return S_ERROR(
"RM raised an exception while trying to delete a remote sandbox"
)
def __init__(self, fromDict=None):
"""c'tor
:param self: self reference
:param fromDict: if false, new request. Can be json string that represents the object, or the dictionary directly
"""
self.__waiting = None
now = datetime.datetime.utcnow().replace(microsecond=0)
self._CreationTime = now
self._SubmitTime = now
self._LastUpdate = now
# the time before which the request should not be executed
# If None, no delay
self._NotBefore = now
self._Status = "Done"
self.JobID = 0
self.Error = None
self.DIRACSetup = None
self.OwnerDN = None
self.RequestName = None
self.OwnerGroup = None
self._SourceComponent = None
self.dmsHelper = DMSHelpers()
proxyInfo = getProxyInfo()
if proxyInfo["OK"]:
proxyInfo = proxyInfo["Value"]
if proxyInfo["validGroup"] and proxyInfo["validDN"]:
self.OwnerDN = proxyInfo["identity"]
self.OwnerGroup = proxyInfo["group"]
self.__operations__ = []
if isinstance(fromDict, str):
fromDict = json.loads(fromDict)
elif not isinstance(fromDict, dict):
fromDict = {}
if "Operations" in fromDict:
for opDict in fromDict.get("Operations", []):
self += Operation(opDict)
del fromDict["Operations"]
for key, value in fromDict.items():
if value:
setattr(self, key, value)
self._notify()
def __setRemovalRequest(self, lfn, ownerDN, ownerGroup):
"""Set removal request with the given credentials"""
oRequest = Request()
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
oRequest.RequestName = os.path.basename(
lfn).strip() + "_removal_request.xml"
oRequest.SourceComponent = "JobCleaningAgent"
removeFile = Operation()
removeFile.Type = "RemoveFile"
removedFile = File()
removedFile.LFN = lfn
removeFile.addFile(removedFile)
oRequest.addOperation(removeFile)
# put the request with the owner certificate to make sure it's still a valid DN
return ReqClient(useCertificates=True,
delegatedDN=ownerDN,
delegatedGroup=ownerGroup).putRequest(oRequest)
def test04StateMachine(self):
""" state machine """
op = Operation()
self.assertEqual(op.Status, "Queued", "1. wrong status %s" % op.Status)
op.addFile(File({"Status": "Waiting"}))
self.assertEqual(op.Status, "Queued", "2. wrong status %s" % op.Status)
op.addFile(File({"Status": "Scheduled"}))
self.assertEqual(op.Status, "Scheduled",
"3. wrong status %s" % op.Status)
op.addFile(File({"Status": "Done"}))
self.assertEqual(op.Status, "Scheduled",
"4. wrong status %s" % op.Status)
op.addFile(File({"Status": "Failed"}))
self.assertEqual(op.Status, "Scheduled",
"5. wrong status %s" % op.Status)
op[3].Status = "Scheduled"
self.assertEqual(op.Status, "Scheduled",
"6. wrong status %s" % op.Status)
op[0].Status = "Scheduled"
self.assertEqual(op.Status, "Scheduled",
"7. wrong status %s" % op.Status)
op[0].Status = "Waiting"
self.assertEqual(op.Status, "Scheduled",
"8. wrong status %s" % op.Status)
for f in op:
f.Status = "Done"
self.assertEqual(op.Status, "Done", "9. wrong status %s" % op.Status)
for f in op:
f.Status = "Failed"
self.assertEqual(op.Status, "Failed", "9. wrong status %s" % op.Status)
def setUp( self ):
""" test setup
:param self: self reference
"""
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
self.request.OwnerGroup = "dirac_user"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.request.addOperation( self.operation )
self.operation.addFile( self.file )
# # xml representation of a whole request
self.xmlStr = self.request.toXML( True )["Value"]
# # request client
self.requestClient = RequestClient()
def test_List():
"""getitem, setitem, delitem and dirty"""
op = Operation()
files = []
for _ in range(5):
f = File()
files.append(f)
op += f
for i in range(len(op)):
assert op[i] == files[i], "__getitem__ failed"
for i in range(len(op)):
op[i] = File({"LFN": "/%s" % i})
assert op[i].LFN == "/%s" % i, "__setitem__ failed"
del op[0]
assert len(op) == 4, "__delitem__ failed"
# opID set
op.OperationID = 1
del op[0]
def test05List(self):
""" getitem, setitem, delitem and dirty """
op = Operation()
files = []
for i in range(5):
f = File()
files.append(f)
op += f
for i in range(len(op)):
self.assertEqual(op[i], files[i], "__getitem__ failed")
for i in range(len(op)):
op[i] = File({"LFN": "/%s" % i})
self.assertEqual(op[i].LFN, "/%s" % i, "__setitem__ failed")
del op[0]
self.assertEqual(len(op), 4, "__delitem__ failed")
# # opID set
op.OperationID = 1
del op[0]
def _cleanUp(self, final):
""" Clean up uploaded data for the LFNs in the list
"""
lfnList = []
for _fileName, metadata in final.items():
lfnList.append(metadata['lfn'])
self.log.verbose("Cleaning up the request, for LFNs: %s" %
', '.join(lfnList))
newRequest = Request()
for op in self.request:
add = True
if op.Type in [
'PutAndRegister', 'ReplicateAndRegister', 'RegisterFile',
'RegisterReplica', 'RemoveReplica'
]:
for files in op:
if files.LFN in lfnList:
add = False
if add:
newRequest.addOperation(op)
self.request = newRequest
self.log.verbose(
"And adding RemoveFile operation for LFNs: %s, just in case" %
', '.join(lfnList))
removeFiles = Operation()
removeFiles.Type = 'RemoveFile'
for lfn in lfnList:
removedFile = File()
removedFile.LFN = lfn
removeFiles.addFile(removedFile)
self.request.addOperation(removeFiles)
def setUp(self):
""" test set up """
self.hiArgs = (("RequestManagement/RequestManager", {
"keepAliveLapse": 10,
"timeout": 5
}), "foo", (12345, {
"Hi": "There!"
}))
self.req = Request({"RequestName": "testRequest"})
self.op = Operation({
"Type": "ForwardDISET",
"Arguments": DEncode.encode(self.hiArgs)
})
self.req += self.op
def setUp(self):
""" test case set up """
gLogger.setLevel('INFO')
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.file2 = File()
self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
self.file2.Checksum = "654321"
self.file2.ChecksumType = "ADLER32"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.operation.addFile(self.file)
self.operation.addFile(self.file2)
proxyInfo = getProxyInfo()['Value']
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = proxyInfo['identity']
self.request.OwnerGroup = proxyInfo['group']
self.request.JobID = 123
self.request.addOperation(self.operation)
# # JSON representation of a whole request
self.jsonStr = self.request.toJSON()['Value']
# # request client
self.requestClient = ReqClient()
self.stressRequests = 1000
self.bulkRequest = 1000
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def test_valid_properties():
operation = Operation()
operation.Arguments = "foobar"
assert operation.Arguments == b"foobar", "wrong Arguments"
operation.SourceSE = "CERN-RAW"
assert operation.SourceSE == "CERN-RAW", "wrong SourceSE"
operation.TargetSE = "CERN-RAW"
assert operation.TargetSE == "CERN-RAW", "wrong TargetSE"
operation.Catalog = ""
assert operation.Catalog == "", "wrong Catalog"
operation.Catalog = "BookkeepingDB"
assert operation.Catalog == "BookkeepingDB", "wrong Catalog"
operation.Error = "error"
assert operation.Error == "error", "wrong Error"
toJSON = operation.toJSON()
assert toJSON["OK"]
def _setFileRemovalRequest(self, lfn, se="", pfn=""):
"""Sets a removal request for a file including all replicas."""
remove = Operation()
remove.Type = "RemoveFile"
if se:
remove.TargetSE = se
rmFile = File()
rmFile.LFN = lfn
if pfn:
rmFile.PFN = pfn
remove.addFile(rmFile)
self.request.addOperation(remove)
return S_OK()
def test01StressBulk(self):
""" stress test bulk """
db = RequestDB()
reqIDs = []
for i in xrange(self.stressRequests):
request = Request({"RequestName": "test-%d" % i})
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
put = db.putRequest(request)
self.assertEqual(put["OK"], True)
reqIDs.append(put['Value'])
loops = self.stressRequests // self.bulkRequest + \
(1 if (self.stressRequests % self.bulkRequest) else 0)
totalSuccessful = 0
time.sleep(1)
startTime = time.time()
for i in xrange(loops):
get = db.getBulkRequests(self.bulkRequest, True)
if "Message" in get:
print(get["Message"])
self.assertEqual(get["OK"], True, "get failed")
totalSuccessful += len(get["Value"])
endTime = time.time()
print("getRequests duration %s " % (endTime - startTime))
self.assertEqual(
totalSuccessful, self.stressRequests,
"Did not retrieve all the requests: %s instead of %s" %
(totalSuccessful, self.stressRequests))
for reqID in reqIDs:
delete = db.deleteRequest(reqID)
self.assertEqual(
delete["OK"], True,
delete['Message'] if 'Message' in delete else 'OK')
def __init__( self, fromDict = None ):
"""c'tor
:param self: self reference
"""
Record.__init__( self )
self.__waiting = None
now = datetime.datetime.utcnow().replace( microsecond = 0 )
self.__data__["CreationTime"] = now
self.__data__["SubmitTime"] = now
self.__data__["LastUpdate"] = now
self.__data__["Status"] = "Done"
self.__data__["JobID"] = 0
self.__data__["RequestID"] = 0
proxyInfo = getProxyInfo()
if proxyInfo["OK"]:
proxyInfo = proxyInfo["Value"]
if proxyInfo["validGroup"] and proxyInfo["validDN"]:
self.OwnerDN = proxyInfo["identity"]
self.OwnerGroup = proxyInfo["group"]
self.__dirty = []
self.__operations__ = TypedList( allowedTypes = Operation )
fromDict = fromDict if fromDict else {}
self.__dirty = fromDict.get( "__dirty", [] )
if "__dirty" in fromDict:
del fromDict["__dirty"]
for opDict in fromDict.get( "Operations", [] ):
self +=Operation( opDict )
if "Operations" in fromDict:
del fromDict["Operations"]
for key, value in fromDict.items():
if key not in self.__data__:
raise AttributeError( "Unknown Request attribute '%s'" % key )
if value:
setattr( self, key, value )
self._notify()
def setUp(self):
"""test case set up"""
self.handlersDict = {"ForwardDISET": "DIRAC/RequestManagementSystem/private/ForwardDISET"}
self.req = Request()
self.req.RequestName = "foobarbaz"
self.req.OwnerGroup = "lhcb_user"
self.req.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=chaen/CN=705305/CN=Christophe Haen"
self.op = Operation({"Type": "ForwardDISET", "Arguments": "tts10:helloWorldee"})
self.req.addOperation(self.op)
self.task = None
self.mockRC = MagicMock()
self.mockObjectOps = MagicMock()
self.mockObjectOps.getSections.return_value = {"OK": True, "Value": ["DataProcessing", "DataManager"]}
self.mockObjectOps.getOptionsDict.return_value = {
"OK": True,
"Value": {"Group": "lhcb_user", "User": "******"},
}
self.mockOps = MagicMock()
self.mockOps.return_value = self.mockObjectOps
def test_db(self):
""" table description """
tableDict = RequestDB.getTableMeta()
self.assertEqual("Request" in tableDict, True)
self.assertEqual("Operation" in tableDict, True)
self.assertEqual("File" in tableDict, True)
self.assertEqual(tableDict["Request"], Request.tableDesc())
self.assertEqual(tableDict["Operation"], Operation.tableDesc())
self.assertEqual(tableDict["File"], File.tableDesc())
# # empty DB at that stage
ret = RequestDB().getDBSummary()
self.assertEqual(ret, {
'OK': True,
'Value': {
'Operation': {},
'Request': {},
'File': {}
}
})
def checkRequestAndOp(listOfLFNs):
req = Request()
req.RequestName = "MyRequest"
op = Operation()
op.Type = "CheckMigration"
op.TargetSE = "Foo-SE"
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = "01130a%0d" % index
oFile.ChecksumType = "adler32"
op.addFile(oFile)
req.addOperation(op)
return req, op
def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info('Setting registration request for %s at %s.' %
(lfn, targetSE))
if not type(catalog) == type([]):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get("Checksum", "")
regFile.ChecksumType = fileDict.get("ChecksumType",
self.defaultChecksumType)
regFile.Size = fileDict.get("Size", 0)
regFile.GUID = fileDict.get("GUID", "")
se = StorageElement(targetSE)
pfn = se.getPfnForLfn(lfn)
if not pfn["OK"] or lfn not in pfn["Value"]['Successful']:
self.log.error(
"unable to get PFN for LFN: %s" %
pfn.get('Message',
pfn.get('Value', {}).get('Failed', {}).get(lfn)))
return pfn
regFile.PFN = pfn["Value"]['Successful'][lfn]
register.addFile(regFile)
self.request.addOperation(register)
return S_OK()
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
"""deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = "ReplicateAndRegister"
if transBody:
try:
_requestType, requestOperation = transBody.split(";")
except AttributeError:
pass
failedTasks = []
# Do not remove sorted, we might pop elements in the loop
for taskID, task in taskDict.items():
transID = task["TransformationID"]
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = task["TargetSE"]
# If there are input files
if task.get("InputData"):
if isinstance(task["InputData"], list):
files = task["InputData"]
elif isinstance(task["InputData"], six.string_types):
files = task["InputData"].split(";")
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
result = self._assignRequestToTask(oRequest, taskDict, transID,
taskID, ownerDN, ownerGroup)
if not result["OK"]:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def test02Authorization(self):
""" Test whether request sets on behalf of others are rejected, unless done with Delegation properties
This test is kind of stupid though, since we do the same thing than the server... not a real test !
"""
request = Request({"RequestName": "unauthorized"})
request.OwnerDN = 'NotMe'
request.OwnerDN = 'AnotherGroup'
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
res = self.requestClient.putRequest(request)
credProperties = getProxyInfo()['Value']['groupProperties']
# If the proxy with which we test has delegation, it should work
if FULL_DELEGATION in credProperties or LIMITED_DELEGATION in credProperties:
self.assertTrue(res['OK'], res)
self.requestClient.deleteRequest(res['Value'])
# otherwise no
else:
self.assertFalse(res['OK'], res)
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
if self.getCSOption("DelayedExternalDeletion", True):
gLogger.info("Setting deletion request")
try:
# We need the hostDN used in order to pass these credentials to the
# SandboxStoreDB..
hostCertLocation, _ = Locations.getHostCertificateAndKeyLocation(
)
hostCert = X509Certificate.X509Certificate()
hostCert.loadFromFile(hostCertLocation)
hostDN = hostCert.getSubjectDN().get("Value")
# use the host authentication to fetch the data
result = self.sandboxDB.getSandboxOwner(
SEName, SEPFN, hostDN, "hosts")
if not result["OK"]:
return result
_owner, ownerDN, ownerGroup = result["Value"]
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % (
SEName, SEPFN, time.time())
request.OwnerDN = ownerDN
request.OwnerGroup = ownerGroup
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile(fileToRemove)
request.addOperation(physicalRemoval)
return ReqClient().putRequest(request)
except Exception as e:
gLogger.exception("Exception while setting deletion request")
return S_ERROR(f"Cannot set deletion request: {e}")
else:
gLogger.info("Deleting external Sandbox")
try:
return StorageElement(SEName).removeFile(SEPFN)
except Exception:
gLogger.exception(
"RM raised an exception while trying to delete a remote sandbox"
)
return S_ERROR(
"RM raised an exception while trying to delete a remote sandbox"
)
def test02Scheduled(self):
""" scheduled request r/w """
db = RequestDB()
req = Request({"RequestName": "FTSTest"})
op = Operation({
"Type": "ReplicateAndRegister",
"TargetSE": "CERN-USER"
})
op += File({
"LFN": "/a/b/c",
"Status": "Scheduled",
"Checksum": "123456",
"ChecksumType": "ADLER32"
})
req += op
put = db.putRequest(req)
self.assertEqual(put["OK"], True,
put['Message'] if 'Message' in put else 'OK')
reqID = put['Value']
peek = db.peekRequest(reqID)
self.assertEqual(peek["OK"], True,
peek['Message'] if 'Message' in peek else 'OK')
peek = peek["Value"]
for op in peek:
opId = op.OperationID
getFTS = db.getScheduledRequest(opId)
self.assertEqual(getFTS["OK"], True, "getScheduled failed")
self.assertEqual(getFTS["Value"].RequestName, "FTSTest",
"wrong request selected")
delete = db.deleteRequest(reqID)
self.assertEqual(delete["OK"], True,
delete['Message'] if 'Message' in delete else 'OK')
def test04Stress(self):
""" stress test """
db = RequestDB()
for i in range(self.i):
request = Request({"RequestName": "test-%d" % i})
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
put = db.putRequest(request)
self.assertEqual(put["OK"], True, "put failed")
for i in range(self.i):
get = db.getRequest("test-%s" % i, False)
if "Message" in get:
print get["Message"]
self.assertEqual(get["OK"], True, "get failed")
for i in range(self.i):
delete = db.deleteRequest("test-%s" % i)
self.assertEqual(delete["OK"], True, "delete failed")
