def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE):
""" Set a request to upload job log files from the output sandbox
Changed to be similar to LHCb createLogUploadRequest
using LHCb LogUpload Request and Removal Request
"""
self.log.info('Setting log upload request for %s at %s' %(targetSE, logFileLFN))
request = self._getRequestContainer()
logUpload = Operation()
logUpload.Type = "LogUpload"
logUpload.TargetSE = targetSE
upFile = File()
upFile.LFN = logFileLFN
logUpload.addFile( upFile )
logRemoval = Operation()
logRemoval.Type = 'RemoveFile'
logRemoval.TargetSE = uploadedSE
logRemoval.addFile( upFile )
request.addOperation ( logUpload )
request.addOperation ( logRemoval )
self.workflow_commons['Request'] = request
return S_OK()
def _createLogUploadRequest(self, targetSE, logFileLFN, uploadedSE):
""" Set a request to upload job log files from the output sandbox
Changed to be similar to LHCb createLogUploadRequest
using LHCb LogUpload Request and Removal Request
"""
self.log.info('Setting log upload request for %s at %s' %
(targetSE, logFileLFN))
request = self._getRequestContainer()
logUpload = Operation()
logUpload.Type = "LogUpload"
logUpload.TargetSE = targetSE
upFile = File()
upFile.LFN = logFileLFN
logUpload.addFile(upFile)
logRemoval = Operation()
logRemoval.Type = 'RemoveFile'
logRemoval.TargetSE = uploadedSE
logRemoval.addFile(upFile)
request.addOperation(logUpload)
request.addOperation(logRemoval)
self.workflow_commons['Request'] = request
return S_OK()
def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ):
if self.getCSOption( "DelayedExternalDeletion", True ):
gLogger.info( "Setting deletion request" )
try:
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() )
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile( fileToRemove )
request.addOperation( physicalRemoval )
return ReqClient().putRequest( request )
except Exception as e:
gLogger.exception( "Exception while setting deletion request" )
return S_ERROR( "Cannot set deletion request: %s" % str( e ) )
else:
gLogger.info( "Deleting external Sandbox" )
try:
return StorageElement( SEName ).removeFile( SEPFN )
except Exception as e:
gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" )
return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def myRequest():
"""Create a request and put it to the db"""
request = Request()
request.RequestName = 'myAwesomeRemovalRequest.xml'
request.JobID = 0
request.SourceComponent = "myScript"
remove = Operation()
remove.Type = "RemoveFile"
lfn = "/ilc/user/s/sailer/test.txt"
rmFile = File()
rmFile.LFN = lfn
remove.addFile( rmFile )
request.addOperation( remove )
isValid = RequestValidator().validate( request )
if not isValid['OK']:
raise RuntimeError( "Failover request is not valid: %s" % isValid['Message'] )
else:
print("It is a GOGOGO")
requestClient = ReqClient()
result = requestClient.putRequest( request )
print(result)
def setUp(self):
""" test case set up """
gLogger.setLevel('NOTICE')
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.file2 = File()
self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
self.file2.Checksum = "654321"
self.file2.ChecksumType = "ADLER32"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.operation.addFile(self.file)
self.operation.addFile(self.file2)
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
self.request.OwnerGroup = "dirac_user"
self.request.JobID = 123
self.request.addOperation(self.operation)
# # JSON representation of a whole request
self.jsonStr = self.request.toJSON()['Value']
# # request client
self.requestClient = ReqClient()
self.stressRequests = 1000
self.bulkRequest = 1000
def setUp(self):
""" test case setup """
self.request = Request({"RequestName": "test1", "JobID": 1})
self.operation1 = Operation({
"Type": "ReplicateAndRegister",
"TargetSE": "CERN-USER"
})
self.file = File({
"LFN": "/a/b/c",
"ChecksumType": "ADLER32",
"Checksum": "123456"
})
self.request.addOperation(self.operation1)
self.operation1.addFile(self.file)
self.operation2 = Operation()
self.operation2.Type = "RemoveFile"
self.operation2.addFile(File({"LFN": "/c/d/e"}))
self.request.addOperation(self.operation2)
# ## set some defaults
gConfig.setOptionValue('DIRAC/Setup', 'Test')
gConfig.setOptionValue('/DIRAC/Setups/Test/RequestManagement', 'Test')
gConfig.setOptionValue(
'/Systems/RequestManagement/Test/Databases/ReqDB/Host',
'localhost')
gConfig.setOptionValue(
'/Systems/RequestManagement/Test/Databases/ReqDB/DBName', 'ReqDB')
gConfig.setOptionValue(
'/Systems/RequestManagement/Test/Databases/ReqDB/User', 'Dirac')
self.i = 1000
def setUp(self):
""" test case set up """
gLogger.setLevel('INFO')
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.file2 = File()
self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
self.file2.Checksum = "654321"
self.file2.ChecksumType = "ADLER32"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.operation.addFile(self.file)
self.operation.addFile(self.file2)
proxyInfo = getProxyInfo()['Value']
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = proxyInfo['identity']
self.request.OwnerGroup = proxyInfo['group']
self.request.JobID = 123
self.request.addOperation(self.operation)
# # JSON representation of a whole request
self.jsonStr = self.request.toJSON()['Value']
# # request client
self.requestClient = ReqClient()
def test_StateMachine():
"""state machine"""
op = Operation()
assert op.Status == "Queued", "1. wrong status %s" % op.Status
op.addFile(File({"Status": "Waiting"}))
assert op.Status == "Queued", "2. wrong status %s" % op.Status
op.addFile(File({"Status": "Scheduled"}))
assert op.Status == "Scheduled", "3. wrong status %s" % op.Status
op.addFile(File({"Status": "Done"}))
assert op.Status == "Scheduled", "4. wrong status %s" % op.Status
op.addFile(File({"Status": "Failed"}))
assert op.Status == "Scheduled", "5. wrong status %s" % op.Status
op[3].Status = "Scheduled"
assert op.Status == "Scheduled", "6. wrong status %s" % op.Status
op[0].Status = "Scheduled"
assert op.Status == "Scheduled", "7. wrong status %s" % op.Status
op[0].Status = "Waiting"
assert op.Status == "Scheduled", "8. wrong status %s" % op.Status
for f in op:
f.Status = "Done"
assert op.Status == "Done", "9. wrong status %s" % op.Status
for f in op:
f.Status = "Failed"
assert op.Status == "Failed", "9. wrong status %s" % op.Status
def myRequest():
"""Create a request and put it to the db"""
request = Request()
request.RequestName = 'myAwesomeRemovalRequest.xml'
request.JobID = 0
request.SourceComponent = "myScript"
remove = Operation()
remove.Type = "RemoveFile"
lfn = "/ilc/user/s/sailer/test.txt"
rmFile = File()
rmFile.LFN = lfn
remove.addFile(rmFile)
request.addOperation(remove)
isValid = RequestValidator().validate(request)
if not isValid['OK']:
raise RuntimeError("Failover request is not valid: %s" %
isValid['Message'])
else:
print "It is a GOGOGO"
requestClient = ReqClient()
result = requestClient.putRequest(request)
print result
def test04StateMachine(self):
""" state machine """
op = Operation()
self.assertEqual(op.Status, "Queued", "1. wrong status %s" % op.Status)
op.addFile(File({"Status": "Waiting"}))
self.assertEqual(op.Status, "Queued", "2. wrong status %s" % op.Status)
op.addFile(File({"Status": "Scheduled"}))
self.assertEqual(op.Status, "Queued", "3. wrong status %s" % op.Status)
op.addFile(File({"Status": "Done"}))
self.assertEqual(op.Status, "Queued", "4. wrong status %s" % op.Status)
op.addFile(File({"Status": "Failed"}))
self.assertEqual(op.Status, "Failed", "5. wrong status %s" % op.Status)
op[3].Status = "Scheduled"
self.assertEqual(op.Status, "Queued", "6. wrong status %s" % op.Status)
op[0].Status = "Scheduled"
self.assertEqual(op.Status, "Scheduled",
"7. wrong status %s" % op.Status)
op[0].Status = "Waiting"
self.assertEqual(op.Status, "Queued", "8. wrong status %s" % op.Status)
for f in op:
f.Status = "Done"
self.assertEqual(op.Status, "Done", "9. wrong status %s" % op.Status)
def __insertRegisterOperation( self, request, operation, toRegister ):
""" add RegisterReplica operation
:param Request request: request instance
:param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
:param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
"""
log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )
byTarget = {}
for ftsFile in toRegister:
if ftsFile.TargetSE not in byTarget:
byTarget.setdefault( ftsFile.TargetSE, [] )
byTarget[ftsFile.TargetSE].append( ftsFile )
log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )
for target, ftsFileList in byTarget.iteritems():
log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
len( ftsFileList ) ) )
registerOperation = Operation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
targetSE = StorageElement( target )
for ftsFile in ftsFileList:
opFile = File()
opFile.LFN = ftsFile.LFN
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
if not pfn["OK"]:
continue
opFile.PFN = pfn["Value"]
registerOperation.addFile( opFile )
request.insertBefore( registerOperation, operation )
return S_OK()
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
if self.getCSOption("DelayedExternalDeletion", True):
gLogger.info("Setting deletion request")
try:
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % (
SEName, SEPFN, time.time())
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile(fileToRemove)
request.addOperation(physicalRemoval)
return ReqClient().putRequest(request)
except Exception as e:
gLogger.exception("Exception while setting deletion request")
return S_ERROR("Cannot set deletion request: %s" % str(e))
else:
gLogger.info("Deleting external Sandbox")
try:
return StorageElement(SEName).removeFile(SEPFN)
except Exception as e:
gLogger.exception(
"RM raised an exception while trying to delete a remote sandbox"
)
return S_ERROR(
"RM raised an exception while trying to delete a remote sandbox"
)
def __insertRegisterOperation( self, request, operation, toRegister ):
""" add RegisterReplica operation
:param Request request: request instance
:param Operation transferOp: 'ReplicateAndRegister' operation for this FTSJob
:param list toRegister: [ FTSDB.FTSFile, ... ] - files that failed to register
"""
log = self.log.getSubLogger( "req_%s/%s/registerFiles" % ( request.RequestID, request.RequestName ) )
byTarget = {}
for ftsFile in toRegister:
if ftsFile.TargetSE not in byTarget:
byTarget.setdefault( ftsFile.TargetSE, [] )
byTarget[ftsFile.TargetSE].append( ftsFile )
log.info( "will create %s 'RegisterReplica' operations" % len( byTarget ) )
for target, ftsFileList in byTarget.iteritems():
log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % ( target,
len( ftsFileList ) ) )
registerOperation = Operation()
registerOperation.Type = "RegisterReplica"
registerOperation.Status = "Waiting"
registerOperation.TargetSE = target
targetSE = StorageElement( target )
for ftsFile in ftsFileList:
opFile = File()
opFile.LFN = ftsFile.LFN
pfn = returnSingleResult( targetSE.getURL( ftsFile.LFN, protocol = self.registrationProtocols ) )
if not pfn["OK"]:
continue
opFile.PFN = pfn["Value"]
registerOperation.addFile( opFile )
request.insertBefore( registerOperation, operation )
return S_OK()
def test06Dirty( self ):
""" dirty records """
db = RequestDB()
r = Request()
r.RequestName = "dirty"
op1 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op1 += File( {"LFN": "/a/b/c/1", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
op2 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op2 += File( {"LFN": "/a/b/c/2", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
op3 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op3 += File( {"LFN": "/a/b/c/3", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
r += op1
r += op2
r += op3
put = db.putRequest( r )
self.assertEqual( put["OK"], True, "1. putRequest failed: %s" % put.get( "Message", "" ) )
reqID = put['Value']
r = db.getRequest( reqID )
self.assertEqual( r["OK"], True, "1. getRequest failed: %s" % r.get( "Message", "" ) )
r = r["Value"]
del r[0]
self.assertEqual( len( r ), 2, "1. len wrong" )
put = db.putRequest( r )
self.assertEqual( put["OK"], True, "2. putRequest failed: %s" % put.get( "Message", "" ) )
reqID = put['Value']
r = db.getRequest( reqID )
self.assertEqual( r["OK"], True, "2. getRequest failed: %s" % r.get( "Message", "" ) )
r = r["Value"]
self.assertEqual( len( r ), 2, "2. len wrong" )
op4 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op4 += File( {"LFN": "/a/b/c/4", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
r[0] = op4
put = db.putRequest( r )
self.assertEqual( put["OK"], True, "3. putRequest failed: %s" % put.get( "Message", "" ) )
reqID = put['Value']
r = db.getRequest( reqID )
self.assertEqual( r["OK"], True, "3. getRequest failed: %s" % r.get( "Message", "" ) )
r = r["Value"]
self.assertEqual( len( r ), 2, "3. len wrong" )
delete = db.deleteRequest( reqID )
self.assertEqual( delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK' )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def __setFileRemovalRequest(self, lfn):
""" Sets a removal request for a file including all replicas.
"""
self.log.info('Setting file removal request for %s' % lfn)
removeFile = Operation()
removeFile.Type = 'RemoveFile'
rmFile = File()
rmFile.LFN = lfn
removeFile.addFile(rmFile)
self.request.addOperation(removeFile)
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has multiple operations
:param transJson: list of lists of string and dictionaries, e.g.:
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
failedTasks = []
for taskID, task in taskDict.items():
transID = task['TransformationID']
if not task.get('InputData'):
self._logError("Error creating request for task",
"%s, No input data" % taskID,
transID=transID)
taskDict.pop(taskID)
continue
files = []
oRequest = Request()
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
# create the operations from the json structure
for operationTuple in transJson:
op = Operation()
op.Type = operationTuple[0]
for parameter, value in operationTuple[1].iteritems():
setattr(op, parameter, value)
for lfn in files:
opFile = File()
opFile.LFN = lfn
op.addFile(opFile)
oRequest.addOperation(op)
result = self._assignRequestToTask(oRequest, taskDict, transID,
taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def test01ctors(self):
""" File construction and (de)serialisation """
# # empty default ctor
theFile = File()
self.assertEqual(isinstance(theFile, File), True)
# # fromDict
try:
theFile = File(self.fromDict)
except AttributeError, error:
print "AttributeError: %s" % str(error)
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has multiple operations
:param transJson: list of lists of string and dictionaries, e.g.:
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
failedTasks = []
for taskID, task in taskDict.items():
transID = task['TransformationID']
if not task.get('InputData'):
self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID)
taskDict.pop(taskID)
continue
files = []
oRequest = Request()
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
# create the operations from the json structure
for operationTuple in transJson:
op = Operation()
op.Type = operationTuple[0]
for parameter, value in operationTuple[1].iteritems():
setattr(op, parameter, value)
for lfn in files:
opFile = File()
opFile.LFN = lfn
op.addFile(opFile)
oRequest.addOperation(op)
result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def addRemovalRequests(self, lfnList):
"""Create removalRequests for lfns in lfnList and add it to the common request"""
request = self._getRequestContainer()
remove = Operation()
remove.Type = "RemoveFile"
for lfn in lfnList:
rmFile = File()
rmFile.LFN = lfn
remove.addFile( rmFile )
request.addOperation( remove )
self.workflow_commons['Request'] = request
def _setFileRemovalRequest(self, lfn, se="", pfn=""):
"""Sets a removal request for a file including all replicas."""
remove = Operation()
remove.Type = "RemoveFile"
if se:
remove.TargetSE = se
rmFile = File()
rmFile.LFN = lfn
if pfn:
rmFile.PFN = pfn
remove.addFile(rmFile)
self.request.addOperation(remove)
return S_OK()
def addRemovalRequests(self, lfnList):
"""Create removalRequests for lfns in lfnList and add it to the common request"""
request = self._getRequestContainer()
remove = Operation()
remove.Type = "RemoveFile"
for lfn in lfnList:
rmFile = File()
rmFile.LFN = lfn
remove.addFile( rmFile )
request.addOperation( remove )
self.workflow_commons['Request'] = request
def getRegisterOperation(self,
opFile,
targetSE,
type='RegisterFile',
catalog=None):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = StorageElement(targetSE).getPfnForLfn(
opFile.LFN).get('Value', {}).get('Successful', {}).get(opFile.LFN)
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
return registerOperation
def getRegisterOperation(self, opFile, targetSE, type="RegisterFile", catalog=None):
"""add RegisterReplica operation for file
:param ~DIRAC.RequestManagementSystem.Client.File.File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = (
StorageElement(targetSE)
.getURL(opFile.LFN, protocol=self.registrationProtocols)
.get("Value", {})
.get("Successful", {})
.get(opFile.LFN)
)
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile(registerFile)
return registerOperation
def __setFileRemovalRequest( self, lfn, se = '', pfn = '' ):
""" Sets a removal request for a file including all replicas.
"""
remove = Operation()
remove.Type = "RemoveFile"
if se:
remove.TargetSE = se
rmFile = File()
rmFile.LFN = lfn
if pfn:
rmFile.PFN = pfn
remove.addFile( rmFile )
self.request.addOperation( remove )
return S_OK()
def test05List(self):
""" getitem, setitem, delitem and dirty """
op = Operation()
files = []
for i in range(5):
f = File()
files.append(f)
op += f
for i in range(len(op)):
self.assertEqual(op[i], files[i], "__getitem__ failed")
for i in range(len(op)):
op[i] = File({"LFN": "/%s" % i})
self.assertEqual(op[i].LFN, "/%s" % i, "__setitem__ failed")
del op[0]
self.assertEqual(len(op), 4, "__delitem__ failed")
# # opID set
op.OperationID = 1
del op[0]
self.assertEqual(op.cleanUpSQL(), None,
"cleanUp failed after __delitem__")
op[0].FileID = 1
del op[0]
self.assertEqual(
op.cleanUpSQL(),
"DELETE FROM `File` WHERE `OperationID` = 1 AND `FileID` IN (1);\n",
"cleanUp failed after __delitem__")
op[0].FileID = 2
op[0] = File({"FileID": 2})
self.assertEqual(
op.cleanUpSQL(),
"DELETE FROM `File` WHERE `OperationID` = 1 AND `FileID` IN (1,2);\n",
"cleanUp failed after __setitem__")
json = op.toJSON()
self.assertEqual("__dirty" in json["Value"], True, "missing __dirty")
op2 = Operation(json["Value"])
self.assertEqual(
op2.cleanUpSQL(),
"DELETE FROM `File` WHERE `OperationID` = 1 AND `FileID` IN (1,2);\n",
"cleanUp failed after JSON")
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
"""deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = "ReplicateAndRegister"
if transBody:
try:
_requestType, requestOperation = transBody.split(";")
except AttributeError:
pass
failedTasks = []
# Do not remove sorted, we might pop elements in the loop
for taskID, task in taskDict.items():
transID = task["TransformationID"]
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = task["TargetSE"]
# If there are input files
if task.get("InputData"):
if isinstance(task["InputData"], list):
files = task["InputData"]
elif isinstance(task["InputData"], six.string_types):
files = task["InputData"].split(";")
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
result = self._assignRequestToTask(oRequest, taskDict, transID,
taskID, ownerDN, ownerGroup)
if not result["OK"]:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def test04Stress(self):
""" stress test """
db = RequestDB()
for i in range(self.stressRequests):
request = Request({"RequestName": "test-%d" % i})
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
put = db.putRequest(request)
self.assertEqual(put["OK"], True, "put failed")
startTime = time.time()
for i in range(self.stressRequests):
get = db.getRequest("test-%s" % i, True)
if "Message" in get:
print get["Message"]
self.assertEqual(get["OK"], True, "get failed")
endTime = time.time()
print "getRequest duration %s " % (endTime - startTime)
for i in range(self.stressRequests):
delete = db.deleteRequest("test-%s" % i)
self.assertEqual(delete["OK"], True, "delete failed")
def test01Stress(self):
""" stress test """
db = RequestDB()
reqIDs = []
for i in xrange(self.stressRequests):
request = Request({"RequestName": "test-%d" % i})
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
put = db.putRequest(request)
self.assertEqual(put["OK"], True,
put['Message'] if 'Message' in put else 'OK')
reqIDs.append(put['Value'])
startTime = time.time()
for reqID in reqIDs:
get = db.getRequest(reqID)
if "Message" in get:
print(get["Message"])
self.assertEqual(get["OK"], True,
get['Message'] if 'Message' in get else 'OK')
endTime = time.time()
print("getRequest duration %s " % (endTime - startTime))
for reqID in reqIDs:
delete = db.deleteRequest(reqID)
self.assertEqual(
delete["OK"], True,
delete['Message'] if 'Message' in delete else 'OK')
def test_ctor():
"""test constructors and (de)serialisation"""
assert isinstance(Operation(), Operation), "empty ctor failed"
# # using fromDict
fromDict = {
"Type": "replicateAndRegister",
"TargetSE": "CERN-USER,PIC-USER",
"SourceSE": None,
}
operation = Operation(fromDict)
assert isinstance(operation, Operation), "fromDict ctor failed"
for key, value in fromDict.items():
assert getattr(operation,
key) == value, "wrong attr value %s (%s) %s" % (
key, getattr(operation, key), value)
# # same with file
operation = Operation(fromDict)
operation.addFile(
File({
"LFN": "/lhcb/user/c/cibak/testFile",
"Checksum": "1234567",
"ChecksumType": "ADLER32",
"Size": 1024,
"Status": "Waiting",
}))
for key, value in fromDict.items():
assert getattr(operation,
key) == value, "wrong attr value %s (%s) %s" % (
key, getattr(operation, key), value)
toJSON = operation.toJSON()
assert toJSON["OK"], "JSON serialization failed"
def test_scheduled(reqDB):
"""scheduled request r/w"""
req = Request({"RequestName": "FTSTest"})
op = Operation({"Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"})
op += File({
"LFN": "/a/b/c",
"Status": "Scheduled",
"Checksum": "123456",
"ChecksumType": "ADLER32"
})
req += op
put = reqDB.putRequest(req)
assert put["OK"], put
reqID = put["Value"]
peek = reqDB.peekRequest(reqID)
assert peek["OK"], peek
peek = peek["Value"]
for op in peek:
opId = op.OperationID
getFTS = reqDB.getScheduledRequest(opId)
assert getFTS["OK"], getFTS
assert getFTS[
"Value"].RequestName == "FTSTest", "Wrong request name %s" % getFTS[
"Value"].RequestName
delete = reqDB.deleteRequest(reqID)
assert delete["OK"], delete
def test_stress(reqDB):
"""stress test"""
reqIDs = []
for i in range(STRESS_REQUESTS):
request = Request({"RequestName": "test-%d" % i})
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
put = reqDB.putRequest(request)
assert put["OK"], put
reqIDs.append(put["Value"])
startTime = time.time()
for reqID in reqIDs:
get = reqDB.getRequest(reqID)
assert get["OK"], get
endTime = time.time()
print("getRequest duration %s " % (endTime - startTime))
for reqID in reqIDs:
delete = reqDB.deleteRequest(reqID)
assert delete["OK"], delete
def test05Scheduled(self):
""" scheduled request r/w """
db = RequestDB()
req = Request({"RequestName": "FTSTest"})
op = Operation({
"Type": "ReplicateAndRegister",
"TargetSE": "CERN-USER"
})
op += File({
"LFN": "/a/b/c",
"Status": "Scheduled",
"Checksum": "123456",
"ChecksumType": "ADLER32"
})
req += op
put = db.putRequest(req)
self.assertEqual(put["OK"], True, "putRequest failed")
peek = db.peekRequest(req.RequestName)
self.assertEqual(peek["OK"], True, "peek failed ")
peek = peek["Value"]
for op in peek:
opId = op.OperationID
getFTS = db.getScheduledRequest(opId)
self.assertEqual(getFTS["OK"], True, "getScheduled failed")
self.assertEqual(getFTS["Value"].RequestName, "FTSTest",
"wrong request selected")
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
if self.getCSOption("DelayedExternalDeletion", True):
gLogger.info("Setting deletion request")
try:
# We need the hostDN used in order to pass these credentials to the
# SandboxStoreDB..
hostCertLocation, _ = Locations.getHostCertificateAndKeyLocation(
)
hostCert = X509Certificate.X509Certificate()
hostCert.loadFromFile(hostCertLocation)
hostDN = hostCert.getSubjectDN().get("Value")
# use the host authentication to fetch the data
result = self.sandboxDB.getSandboxOwner(
SEName, SEPFN, hostDN, "hosts")
if not result["OK"]:
return result
_owner, ownerDN, ownerGroup = result["Value"]
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % (
SEName, SEPFN, time.time())
request.OwnerDN = ownerDN
request.OwnerGroup = ownerGroup
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile(fileToRemove)
request.addOperation(physicalRemoval)
return ReqClient().putRequest(request)
except Exception as e:
gLogger.exception("Exception while setting deletion request")
return S_ERROR(f"Cannot set deletion request: {e}")
else:
gLogger.info("Deleting external Sandbox")
try:
return StorageElement(SEName).removeFile(SEPFN)
except Exception:
gLogger.exception(
"RM raised an exception while trying to delete a remote sandbox"
)
return S_ERROR(
"RM raised an exception while trying to delete a remote sandbox"
)
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split(';')
except AttributeError:
pass
failedTasks = []
# Do not remove sorted, we might pop elements in the loop
for taskID, task in taskDict.iteritems():
transID = task['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = task['TargetSE']
# If there are input files
if task.get('InputData'):
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def _getFileProperties(self, fileID, columnNames=None):
""" select :columnNames: from File table """
columnNames = columnNames if columnNames else File.tableDesc(
)["Fields"].keys()
columnNames = ",".join(
['`%s`' % str(columnName) for columnName in columnNames])
return "SELECT %s FROM `File` WHERE `FileID` = %s;" % (columnNames,
int(fileID))
def setBKRegistrationRequest(self, lfn, error='',
metaData={'Checksum': 'justSomething',
'ChecksumType': 'ADLER32',
'GUID': 'aGUID'}):
""" Set a BK registration request for changing the replica flag.
Uses the global request object (self.request).
"""
if error:
self.log.info('BK registration for %s failed with message: "%s" setting failover request' % (lfn, error))
else:
self.log.info('Setting BK registration request for %s' % (lfn))
regFile = Operation()
regFile.Type = 'RegisterFile'
regFile.Catalog = 'BookkeepingDB'
bkFile = File()
bkFile.LFN = lfn
# this should NOT be needed... but RMS complains!
bkFile.PFN = lfn
bkFile.GUID = metaData['GUID']
bkFile.Checksum = metaData['Checksum']
bkFile.ChecksumType = metaData['ChecksumType']
regFile.addFile(bkFile)
res = self.request.addOperation(regFile)
if not res['OK']:
raise RuntimeError(res['Message'])
def __setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE
:param list catalog: list of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get( "Checksum", "" )
regFile.ChecksumType = fileDict.get( "ChecksumType", "" )
regFile.Size = fileDict.get( "Size", 0 )
regFile.GUID = fileDict.get( "GUID", "" )
se = StorageElement( targetSE )
pfn = se.getPfnForLfn( lfn )
if not pfn["OK"]:
self.log.error( "unable to get PFN for LFN: %s" % pfn["Message"] )
return pfn
regFile.PFN = pfn["Value"]
register.addFile( regFile )
self.request.addOperation( register )
return S_OK()
def _setFileReplicationRequest(self,
lfn,
targetSE,
fileMetaDict,
sourceSE=""):
"""Sets a registration request."""
self.log.info("Setting ReplicateAndRegister request",
"for %s to %s" % (lfn, targetSE))
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.TargetSE = targetSE
if sourceSE:
transfer.SourceSE = sourceSE
trFile = File()
trFile.LFN = lfn
cksm = fileMetaDict.get("Checksum", None)
cksmType = fileMetaDict.get("ChecksumType", self.defaultChecksumType)
if cksm and cksmType:
trFile.Checksum = cksm
trFile.ChecksumType = cksmType
size = fileMetaDict.get("Size", 0)
if size:
trFile.Size = size
guid = fileMetaDict.get("GUID", "")
if guid:
trFile.GUID = guid
transfer.addFile(trFile)
self.request.addOperation(transfer)
return S_OK()
def test01TableDesc(self):
""" table description """
tableDict = RequestDB.getTableMeta()
self.assertEqual("Request" in tableDict, True)
self.assertEqual("Operation" in tableDict, True)
self.assertEqual("File" in tableDict, True)
self.assertEqual(tableDict["Request"], Request.tableDesc())
self.assertEqual(tableDict["Operation"], Operation.tableDesc())
self.assertEqual(tableDict["File"], File.tableDesc())
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
""" Set removal request with the given credentials
"""
oRequest = Request()
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml'
oRequest.SourceComponent = 'JobCleaningAgent'
removeFile = Operation()
removeFile.Type = 'RemoveFile'
removedFile = File()
removedFile.LFN = lfn
removeFile.addFile( removedFile )
oRequest.addOperation( removeFile )
return ReqClient().putRequest( oRequest )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if type( paramDict['InputData'] ) == type( [] ):
files = paramDict['InputData']
elif type( paramDict['InputData'] ) == type( '' ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
oRequest.OwnerDN = owner
oRequest.OwnerGroup = ownerGroup
isValid = gRequestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def test__addMetadataToFiles( self ):
resMeta = {'OK': True,
'Value': {'Failed': {},
'Successful': {'/lhcb/1.dst': {'ChecksumType': 'AD',
'Checksum': '123456',
'CreationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ),
'GUID': '92F9CE97-7A62-E311-8401-0025907FD430',
'Mode': 436,
'ModificationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ),
'NumberOfLinks': 1,
'Size': 5846023777,
'Status': '-'},
'/lhcb/2.dst': {'ChecksumType': 'AD',
'Checksum': '987654',
'CreationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ),
'GUID': 'DAE4933A-C162-E311-8A6B-003048FEAF04',
'Mode': 436,
'ModificationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ),
'NumberOfLinks': 1,
'Size': 5893396937,
'Status': '-'}}}}
self.rr.fc.getFileMetadata.return_value = resMeta
file1 = File()
file1.LFN = '/lhcb/1.dst'
file2 = File()
file2.LFN = '/lhcb/2.dst'
toSchedule = {'/lhcb/1.dst': [file1, ['SE1'], ['SE2', 'SE3']],
'/lhcb/2.dst': [file2, ['SE4'], ['SE5', 'SE6']]}
res = self.rr._addMetadataToFiles( toSchedule )
self.assertTrue(res['OK'])
for lfn in toSchedule:
self.assertEqual( res['Value'][lfn].LFN, lfn )
for attr in ('GUID', 'Size', 'Checksum'):
self.assertEqual( getattr(res['Value'][lfn],attr), resMeta['Value']['Successful'][lfn][attr] )
# AD should be transformed into Adler32
self.assertEqual( res['Value'][lfn].ChecksumType, "ADLER32" )
def __setReplicaRemovalRequest( self, lfn, se ):
""" Sets a removal request for a replica.
:param str lfn: LFN
:param se:
"""
if type( se ) == str:
se = ",".join( [ se.strip() for se in se.split( "," ) if se.strip() ] )
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = se
replicaToRemove = File()
replicaToRemove.LFN = lfn
removeReplica.addFile( replicaToRemove )
self.request.addOperation( removeReplica )
return S_OK()
def test__addMetadataToFiles( self ):
resMeta = {'OK': True,
'Value': {'Failed': {},
'Successful': {'/lhcb/1.dst': {'ChecksumType': 'AD',
'Checksum': '123456',
'CreationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ),
'GUID': '92F9CE97-7A62-E311-8401-0025907FD430',
'Mode': 436,
'ModificationDate': datetime.datetime( 2013, 12, 11, 20, 20, 21 ),
'NumberOfLinks': 1,
'Size': 5846023777,
'Status': '-'},
'/lhcb/2.dst': {'ChecksumType': 'AD',
'Checksum': '987654',
'CreationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ),
'GUID': 'DAE4933A-C162-E311-8A6B-003048FEAF04',
'Mode': 436,
'ModificationDate': datetime.datetime( 2013, 12, 12, 6, 26, 52 ),
'NumberOfLinks': 1,
'Size': 5893396937,
'Status': '-'}}}}
self.rr.fc.getFileMetadata.return_value = resMeta
file1 = File()
file1.LFN = '/lhcb/1.dst'
file2 = File()
file2.LFN = '/lhcb/2.dst'
toSchedule = {'/lhcb/1.dst': [file1, ['SE1'], ['SE2', 'SE3']],
'/lhcb/2.dst': [file2, ['SE4'], ['SE5', 'SE6']]}
res = self.rr._addMetadataToFiles( toSchedule )
self.assert_( res['OK'] )
self.assertEqual( json.loads( res['Value'][0][0] )['LFN'], resMeta['Value']['Successful'].keys()[0] )
self.assertEqual( json.loads( res['Value'][0][0] )['Size'], resMeta['Value']['Successful'].values()[0]['Size'] )
self.assertEqual( json.loads( res['Value'][1][0] )['LFN'], resMeta['Value']['Successful'].keys()[1] )
self.assertEqual( json.loads( res['Value'][1][0] )['Size'], resMeta['Value']['Successful'].values()[1]['Size'] )
def __setFileReplicationRequest( self, lfn, targetSE, fileMetaDict, sourceSE = '' ):
""" Sets a registration request.
"""
self.log.info( 'Setting replication request for %s to %s' % ( lfn, targetSE ) )
transfer = Operation()
transfer.Type = "ReplicateAndRegister"
transfer.TargetSE = targetSE
if sourceSE:
transfer.SourceSE = sourceSE
trFile = File()
trFile.LFN = lfn
cksm = fileMetaDict.get( "Checksum", None )
cksmType = fileMetaDict.get( "ChecksumType", None )
if cksm and cksmType:
trFile.Checksum = cksm
trFile.ChecksumType = cksmType
size = fileMetaDict.get( "Size", 0 )
if size:
trFile.Size = size
guid = fileMetaDict.get( "GUID", "" )
if guid:
trFile.GUID = guid
transfer.addFile( trFile )
self.request.addOperation( transfer )
return S_OK()
def test_db( self ):
""" table description """
tableDict = RequestDB.getTableMeta()
self.assertEqual( "Request" in tableDict, True )
self.assertEqual( "Operation" in tableDict, True )
self.assertEqual( "File" in tableDict, True )
self.assertEqual( tableDict["Request"], Request.tableDesc() )
self.assertEqual( tableDict["Operation"], Operation.tableDesc() )
self.assertEqual( tableDict["File"], File.tableDesc() )
# # empty DB at that stage
ret = RequestDB().getDBSummary()
self.assertEqual( ret,
{ 'OK': True,
'Value': { 'Operation': {}, 'Request': {}, 'File': {} } } )
def getRegisterOperation( self, opFile, targetSE ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = "RegisterFile"
registerOperation.TargetSE = targetSE
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = opFile.PFN
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
return registerOperation
def addRegisterReplica( self, opFile, targetSE ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = "RegisterFile"
registerOperation.TargetSE = targetSE
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = opFile.PFN
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
self.request.insertAfter( registerOperation, self.operation )
return S_OK()
def _setRegistrationRequest( self, lfn, targetSE, fileDict, catalog ):
""" Sets a registration request
:param str lfn: LFN
:param list se: list of SE (or just string)
:param list catalog: list (or string) of catalogs to use
:param dict fileDict: file metadata
"""
self.log.info( 'Setting registration request for %s at %s.' % ( lfn, targetSE ) )
if not isinstance( catalog, list ):
catalog = [catalog]
for cat in catalog:
register = Operation()
register.Type = "RegisterFile"
register.Catalog = cat
register.TargetSE = targetSE
regFile = File()
regFile.LFN = lfn
regFile.Checksum = fileDict.get( "Checksum", "" )
regFile.ChecksumType = fileDict.get( "ChecksumType", self.defaultChecksumType )
regFile.Size = fileDict.get( "Size", 0 )
regFile.GUID = fileDict.get( "GUID", "" )
se = StorageElement( targetSE )
pfn = se.getURL( lfn, self.registrationProtocols )
if not pfn["OK"] or lfn not in pfn["Value"]['Successful']:
self.log.error( "Unable to get PFN for LFN", "%s" % pfn.get( 'Message', pfn.get( 'Value', {} ).get( 'Failed', {} ).get( lfn ) ) )
return pfn
regFile.PFN = pfn["Value"]['Successful'][lfn]
register.addFile( regFile )
self.request.addOperation( register )
return S_OK()
def getRegisterOperation( self, opFile, targetSE, type = 'RegisterFile', catalog = None ):
""" add RegisterReplica operation for file
:param File opFile: operation file
:param str targetSE: target SE
"""
# # add RegisterReplica operation
registerOperation = Operation()
registerOperation.Type = type
registerOperation.TargetSE = targetSE
if catalog:
registerOperation.Catalog = catalog
registerFile = File()
registerFile.LFN = opFile.LFN
registerFile.PFN = StorageElement( targetSE ).getURL( opFile.LFN, protocol = self.registrationProtocols ).get( 'Value', {} ).get( 'Successful', {} ).get( opFile.LFN )
registerFile.GUID = opFile.GUID
registerFile.Checksum = opFile.Checksum
registerFile.ChecksumType = opFile.ChecksumType
registerFile.Size = opFile.Size
registerOperation.addFile( registerFile )
return registerOperation
def createRequest( reqType ):
r = Request()
# Simple failover
op1 = Operation()
f = File()
f.LFN = '/This/is/an/LFN'
op1.addFile( f )
op1.Type = 'ReplicateAndRegister'
op1.SourceSE = 'CERN-FAILOVER'
op1.TargetSE = 'CERN-BUFFER'
r.addOperation( op1 )
op2 = Operation()
op2.addFile( f )
op2.Type = 'RemoveReplica'
op2.TargetSE = 'CERN-FAILOVER'
r.addOperation( op2 )
if reqType == 0:
return r
# two files for Failover
f1 = File()
f1.LFN = '/This/is/a/second/LFN'
op3 = Operation()
op3.addFile( f1 )
op3.Type = 'ReplicateAndRegister'
op3.SourceSE = 'CERN-FAILOVER'
op3.TargetSE = 'CERN-BUFFER'
r.addOperation( op3 )
op3 = Operation()
op3.addFile( f1 )
op3.Type = 'RemoveReplica'
op3.TargetSE = 'CERN-FAILOVER'
r.addOperation( op3 )
if reqType == 1:
return r
op = Operation()
op.Type = 'ForwardDiset'
if reqType == 2:
r.addOperation( op )
return r
r.insertBefore( op, r[0] )
if reqType == 3:
return r
op4 = Operation()
op4.Type = 'ForwardDiset'
r.addOperation( op4 )
if reqType == 4:
return r
# 2 different FAILOVER SEs: removal not optimized
r[1].SourceSE = 'RAL-FAILOVER'
r[2].SourceSE = 'RAL-FAILOVER'
if reqType == 5:
return r
# 2 different destinations, same FAILOVER: replication not optimized
r[3].SourceSE = 'RAL-FAILOVER'
r[4].SourceSE = 'RAL-FAILOVER'
r[3].TargetSE = 'RAL-BUFFER'
if reqType == 6:
return r
print 'This should not happen, reqType =', reqType
replicateAndRegister.Type = 'ReplicateAndRegister'
replicateAndRegister.TargetSE = targetSE
res = fc.getFileMetadata( lfnList )
if not res['OK']:
print "Can't get file metadata: %s" % res['Message']
DIRAC.exit( 1 )
if res['Value']['Failed']:
print "Could not get the file metadata of the following, so skipping them:"
for fFile in res['Value']['Failed']:
print fFile
lfnMetadata = res['Value']['Successful']
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
rarFile.Size = lfnMetadata[lfn]['Size']
rarFile.Checksum = lfnMetadata[lfn]['Checksum']
rarFile.GUID = lfnMetadata[lfn]['GUID']
rarFile.ChecksumType = 'ADLER32'
replicateAndRegister.addFile( rarFile )
oRequest.addOperation( replicateAndRegister )
isValid = RequestValidator().validate( oRequest )
if not isValid['OK']:
print "Request is not valid: ", isValid['Message']
DIRAC.exit( 1 )
result = reqClient.putRequest( oRequest )
if result['OK']:
def _getFileProperties( self, fileID, columnNames = None ): """ select :columnNames: from File table """ columnNames = columnNames if columnNames else File.tableDesc()["Fields"].keys() columnNames = ",".join( [ '`%s`' % str( columnName ) for columnName in columnNames ] ) return "SELECT %s FROM `File` WHERE `FileID` = %s;" % ( columnNames, int( fileID ) )
DIRAC.exit( -1 )
if not os.path.isfile( PFN ):
gLogger.error( "%s is not a file" % PFN )
DIRAC.exit( -1 )
PFN = os.path.abspath( PFN )
size = os.path.getsize( PFN )
adler32 = fileAdler( PFN )
request = Request()
request.RequestName = requestName
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = targetSE
opFile = File()
opFile.LFN = LFN
opFile.PFN = PFN
opFile.Size = size
opFile.Checksum = adler32
opFile.ChecksumType = "ADLER32"
putAndRegister.addFile( opFile )
request.addOperation( putAndRegister )
reqClient = ReqClient()
putRequest = reqClient.putRequest( request )
if not putRequest["OK"]:
gLogger.error( "unable to put request '%s': %s" % ( requestName, putRequest["Message"] ) )
DIRAC.exit( -1 )
gLogger.always( "Request '%s' has been put to ReqDB for execution." % requestName )
gLogger.always( "You can monitor its status using command: 'dirac-rms-show-request %s'" % requestName )
error = -1
continue
count += 1
request = Request()
request.RequestName = requestName if not multiRequests else '%s_%d' % ( requestName, count )
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = ",".join( targetSEs )
if catalog is not None:
replicateAndRegister.Catalog = catalog
for lfn in lfnChunk:
metaDict = metaDatas["Successful"][lfn]
opFile = File()
opFile.LFN = lfn
opFile.Size = metaDict["Size"]
if "Checksum" in metaDict:
# # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
opFile.Checksum = metaDict["Checksum"]
opFile.ChecksumType = "ADLER32"
replicateAndRegister.addFile( opFile )
request.addOperation( replicateAndRegister )
putRequest = reqClient.putRequest( request )
if not putRequest["OK"]:
gLogger.error( "unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) )
error = -1
def test02props( self ):
""" test props and attributes """
theFile = File()
# valid props
theFile.FileID = 1
self.assertEqual( theFile.FileID, 1 )
theFile.Status = "Done"
self.assertEqual( theFile.Status, "Done" )
theFile.LFN = "/some/path/somewhere"
self.assertEqual( theFile.LFN, "/some/path/somewhere" )
theFile.PFN = "/some/path/somewhere"
self.assertEqual( theFile.PFN, "/some/path/somewhere" )
theFile.Attempt = 1
self.assertEqual( theFile.Attempt, 1 )
theFile.Size = 1
self.assertEqual( theFile.Size, 1 )
theFile.GUID = "2bbabe80-e2f1-11e1-9b23-0800200c9a66"
self.assertEqual( theFile.GUID, "2bbabe80-e2f1-11e1-9b23-0800200c9a66" )
theFile.ChecksumType = "adler32"
self.assertEqual( theFile.ChecksumType, "ADLER32" )
theFile.Checksum = "123456"
self.assertEqual( theFile.Checksum, "123456" )
# #
theFile.Checksum = None
theFile.ChecksumType = None
self.assertEqual( theFile.Checksum, "" )
self.assertEqual( theFile.ChecksumType, "" )
# # invalid props
# FileID
try:
theFile.FileID = "foo"
except Exception, error:
self.assertEqual( isinstance( error, ValueError ), True )
def test05FTS( self ):
""" FTS state machine """
req = Request()
req.RequestName = "FTSTest"
ftsTransfer = Operation()
ftsTransfer.Type = "ReplicateAndRegister"
ftsTransfer.TargetSE = "CERN-USER"
ftsFile = File()
ftsFile.LFN = "/a/b/c"
ftsFile.Checksum = "123456"
ftsFile.ChecksumType = "Adler32"
ftsTransfer.addFile( ftsFile )
req.addOperation( ftsTransfer )
self.assertEqual( req.Status, "Waiting", "1. wrong request status: %s" % req.Status )
self.assertEqual( ftsTransfer.Status, "Waiting", "1. wrong ftsStatus status: %s" % ftsTransfer.Status )
# # scheduled
ftsFile.Status = "Scheduled"
self.assertEqual( ftsTransfer.Status, "Scheduled", "2. wrong status for ftsTransfer: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Scheduled", "2. wrong status for request: %s" % req.Status )
# # add new operation before FTS
insertBefore = Operation()
insertBefore.Type = "RegisterReplica"
insertBefore.TargetSE = "CERN-USER"
insertFile = File()
insertFile.LFN = "/a/b/c"
insertFile.PFN = "http://foo/bar"
insertBefore.addFile( insertFile )
req.insertBefore( insertBefore, ftsTransfer )
self.assertEqual( insertBefore.Status, "Waiting", "3. wrong status for insertBefore: %s" % insertBefore.Status )
self.assertEqual( ftsTransfer.Status, "Scheduled", "3. wrong status for ftsStatus: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Waiting", "3. wrong status for request: %s" % req.Status )
# # prev done
insertFile.Status = "Done"
self.assertEqual( insertBefore.Status, "Done", "4. wrong status for insertBefore: %s" % insertBefore.Status )
self.assertEqual( ftsTransfer.Status, "Scheduled", "4. wrong status for ftsStatus: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Scheduled", "4. wrong status for request: %s" % req.Status )
# # reschedule
ftsFile.Status = "Waiting"
self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status )
self.assertEqual( ftsTransfer.Status, "Waiting", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Waiting", "5. wrong status for request: %s" % req.Status )
# # fts done
ftsFile.Status = "Done"
self.assertEqual( insertBefore.Status, "Done", "5. wrong status for insertBefore: %s" % insertBefore.Status )
self.assertEqual( ftsTransfer.Status, "Done", "5. wrong status for ftsStatus: %s" % ftsTransfer.Status )
self.assertEqual( req.Status, "Done", "5. wrong status for request: %s" % req.Status )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if not taskDict:
return S_OK({})
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
# Do not remove sorted, we might pop elements in the loop
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
# If there are input files
if paramDict['InputData']:
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
self.log.error( "Error creating request for task", "%s %s" % ( taskID, isValid ) )
# This works because we loop over a copy of the keys !
taskDict.pop( taskID )
continue
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
