def __submitRMSOp(self, target_se, lfns_chunk_dict, whichRMSOp='ReplicateAndRegister' ):
""" target_se : SE name to which to replicate
lfns_chunk_dict : LFNS dict with 100 lfns as key andeEach lfn has 'Size', 'Checksum'
whichRMSOp: Choose from RMP operation - ReplicateAndRegister, ReplicateAndRemove, PutAndRegister
"""
## Setup request
request = Request()
request.RequestName = "DDM_"+ str(target_se) + datetime.datetime.now().strftime("_%Y%m%d_%H%M%S")
myOp = Operation()
myOp.Type = whichRMSOp
myOp.TargetSE = target_se
## Add LFNS to operations
for lfn in lfns_chunk_dict.keys():
opFile = File()
opFile.LFN = lfn
opFile.Size = lfns_chunk_dict[lfn]['Size']
if "Checksum" in lfns_chunk_dict[lfn]:
opFile.Checksum = lfns_chunk_dict[lfn]['Checksum']
opFile.ChecksumType = 'ADLER32'
## Add file to operation
myOp.addFile( opFile )
request.addOperation( myOp )
reqClient = ReqClient()
putRequest = reqClient.putRequest( request )
if not putRequest["OK"]:
gLogger.error( "Unable to put request '%s': %s" % ( request.RequestName, putRequest["Message"] ) )
return S_ERROR("Problem submitting to RMS.")
def archiveRequestAndOp(listOfLFNs):
"""Return a tuple of the request and operation."""
req = Request()
req.RequestName = "MyRequest"
op = Operation()
switches = {}
archiveLFN = "/vo/tars/myTar.tar"
op.Arguments = DEncode.encode({
"SourceSE":
switches.get("SourceSE", "SOURCE-SE"),
"TarballSE":
switches.get("TarballSE", "TARBALL-SE"),
"RegisterDescendent":
False,
"ArchiveLFN":
archiveLFN,
})
op.Type = "ArchiveFiles"
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = "01130a%0d" % index
oFile.ChecksumType = "adler32"
op.addFile(oFile)
req.addOperation(op)
return req, op
def archiveRequestAndOp(listOfLFNs):
"""Return a tuple of the request and operation."""
req = Request()
req.RequestName = 'MyRequest'
op = Operation()
switches = {}
archiveLFN = '/vo/tars/myTar.tar'
op.Arguments = DEncode.encode({
'SourceSE':
switches.get('SourceSE', 'SOURCE-SE'),
'TarballSE':
switches.get('TarballSE', 'TARBALL-SE'),
'RegisterDescendent':
False,
'ArchiveLFN':
archiveLFN
})
op.Type = 'ArchiveFiles'
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = '01130a%0d' % index
oFile.ChecksumType = 'adler32'
op.addFile(oFile)
req.addOperation(op)
return req, op
def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ):
if self.getCSOption( "DelayedExternalDeletion", True ):
gLogger.info( "Setting deletion request" )
try:
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() )
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile( fileToRemove )
request.addOperation( physicalRemoval )
return ReqClient().putRequest( request )
except Exception as e:
gLogger.exception( "Exception while setting deletion request" )
return S_ERROR( "Cannot set deletion request: %s" % str( e ) )
else:
gLogger.info( "Deleting external Sandbox" )
try:
return StorageElement( SEName ).removeFile( SEPFN )
except Exception as e:
gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" )
return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )
def setUp(self):
""" test case set up """
gLogger.setLevel('NOTICE')
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.file2 = File()
self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
self.file2.Checksum = "654321"
self.file2.ChecksumType = "ADLER32"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.operation.addFile(self.file)
self.operation.addFile(self.file2)
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
self.request.OwnerGroup = "dirac_user"
self.request.JobID = 123
self.request.addOperation(self.operation)
# # JSON representation of a whole request
self.jsonStr = self.request.toJSON()['Value']
# # request client
self.requestClient = ReqClient()
self.stressRequests = 1000
self.bulkRequest = 1000
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
if self.getCSOption("DelayedExternalDeletion", True):
gLogger.info("Setting deletion request")
try:
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % (
SEName, SEPFN, time.time())
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile(fileToRemove)
request.addOperation(physicalRemoval)
return ReqClient().putRequest(request)
except Exception as e:
gLogger.exception("Exception while setting deletion request")
return S_ERROR("Cannot set deletion request: %s" % str(e))
else:
gLogger.info("Deleting external Sandbox")
try:
return StorageElement(SEName).removeFile(SEPFN)
except Exception as e:
gLogger.exception(
"RM raised an exception while trying to delete a remote sandbox"
)
return S_ERROR(
"RM raised an exception while trying to delete a remote sandbox"
)
class OperationHandlerBaseTests( unittest.TestCase ):
"""
.. class:: OperationHandlerBaseTests
"""
def setUp( self ):
""" test set up """
self.req = Request()
self.req.RequestName = "testRequest"
self.op = Operation( {"Type" : "ForwardDISET", "Arguments" : "foobar" } )
self.req.addOperation( self.op )
self.baseOp = OperationHandlerBase()
def tearDown( self ):
""" test tear down """
del self.baseOp
del self.op
del self.req
def testOperationHandlerBase( self ):
""" base op test """
self.baseOp.setOperation( self.op )
# # log is there
self.assertEqual( "log" in dir( self.baseOp ), True, "log missing" )
# # operation is there
self.assertEqual( "operation" in dir( self.baseOp ), True, "operation is missing" )
# # request is there
self.assertEqual( "request" in dir( self.baseOp ), True, "request is missing" )
# # __call__ not implemented
self.assertRaises( NotImplementedError, self.baseOp )
# # replica manager
self.assertEqual( isinstance( self.baseOp.dm, DataManager ), True, "DataManager is missing" )
def setUp(self):
""" test case setup """
self.request = Request({"RequestName": "test1", "JobID": 1})
self.operation1 = Operation({
"Type": "ReplicateAndRegister",
"TargetSE": "CERN-USER"
})
self.file = File({
"LFN": "/a/b/c",
"ChecksumType": "ADLER32",
"Checksum": "123456"
})
self.request.addOperation(self.operation1)
self.operation1.addFile(self.file)
self.operation2 = Operation()
self.operation2.Type = "RemoveFile"
self.operation2.addFile(File({"LFN": "/c/d/e"}))
self.request.addOperation(self.operation2)
# ## set some defaults
gConfig.setOptionValue('DIRAC/Setup', 'Test')
gConfig.setOptionValue('/DIRAC/Setups/Test/RequestManagement', 'Test')
gConfig.setOptionValue(
'/Systems/RequestManagement/Test/Databases/ReqDB/Host',
'localhost')
gConfig.setOptionValue(
'/Systems/RequestManagement/Test/Databases/ReqDB/DBName', 'ReqDB')
gConfig.setOptionValue(
'/Systems/RequestManagement/Test/Databases/ReqDB/User', 'Dirac')
self.i = 1000
def setUp(self):
""" test case set up """
gLogger.setLevel('INFO')
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.file2 = File()
self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
self.file2.Checksum = "654321"
self.file2.ChecksumType = "ADLER32"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.operation.addFile(self.file)
self.operation.addFile(self.file2)
proxyInfo = getProxyInfo()['Value']
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = proxyInfo['identity']
self.request.OwnerGroup = proxyInfo['group']
self.request.JobID = 123
self.request.addOperation(self.operation)
# # JSON representation of a whole request
self.jsonStr = self.request.toJSON()['Value']
# # request client
self.requestClient = ReqClient()
class RequestTaskTests( unittest.TestCase ):
"""
.. class:: RequestTaskTests
"""
def setUp( self ):
""" test case set up """
self.handlersDict = { "ForwardDISET" : "DIRAC/RequestManagementSystem/private/ForwardDISET" }
self.req = Request()
self.req.RequestName = "foobarbaz"
self.req.OwnerGroup = "dirac_user"
self.req.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
self.op = Operation( { "Type": "ForwardDISET", "Arguments" : "tts10:helloWorldee" } )
self.req.addOperation( self.op )
self.task = None
def tearDown( self ):
""" test case tear down """
del self.req
del self.op
del self.task
def testAPI( self ):
""" test API """
self.task = RequestTask( self.req.toXML()["Value"], self.handlersDict )
self.task.requestClient = Mock( return_value = Mock(spec=RequestClient) )
self.task.requestClient().updateRequest = Mock()
self.task.requestClient().updateRequest.return_value = { "OK" : True, "Value" : None }
ret = self.task()
self.assertEqual( ret["OK"], True , "call failed")
def export_putRequest(self, requestJSON):
"""put a new request into RequestDB
:param cls: class ref
:param str requestJSON: request serialized to JSON format
"""
requestDict = json.loads(requestJSON)
requestName = requestDict.get("RequestID", requestDict.get("RequestName", "***UNKNOWN***"))
request = Request(requestDict)
# Check whether the credentials in the Requests are correct and allowed to be set
isAuthorized = RequestValidator.setAndCheckRequestOwner(request, self.getRemoteCredentials())
if not isAuthorized:
return S_ERROR(DErrno.ENOAUTH, "Credentials in the requests are not allowed")
optimized = request.optimize()
if optimized.get("Value", False):
gLogger.debug("putRequest: request was optimized")
else:
gLogger.debug("putRequest: request unchanged", optimized.get("Message", "Nothing could be optimized"))
valid = self.validate(request)
if not valid["OK"]:
gLogger.error("putRequest: request %s not valid: %s" % (requestName, valid["Message"]))
return valid
# If NotBefore is not set or user defined, we calculate its value
now = datetime.datetime.utcnow().replace(microsecond=0)
extraDelay = datetime.timedelta(0)
if request.Status not in Request.FINAL_STATES and (not request.NotBefore or request.NotBefore < now):
# We don't delay if it is the first insertion
if getattr(request, "RequestID", 0):
# If it is a constant delay, just set it
if self.constantRequestDelay:
extraDelay = datetime.timedelta(minutes=self.constantRequestDelay)
else:
# If there is a waiting Operation with Files
op = request.getWaiting().get("Value")
if op and len(op):
attemptList = [opFile.Attempt for opFile in op if opFile.Status == "Waiting"]
if attemptList:
maxWaitingAttempt = max([opFile.Attempt for opFile in op if opFile.Status == "Waiting"])
# In case it is the first attempt, extraDelay is 0
# maxWaitingAttempt can be None if the operation has no File, like the ForwardDiset
extraDelay = datetime.timedelta(
minutes=2 * math.log(maxWaitingAttempt) if maxWaitingAttempt else 0
)
request.NotBefore = now + extraDelay
gLogger.info(
"putRequest: request %s not before %s (extra delay %s)"
% (request.RequestName, request.NotBefore, extraDelay)
)
requestName = request.RequestName
gLogger.info("putRequest: Attempting to set request '%s'" % requestName)
return self.__requestDB.putRequest(request)
def export_putRequest(cls, requestJSON):
""" put a new request into RequestDB
:param cls: class ref
:param str requestJSON: request serialized to JSON format
"""
requestName = requestJSON.get("RequestName", "***UNKNOWN***")
request = Request(requestJSON)
requestID = request.RequestID
optimized = request.optimize()
if optimized.get("Value", False):
if request.RequestID == 0 and requestID != 0:
# A new request has been created, delete the old one
delete = cls.__requestDB.deleteRequest(request.RequestName)
if not delete['OK']:
return delete
gLogger.debug(
"putRequest: request was optimized and removed for a new insertion"
)
else:
gLogger.debug("putRequest: request was optimized")
else:
gLogger.debug(
"putRequest: request unchanged",
optimized.get("Message", "Nothing could be optimized"))
valid = cls.validate(request)
if not valid["OK"]:
gLogger.error("putRequest: request %s not valid: %s" %
(requestName, valid["Message"]))
return valid
requestName = request.RequestName
gLogger.info("putRequest: Attempting to set request '%s'" %
requestName)
return cls.__requestDB.putRequest(request)
def setUp(self):
""" test set up """
self.req = Request()
self.req.RequestName = "testRequest"
self.op = Operation({"Type": "ForwardDISET", "Arguments": "foobar"})
self.req.addOperation(self.op)
self.baseOp = OperationHandlerBase()
def setUp(self):
""" test case set up """
self.handlersDict = {
"ForwardDISET":
"DIRAC/RequestManagementSystem/private/ForwardDISET"
}
self.req = Request()
self.req.RequestName = "foobarbaz"
self.req.OwnerGroup = "lhcb_user"
self.req.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=chaen/CN=705305/CN=Christophe Haen"
self.op = Operation({
"Type": "ForwardDISET",
"Arguments": "tts10:helloWorldee"
})
self.req.addOperation(self.op)
self.task = None
self.mockRC = MagicMock()
self.mockObjectOps = MagicMock()
self.mockObjectOps.getSections.return_value = {
'OK': True,
'Value': ['DataProcessing', 'DataManager']
}
self.mockObjectOps.getOptionsDict.return_value = {
'OK': True,
'Value': {
'Group': 'lhcb_user',
'User': '******'
}
}
self.mockOps = MagicMock()
self.mockOps.return_value = self.mockObjectOps
def test01CtorSerilization(self):
""" c'tor and serialization """
# # empty c'tor
req = Request()
self.assertEqual(isinstance(req, Request), True)
self.assertEqual(req.JobID, 0)
self.assertEqual(req.Status, "Waiting")
req = Request(self.fromDict)
self.assertEqual(isinstance(req, Request), True)
self.assertEqual(req.RequestName, "test")
self.assertEqual(req.JobID, 12345)
self.assertEqual(req.Status, "Waiting")
toJSON = req.toJSON()
self.assertEqual(toJSON["OK"], True, "JSON serialization failed")
fromJSON = toJSON["Value"]
req = Request(fromJSON)
toSQL = req.toSQL()
self.assertEqual(toSQL["OK"], True)
toSQL = toSQL["Value"]
self.assertEqual(toSQL.startswith("INSERT"), True)
req.RequestID = 1
toSQL = req.toSQL()
self.assertEqual(toSQL["OK"], True)
toSQL = toSQL["Value"]
self.assertEqual(toSQL.startswith("UPDATE"), True)
def export_putRequest(cls, requestJSON):
""" put a new request into RequestDB
:param cls: class ref
:param str requestJSON: request serialized to JSON format
"""
requestName = requestJSON.get("RequestName", "***UNKNOWN***")
request = Request(requestJSON)
optimized = request.optimize()
if optimized.get("Value", False):
gLogger.debug("putRequest: request was optimized")
else:
gLogger.debug(
"putRequest: request unchanged",
optimized.get("Message", "Nothing could be optimize"))
valid = cls.validate(request)
if not valid["OK"]:
gLogger.error("putRequest: request %s not valid: %s" %
(requestName, valid["Message"]))
return valid
requestName = request.RequestName
gLogger.info("putRequest: Attempting to set request '%s'" %
requestName)
return cls.__requestDB.putRequest(request)
class MoveReplicaSuccess( ReqOpsTestCase ):
def setUp(self):
self.op = Operation()
self.op.Type = "MoveFile"
self.op.SourceSE = "%s,%s" % ( "sourceSE1", "sourceSE2" )
self.op.TargetSE = "%s,%s" % ( "targetSE1", "targetSE2" )
self.File = File()
self.File.LFN = '/cta/file1'
self.File.Size = 2L
self.File.Checksum = '011300a2'
self.File.ChecksumType = "adler32"
self.op.addFile( self.File )
self.req = Request()
self.req.addOperation( self.op )
self.mr = MoveReplica( self.op )
self.mr.dm = MagicMock()
self.mr.fc = MagicMock()
# This test needs to be fixed. It currently fails because StorageElement is not mocked
'''def test__dmTransfer( self ):
successful = {}
for sourceSE in self.op.sourceSEList:
successful[sourceSE] = 'dips://' + sourceSE.lower() + ':9148/DataManagement/StorageElement' + self.File.LFN
res = {'OK': True, 'Value': {'Successful': {self.File.LFN : successful}, 'Failed': {}}}
self.mr.dm.getActiveReplicas.return_value = res
res = {'OK': True, 'Value': {'Successful': {self.File.LFN : {'register': 0.1228799819946289, 'replicate': 9.872732877731323}}, 'Failed': {}}}
self.mr.dm.replicateAndRegister.return_value = res
res = self.mr.dmTransfer( self.File )
self.assertTrue( res['OK'] )
self.assertEqual( self.mr.operation.__files__[0].Status, 'Waiting' )
self.assertEqual( self.mr.operation.Status, 'Waiting' )
self.assertEqual( self.mr.request.Status, 'Waiting' )'''
def test__dmRemoval( self ):
res = {'OK': True, 'Value': {'Successful': { self.File.LFN : {'DIRACFileCatalog': True}}, 'Failed': {}}}
self.mr.dm.removeReplica.return_value = res
toRemoveDict = {self.File.LFN: self.File}
targetSEs = self.op.sourceSEList
res = self.mr.dmRemoval( toRemoveDict, targetSEs )
self.assertTrue( res['OK'] )
resvalue = dict( [ ( targetSE, '' ) for targetSE in targetSEs ] )
self.assertEqual( res['Value'], {self.File.LFN: resvalue} )
self.assertEqual( self.mr.operation.__files__[0].Status, 'Done' )
self.assertEqual( self.mr.operation.Status, 'Done' )
self.assertEqual( self.mr.request.Status, 'Done' )
class MoveReplicaSuccess( ReqOpsTestCase ):
def setUp(self):
self.op = Operation()
self.op.Type = "MoveFile"
self.op.SourceSE = "%s,%s" % ( "sourceSE1", "sourceSE2" )
self.op.TargetSE = "%s,%s" % ( "targetSE1", "targetSE2" )
self.File = File()
self.File.LFN = '/cta/file1'
self.File.Size = 2L
self.File.Checksum = '011300a2'
self.File.ChecksumType = "adler32"
self.op.addFile( self.File )
self.req = Request()
self.req.addOperation( self.op )
self.mr = MoveReplica( self.op )
self.mr.dm = MagicMock()
self.mr.fc = MagicMock()
# This test needs to be fixed. It currently fails because StorageElement is not mocked
'''def test__dmTransfer( self ):
successful = {}
for sourceSE in self.op.sourceSEList:
successful[sourceSE] = 'dips://' + sourceSE.lower() + ':9148/DataManagement/StorageElement' + self.File.LFN
res = {'OK': True, 'Value': {'Successful': {self.File.LFN : successful}, 'Failed': {}}}
self.mr.dm.getActiveReplicas.return_value = res
res = {'OK': True, 'Value': {'Successful': {self.File.LFN : {'register': 0.1228799819946289, 'replicate': 9.872732877731323}}, 'Failed': {}}}
self.mr.dm.replicateAndRegister.return_value = res
res = self.mr.dmTransfer( self.File )
self.assertTrue( res['OK'] )
self.assertEqual( self.mr.operation.__files__[0].Status, 'Waiting' )
self.assertEqual( self.mr.operation.Status, 'Waiting' )
self.assertEqual( self.mr.request.Status, 'Waiting' )'''
def test__dmRemoval( self ):
res = {'OK': True, 'Value': {'Successful': { self.File.LFN : {'DIRACFileCatalog': True}}, 'Failed': {}}}
self.mr.dm.removeReplica.return_value = res
toRemoveDict = {self.File.LFN: self.File}
targetSEs = self.op.sourceSEList
res = self.mr.dmRemoval( toRemoveDict, targetSEs )
self.assertTrue( res['OK'] )
resvalue = dict( [ ( targetSE, '' ) for targetSE in targetSEs ] )
self.assertEqual( res['Value'], {self.File.LFN: resvalue} )
self.assertEqual( self.mr.operation.__files__[0].Status, 'Done' )
self.assertEqual( self.mr.operation.Status, 'Done' )
self.assertEqual( self.mr.request.Status, 'Done' )
def createRequest(requestID, opType, opStatus, fileStatus, lfnError=" ",
lfn="/ilc/fake/lfn"):
"""Create a request."""
req = Request({"RequestID": requestID})
op = Operation({"Type": opType, "Status": opStatus})
op.addFile(File({"LFN": lfn, "Status": fileStatus, "Error": lfnError}))
req.addOperation(op)
return req
def export_putRequest(cls, requestJSON):
""" put a new request into RequestDB
:param cls: class ref
:param str requestJSON: request serialized to JSON format
"""
requestDict = json.loads(requestJSON)
requestName = requestDict.get(
"RequestID", requestDict.get('RequestName', "***UNKNOWN***"))
request = Request(requestDict)
optimized = request.optimize()
if optimized.get("Value", False):
gLogger.debug("putRequest: request was optimized")
else:
gLogger.debug(
"putRequest: request unchanged",
optimized.get("Message", "Nothing could be optimized"))
valid = cls.validate(request)
if not valid["OK"]:
gLogger.error("putRequest: request %s not valid: %s" %
(requestName, valid["Message"]))
return valid
# If NotBefore is not set or user defined, we calculate its value
now = datetime.datetime.utcnow().replace(microsecond=0)
extraDelay = datetime.timedelta(0)
if request.Status not in Request.FINAL_STATES and (
not request.NotBefore or request.NotBefore < now):
op = request.getWaiting().get('Value')
# If there is a waiting Operation with Files
if op and len(op):
attemptList = [
opFile.Attempt for opFile in op
if opFile.Status == "Waiting"
]
if attemptList:
maxWaitingAttempt = max([
opFile.Attempt for opFile in op
if opFile.Status == "Waiting"
])
# In case it is the first attempt, extraDelay is 0
# maxWaitingAttempt can be None if the operation has no File, like the ForwardDiset
extraDelay = datetime.timedelta(
minutes=2 *
math.log(maxWaitingAttempt) if maxWaitingAttempt else 0
)
request.NotBefore = now + extraDelay
gLogger.info("putRequest: request %s not before %s (extra delay %s)" %
(request.RequestName, request.NotBefore, extraDelay))
requestName = request.RequestName
gLogger.info("putRequest: Attempting to set request '%s'" %
requestName)
return cls.__requestDB.putRequest(request)
def __init__(
self,
requestJSON,
handlersDict,
csPath,
agentName,
standalone=False,
requestClient=None,
rmsMonitoring=False):
"""c'tor
:param self: self reference
:param str requestJSON: request serialized to JSON
:param dict opHandlers: operation handlers
"""
self.request = Request(requestJSON)
# # csPath
self.csPath = csPath
# # agent name
self.agentName = agentName
# # standalone flag
self.standalone = standalone
# # handlers dict
self.handlersDict = handlersDict
# # handlers class def
self.handlers = {}
# # own sublogger
self.log = gLogger.getSubLogger("pid_%s/%s" % (os.getpid(), self.request.RequestName))
# # get shifters info
self.__managersDict = {}
shifterProxies = self.__setupManagerProxies()
if not shifterProxies["OK"]:
self.log.error("Cannot setup shifter proxies", shifterProxies["Message"])
# This flag which is set and sent from the RequestExecutingAgent and is False by default.
self.rmsMonitoring = rmsMonitoring
if self.rmsMonitoring:
self.rmsMonitoringReporter = MonitoringReporter(monitoringType="RMSMonitoring")
else:
# # initialize gMonitor
gMonitor.setComponentType(gMonitor.COMPONENT_AGENT)
gMonitor.setComponentName(self.agentName)
gMonitor.initialize()
# # own gMonitor activities
gMonitor.registerActivity("RequestAtt", "Requests processed",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("RequestFail", "Requests failed",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("RequestOK", "Requests done",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM)
if requestClient is None:
self.requestClient = ReqClient()
else:
self.requestClient = requestClient
def test06Dirty( self ):
""" dirty records """
db = RequestDB()
r = Request()
r.RequestName = "dirty"
op1 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op1 += File( {"LFN": "/a/b/c/1", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
op2 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op2 += File( {"LFN": "/a/b/c/2", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
op3 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op3 += File( {"LFN": "/a/b/c/3", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
r += op1
r += op2
r += op3
put = db.putRequest( r )
self.assertEqual( put["OK"], True, "1. putRequest failed: %s" % put.get( "Message", "" ) )
reqID = put['Value']
r = db.getRequest( reqID )
self.assertEqual( r["OK"], True, "1. getRequest failed: %s" % r.get( "Message", "" ) )
r = r["Value"]
del r[0]
self.assertEqual( len( r ), 2, "1. len wrong" )
put = db.putRequest( r )
self.assertEqual( put["OK"], True, "2. putRequest failed: %s" % put.get( "Message", "" ) )
reqID = put['Value']
r = db.getRequest( reqID )
self.assertEqual( r["OK"], True, "2. getRequest failed: %s" % r.get( "Message", "" ) )
r = r["Value"]
self.assertEqual( len( r ), 2, "2. len wrong" )
op4 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
op4 += File( {"LFN": "/a/b/c/4", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )
r[0] = op4
put = db.putRequest( r )
self.assertEqual( put["OK"], True, "3. putRequest failed: %s" % put.get( "Message", "" ) )
reqID = put['Value']
r = db.getRequest( reqID )
self.assertEqual( r["OK"], True, "3. getRequest failed: %s" % r.get( "Message", "" ) )
r = r["Value"]
self.assertEqual( len( r ), 2, "3. len wrong" )
delete = db.deleteRequest( reqID )
self.assertEqual( delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK' )
def __init__(self,request=None,init=True):
# A common set of attributes that define requests.
self.requestAttributes = ['SubRequestID','TargetSE','Status','Operation','SourceSE','Catalogue','SpaceToken','RequestType']
# Possible keys to define the files in the request.
self.fileAttributes = ['LFN','Size','PFN','GUID','Md5','Addler','Status','Attempt','FileID']
# Possible keys to define the dataset in the request.
self.datasetAttributes = ['Handle']
Request.__init__(self,request,init)
def setUp( self ):
""" test case set up """
self.handlersDict = { "ForwardDISET" : "DIRAC/RequestManagementSystem/private/ForwardDISET" }
self.req = Request()
self.req.RequestName = "foobarbaz"
self.req.OwnerGroup = "dirac_user"
self.req.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
self.op = Operation( { "Type": "ForwardDISET", "Arguments" : "tts10:helloWorldee" } )
self.req.addOperation( self.op )
self.task = None
def taskToRequest(self, taskID, task, transID):
"""Convert a task into an RMS with a single ForwardDISET
Operation whose attribute is the number of files in the task
"""
req = Request()
op = Operation()
op.Type = "ForwardDISET"
op.Arguments = json.dumps(len(task["InputData"]) * self.factor)
req.addOperation(op)
return req
def createRequest(self, requestName, lfnChunk):
"""Create the Request."""
request = Request()
request.RequestName = requestName
replicate = Operation()
replicate.Type = 'ReplicateAndRegister'
replicate.TargetSE = self.switches.get('TargetSE')
self.addLFNs(replicate, lfnChunk, addPFN=True)
request.addOperation(replicate)
if self.switches.get('CheckMigration'):
checkMigration = Operation()
checkMigration.Type = 'CheckMigration'
checkMigration.TargetSE = self.switches.get('TargetSE')
self.addLFNs(checkMigration, lfnChunk, addPFN=True)
request.addOperation(checkMigration)
removeReplicas = Operation()
removeReplicas.Type = 'RemoveReplica'
removeReplicas.TargetSE = ','.join(self.switches.get('SourceSE', []))
self.addLFNs(removeReplicas, lfnChunk)
request.addOperation(removeReplicas)
return request
def createRequest(self, requestName, lfnChunk):
"""Create the Request."""
request = Request()
request.RequestName = requestName
replicate = Operation()
replicate.Type = "ReplicateAndRegister"
replicate.TargetSE = self.switches.get("TargetSE")
self.addLFNs(replicate, lfnChunk, addPFN=True)
request.addOperation(replicate)
if self.switches.get("CheckMigration"):
checkMigration = Operation()
checkMigration.Type = "CheckMigration"
checkMigration.TargetSE = self.switches.get("TargetSE")
self.addLFNs(checkMigration, lfnChunk, addPFN=True)
request.addOperation(checkMigration)
removeReplicas = Operation()
removeReplicas.Type = "RemoveReplica"
removeReplicas.TargetSE = ",".join(self.switches.get("SourceSE", []))
self.addLFNs(removeReplicas, lfnChunk)
request.addOperation(removeReplicas)
return request
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has multiple operations
:param transJson: list of lists of string and dictionaries, e.g.:
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
failedTasks = []
for taskID, task in taskDict.items():
transID = task['TransformationID']
if not task.get('InputData'):
self._logError("Error creating request for task",
"%s, No input data" % taskID,
transID=transID)
taskDict.pop(taskID)
continue
files = []
oRequest = Request()
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
# create the operations from the json structure
for operationTuple in transJson:
op = Operation()
op.Type = operationTuple[0]
for parameter, value in operationTuple[1].iteritems():
setattr(op, parameter, value)
for lfn in files:
opFile = File()
opFile.LFN = lfn
op.addFile(opFile)
oRequest.addOperation(op)
result = self._assignRequestToTask(oRequest, taskDict, transID,
taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
class RequestTaskTests(unittest.TestCase):
"""
.. class:: RequestTaskTests
"""
def setUp(self):
"""test case set up"""
self.handlersDict = {"ForwardDISET": "DIRAC/RequestManagementSystem/private/ForwardDISET"}
self.req = Request()
self.req.RequestName = "foobarbaz"
self.req.OwnerGroup = "lhcb_user"
self.req.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=chaen/CN=705305/CN=Christophe Haen"
self.op = Operation({"Type": "ForwardDISET", "Arguments": "tts10:helloWorldee"})
self.req.addOperation(self.op)
self.task = None
self.mockRC = MagicMock()
self.mockObjectOps = MagicMock()
self.mockObjectOps.getSections.return_value = {"OK": True, "Value": ["DataProcessing", "DataManager"]}
self.mockObjectOps.getOptionsDict.return_value = {
"OK": True,
"Value": {"Group": "lhcb_user", "User": "******"},
}
self.mockOps = MagicMock()
self.mockOps.return_value = self.mockObjectOps
def tearDown(self):
"""test case tear down"""
del self.req
del self.op
del self.task
def testAPI(self):
"""test API"""
rt = importlib.import_module("DIRAC.RequestManagementSystem.private.RequestTask")
rt.gMonitor = MagicMock()
rt.Operations = self.mockOps
rt.CS = MagicMock()
self.task = RequestTask(
self.req.toJSON()["Value"],
self.handlersDict,
"csPath",
"RequestManagement/RequestExecutingAgent",
requestClient=self.mockRC,
)
self.task.requestClient = Mock(return_value=Mock(spec=ReqClient))
self.task.requestClient().updateRequest = Mock()
self.task.requestClient().updateRequest.return_value = {"OK": True, "Value": None}
ret = self.task()
self.assertEqual(ret["OK"], True, "call failed")
ret = self.task.setupProxy()
print(ret)
def _sendToFailover( rpcStub ): """ Create a ForwardDISET operation for failover """ request = Request() request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ) forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode( rpcStub ) request.addOperation( forwardDISETOp ) return ReqClient().putRequest( request )
def _sendToFailover( rpcStub ): """ Create a ForwardDISET operation for failover """ request = Request() request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ) forwardDISETOp = Operation() forwardDISETOp.Type = "ForwardDISET" forwardDISETOp.Arguments = DEncode.encode( rpcStub ) request.addOperation( forwardDISETOp ) return ReqClient().putRequest( request )
def export_putRequest(cls, requestJSON):
""" put a new request into RequestDB
:param cls: class ref
:param str requestJSON: request serialized to JSON format
"""
requestDict = json.loads(requestJSON)
requestName = requestDict.get("RequestID", requestDict.get("RequestName", "***UNKNOWN***"))
request = Request(requestDict)
optimized = request.optimize()
if optimized.get("Value", False):
gLogger.debug("putRequest: request was optimized")
else:
gLogger.debug("putRequest: request unchanged", optimized.get("Message", "Nothing could be optimized"))
valid = cls.validate(request)
if not valid["OK"]:
gLogger.error("putRequest: request %s not valid: %s" % (requestName, valid["Message"]))
return valid
# If NotBefore is not set or user defined, we calculate its value
now = datetime.datetime.utcnow().replace(microsecond=0)
extraDelay = datetime.timedelta(0)
if request.Status not in Request.FINAL_STATES and (not request.NotBefore or request.NotBefore < now):
# We don't delay if it is the first insertion
if getattr(request, "RequestID", 0):
# If it is a constant delay, just set it
if cls.constantRequestDelay:
extraDelay = datetime.timedelta(minutes=cls.constantRequestDelay)
else:
# If there is a waiting Operation with Files
op = request.getWaiting().get("Value")
if op and len(op):
attemptList = [opFile.Attempt for opFile in op if opFile.Status == "Waiting"]
if attemptList:
maxWaitingAttempt = max([opFile.Attempt for opFile in op if opFile.Status == "Waiting"])
# In case it is the first attempt, extraDelay is 0
# maxWaitingAttempt can be None if the operation has no File, like the ForwardDiset
extraDelay = datetime.timedelta(
minutes=2 * math.log(maxWaitingAttempt) if maxWaitingAttempt else 0
)
request.NotBefore = now + extraDelay
gLogger.info(
"putRequest: request %s not before %s (extra delay %s)"
% (request.RequestName, request.NotBefore, extraDelay)
)
requestName = request.RequestName
gLogger.info("putRequest: Attempting to set request '%s'" % requestName)
return cls.__requestDB.putRequest(request)
def test07Authorization(self):
""" Test whether request sets on behalf of others are rejected"""
request = Request({"RequestName": "unauthorized"})
request.OwnerDN = 'NotMe'
request.OwnerDN = 'AnotherGroup'
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
res = self.requestClient.putRequest(request)
self.assertFalse(res['OK'], res)
def __init__( self, requestObject = None ):
""" Constructor function, can specify request object to instantiate
FailoverTransfer or a new request object is created.
"""
self.log = gLogger.getSubLogger( "FailoverTransfer" )
self.replicaMgr = ReplicaManager()
self.request = requestObject
if not self.request:
self.request = Request()
self.request.RequestName = 'default_request.xml'
self.request.SourceComponent = 'FailoverTransfer'
def test_setGet(self):
res = self.transformation.setTransformationName('TestTName')
self.assertTrue(res['OK'])
description = 'Test transformation description'
res = self.transformation.setDescription(description)
longDescription = 'Test transformation long description'
res = self.transformation.setLongDescription(longDescription)
self.assertTrue(res['OK'])
res = self.transformation.setType('MCSimulation')
self.assertTrue(res['OK'])
res = self.transformation.setPlugin('aPlugin')
self.assertTrue(res['OK'])
# # Test DataOperation Body
res = self.transformation.setBody("")
self.assertTrue(res['OK'])
self.assertEqual(self.transformation.paramValues["Body"], "")
res = self.transformation.setBody("_requestType;RemoveReplica")
self.assertTrue(res['OK'])
self.assertEqual(self.transformation.paramValues["Body"], "_requestType;RemoveReplica")
# #Json will turn tuples to lists and strings to unicode
transBody = [[u"ReplicateAndRegister", {u"SourceSE": u"FOO-SRM", u"TargetSE": u"BAR-SRM"}],
[u"RemoveReplica", {u"TargetSE": u"FOO-SRM"}],
]
res = self.transformation.setBody(transBody)
self.assertTrue(res['OK'])
self.assertEqual(self.transformation.paramValues["Body"], json.dumps(transBody))
# # This is not true if any of the keys or values are not strings, e.g., integers
self.assertEqual(json.loads(self.transformation.paramValues["Body"]), transBody)
with self.assertRaisesRegexp(TypeError, "Expected list"):
self.transformation.setBody({"ReplicateAndRegister": {"foo": "bar"}})
with self.assertRaisesRegexp(TypeError, "Expected tuple"):
self.transformation.setBody(["ReplicateAndRegister", "RemoveReplica"])
with self.assertRaisesRegexp(TypeError, "Expected 2-tuple"):
self.transformation.setBody([("ReplicateAndRegister", "RemoveReplica", "LogUpload")])
with self.assertRaisesRegexp(TypeError, "Expected string"):
self.transformation.setBody([(123, "Parameter:Value")])
with self.assertRaisesRegexp(TypeError, "Expected dictionary"):
self.transformation.setBody([("ReplicateAndRegister", "parameter=foo")])
with self.assertRaisesRegexp(TypeError, "Expected string"):
self.transformation.setBody([("ReplicateAndRegister", {123: "foo"})])
with self.assertRaisesRegexp(ValueError, "Unknown attribute"):
self.transformation.setBody([("ReplicateAndRegister", {"Request": Request()})])
with self.assertRaisesRegexp(TypeError, "Cannot encode"):
self.transformation.setBody([("ReplicateAndRegister", {"Arguments": Request()})])
def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has multiple operations
:param transJson: list of lists of string and dictionaries, e.g.:
.. code :: python
body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
]
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
failedTasks = []
for taskID, task in taskDict.items():
transID = task['TransformationID']
if not task.get('InputData'):
self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID)
taskDict.pop(taskID)
continue
files = []
oRequest = Request()
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
# create the operations from the json structure
for operationTuple in transJson:
op = Operation()
op.Type = operationTuple[0]
for parameter, value in operationTuple[1].iteritems():
setattr(op, parameter, value)
for lfn in files:
opFile = File()
opFile.LFN = lfn
op.addFile(opFile)
oRequest.addOperation(op)
result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
class RequestTaskTests( unittest.TestCase ):
"""
.. class:: RequestTaskTests
"""
def setUp( self ):
""" test case set up """
self.handlersDict = { "ForwardDISET" : "DIRAC/RequestManagementSystem/private/ForwardDISET" }
self.req = Request()
self.req.RequestName = "foobarbaz"
self.req.OwnerGroup = "lhcb_user"
self.req.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=chaen/CN=705305/CN=Christophe Haen"
self.op = Operation( { "Type": "ForwardDISET", "Arguments" : "tts10:helloWorldee" } )
self.req.addOperation( self.op )
self.task = None
self.mockRC = MagicMock()
self.mockObjectOps = MagicMock()
self.mockObjectOps.getSections.return_value = {'OK': True,
'Value': ['DataProcessing',
'DataManager']}
self.mockObjectOps.getOptionsDict.return_value = {'OK': True,
'Value': {'Group': 'lhcb_user', 'User': '******'}}
self.mockOps = MagicMock()
self.mockOps.return_value = self.mockObjectOps
def tearDown( self ):
""" test case tear down """
del self.req
del self.op
del self.task
def testAPI( self ):
""" test API
"""
rt = importlib.import_module( 'DIRAC.RequestManagementSystem.private.RequestTask' )
rt.gMonitor = MagicMock()
rt.Operations = self.mockOps
rt.CS = MagicMock()
self.task = RequestTask( self.req.toJSON()["Value"], self.handlersDict, 'csPath', 'RequestManagement/RequestExecutingAgent',
requestClient = self.mockRC )
self.task.requestClient = Mock( return_value = Mock( spec = ReqClient ) )
self.task.requestClient().updateRequest = Mock()
self.task.requestClient().updateRequest.return_value = { "OK" : True, "Value" : None }
ret = self.task()
self.assertEqual( ret["OK"], True , "call failed" )
ret = self.task.setupProxy()
print ret
class RequestManagerHandlerTests( unittest.TestCase ):
"""
.. class:: RequestManagerHandlerTests
"""
def setUp( self ):
""" test setup
:param self: self reference
"""
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
self.request.OwnerGroup = "dirac_user"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.request.addOperation( self.operation )
self.operation.addFile( self.file )
# # xml representation of a whole request
self.xmlStr = self.request.toXML( True )["Value"]
# # request client
self.requestClient = RequestClient()
def tearDown( self ):
""" test case tear down """
del self.request
del self.operation
del self.file
del self.xmlStr
def test01PutRequest( self ):
""" test set request """
put = self.requestClient.putRequest( self.request )
self.assertEqual( put["OK"], True, "put failed" )
def test02GetRequest( self ):
""" test get request """
get = self.requestClient.getRequest( self.request.RequestName )
self.assertEqual( get["OK"], True, "get failed" )
def test03DeleteRequest( self ):
""" test delete request """
delete = self.requestClient.deleteRequest( "test" )
self.assertEqual( delete["OK"], True, "delete failed" )
def buildRequest(self, owner, group, sourceSE, targetSE1, targetSE2):
files = self.files(owner, group)
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
putFile = File()
putFile.LFN = lfn
putFile.PFN = fname
putFile.Checksum = checksum
putFile.ChecksumType = "adler32"
putFile.Size = size
putFile.GUID = guid
putAndRegister.addFile(putFile)
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = "%s,%s" % (targetSE1, targetSE2)
for fname, lfn, size, checksum, guid in files:
repFile = File()
repFile.LFN = lfn
repFile.Size = size
repFile.Checksum = checksum
repFile.ChecksumType = "adler32"
replicateAndRegister.addFile(repFile)
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
removeReplica.addFile(File({"LFN": lfn}))
removeFile = Operation()
removeFile.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFile.addFile(File({"LFN": lfn}))
removeFileInit = Operation()
removeFileInit.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFileInit.addFile(File({"LFN": lfn}))
req = Request()
req.addOperation(removeFileInit)
req.addOperation(putAndRegister)
req.addOperation(replicateAndRegister)
req.addOperation(removeReplica)
req.addOperation(removeFile)
return req
def checkRequestAndOp(listOfLFNs):
req = Request()
req.RequestName = 'MyRequest'
op = Operation()
op.Type = 'CheckMigration'
for index, lfn in enumerate(listOfLFNs):
oFile = File()
oFile.LFN = lfn
oFile.Size = index
oFile.Checksum = '01130a%0d' % index
oFile.ChecksumType = 'adler32'
op.addFile(oFile)
req.addOperation(op)
return req, op
class ReqClientTestCase(unittest.TestCase):
"""
.. class:: ReqClientTestCase
"""
def setUp(self):
""" test case set up """
gLogger.setLevel('INFO')
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.file2 = File()
self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
self.file2.Checksum = "654321"
self.file2.ChecksumType = "ADLER32"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.operation.addFile(self.file)
self.operation.addFile(self.file2)
proxyInfo = getProxyInfo()['Value']
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = proxyInfo['identity']
self.request.OwnerGroup = proxyInfo['group']
self.request.JobID = 123
self.request.addOperation(self.operation)
# # JSON representation of a whole request
self.jsonStr = self.request.toJSON()['Value']
# # request client
self.requestClient = ReqClient()
self.stressRequests = 1000
self.bulkRequest = 1000
def tearDown(self):
""" clean up """
del self.request
del self.operation
del self.file
del self.jsonStr
class ReqClientTestCase( unittest.TestCase ):
"""
.. class:: ReqClientTestCase
"""
def setUp( self ):
""" test case set up """
gLogger.setLevel( 'NOTICE' )
self.file = File()
self.file.LFN = "/lhcb/user/c/cibak/testFile"
self.file.Checksum = "123456"
self.file.ChecksumType = "ADLER32"
self.file2 = File()
self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
self.file2.Checksum = "654321"
self.file2.ChecksumType = "ADLER32"
self.operation = Operation()
self.operation.Type = "ReplicateAndRegister"
self.operation.TargetSE = "CERN-USER"
self.operation.addFile( self.file )
self.operation.addFile( self.file2 )
self.request = Request()
self.request.RequestName = "RequestManagerHandlerTests"
self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
self.request.OwnerGroup = "dirac_user"
self.request.JobID = 123
self.request.addOperation( self.operation )
# # JSON representation of a whole request
self.jsonStr = self.request.toJSON()['Value']
# # request client
self.requestClient = ReqClient()
self.stressRequests = 1000
self.bulkRequest = 1000
def tearDown( self ):
""" clean up """
del self.request
del self.operation
del self.file
del self.jsonStr
def test03sql( self ):
""" sql insert or update """
operation = Operation()
operation.Type = "ReplicateAndRegister"
request = Request()
request.RequestName = "testRequest"
request.RequestID = 1
# # no parent request set
try:
operation.toSQL()
except Exception, error:
self.assertEqual( isinstance( error, AttributeError ), True, "wrong exc raised" )
self.assertEqual( str( error ), "RequestID not set", "wrong exc reason" )
def _sendToFailover( rpcStub ):
""" Create a ForwardDISET operation for failover
"""
try:
request = Request()
request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() )
forwardDISETOp = Operation()
forwardDISETOp.Type = "ForwardDISET"
forwardDISETOp.Arguments = DEncode.encode( rpcStub )
request.addOperation( forwardDISETOp )
return ReqClient().putRequest( request )
# We catch all the exceptions, because it should never crash
except Exception as e: # pylint: disable=broad-except
return S_ERROR( ERMSUKN, "Exception sending accounting failover request: %s" % repr( e ) )
def setUp( self ):
""" test set up """
self.req = Request()
self.req.RequestName = "testRequest"
self.op = Operation( {"Type" : "ForwardDISET", "Arguments" : "foobar" } )
self.req.addOperation( self.op )
self.baseOp = OperationHandlerBase()
def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup):
""" deal with a Request that has just one operation, as it was sofar
:param transBody: string, can be an empty string
:param dict taskDict: dictionary of tasks, modified in this function
:param str ownerDN: certificate DN used for the requests
:param str onwerGroup: dirac group used for the requests
:returns: None
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split(';')
except AttributeError:
pass
failedTasks = []
# Do not remove sorted, we might pop elements in the loop
for taskID, task in taskDict.iteritems():
transID = task['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = task['TargetSE']
# If there are input files
if task.get('InputData'):
if isinstance(task['InputData'], list):
files = task['InputData']
elif isinstance(task['InputData'], basestring):
files = task['InputData'].split(';')
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
if not result['OK']:
failedTasks.append(taskID)
# Remove failed tasks
for taskID in failedTasks:
taskDict.pop(taskID)
def buildRequest( self, owner, group, sourceSE, targetSE1, targetSE2 ):
files = self.files( owner, group )
putAndRegister = Operation()
putAndRegister.Type = "PutAndRegister"
putAndRegister.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
putFile = File()
putFile.LFN = lfn
putFile.PFN = fname
putFile.Checksum = checksum
putFile.ChecksumType = "adler32"
putFile.Size = size
putFile.GUID = guid
putAndRegister.addFile( putFile )
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = "%s,%s" % ( targetSE1, targetSE2 )
for fname, lfn, size, checksum, guid in files:
repFile = File()
repFile.LFN = lfn
repFile.Size = size
repFile.Checksum = checksum
repFile.ChecksumType = "adler32"
replicateAndRegister.addFile( repFile )
removeReplica = Operation()
removeReplica.Type = "RemoveReplica"
removeReplica.TargetSE = sourceSE
for fname, lfn, size, checksum, guid in files:
removeReplica.addFile( File( {"LFN": lfn } ) )
removeFile = Operation()
removeFile.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFile.addFile( File( {"LFN": lfn } ) )
removeFileInit = Operation()
removeFileInit.Type = "RemoveFile"
for fname, lfn, size, checksum, guid in files:
removeFileInit.addFile( File( {"LFN": lfn } ) )
req = Request()
req.addOperation( removeFileInit )
req.addOperation( putAndRegister )
req.addOperation( replicateAndRegister )
req.addOperation( removeReplica )
req.addOperation( removeFile )
return req
def test01CtorSerilization( self ):
""" c'tor and serialization """
# # empty c'tor
req = Request()
self.assertEqual( isinstance( req, Request ), True )
self.assertEqual( req.JobID, 0 )
self.assertEqual( req.Status, "Waiting" )
req = Request( self.fromDict )
self.assertEqual( isinstance( req, Request ), True )
self.assertEqual( req.RequestName, "test" )
self.assertEqual( req.JobID, 12345 )
self.assertEqual( req.Status, "Waiting" )
toJSON = req.toJSON()
self.assertEqual( toJSON["OK"], True, "JSON serialization failed" )
fromJSON = toJSON["Value"]
req = Request( fromJSON )
toSQL = req.toSQL()
self.assertEqual( toSQL["OK"], True )
toSQL = toSQL["Value"]
self.assertEqual( toSQL.startswith( "INSERT" ), True )
req.RequestID = 1
toSQL = req.toSQL()
self.assertEqual( toSQL["OK"], True )
toSQL = toSQL["Value"]
self.assertEqual( toSQL.startswith( "UPDATE" ), True )
def test01TableDesc(self):
""" table description """
tableDict = RequestDB.getTableMeta()
self.assertEqual("Request" in tableDict, True)
self.assertEqual("Operation" in tableDict, True)
self.assertEqual("File" in tableDict, True)
self.assertEqual(tableDict["Request"], Request.tableDesc())
self.assertEqual(tableDict["Operation"], Operation.tableDesc())
self.assertEqual(tableDict["File"], File.tableDesc())
def test07List( self ):
""" setitem, delitem, getitem and dirty """
r = Request()
ops = [ Operation() for i in range( 5 ) ]
for op in ops:
r.addOperation( op )
for i, op in enumerate( ops ):
self.assertEqual( op, r[i], "__getitem__ failed" )
op = Operation()
r[0] = op
self.assertEqual( op, r[0], "__setitem__ failed" )
del r[0]
self.assertEqual( len( r ), 4, "__delitem__ failed" )
def export_putRequest( cls, requestJSON ):
""" put a new request into RequestDB
:param cls: class ref
:param str requestJSON: request serialized to JSON format
"""
requestName = requestJSON.get( "RequestName", "***UNKNOWN***" )
request = Request( requestJSON )
optimized = request.optimize()
if optimized.get("Value", False):
gLogger.debug( "putRequest: request was optimized" )
else:
gLogger.debug( "putRequest: request unchanged", optimized.get( "Message", "Nothing could be optimize" ) )
valid = cls.validate( request )
if not valid["OK"]:
gLogger.error( "putRequest: request %s not valid: %s" % ( requestName, valid["Message"] ) )
return valid
requestName = request.RequestName
gLogger.info( "putRequest: Attempting to set request '%s'" % requestName )
return cls.__requestDB.putRequest( request )
def test07Authorization(self):
""" Test whether request sets on behalf of others are rejected, unless done with Delegation properties
This test is kind of stupid though, since we do the same thing than the server... not a real test !
"""
request = Request({"RequestName": "unauthorized"})
request.OwnerDN = 'NotMe'
request.OwnerDN = 'AnotherGroup'
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
res = self.requestClient.putRequest(request)
credProperties = getProxyInfo()['Value']['groupProperties']
# If the proxy with which we test has delegation, it should work
if FULL_DELEGATION in credProperties or LIMITED_DELEGATION in credProperties:
self.assertTrue(res['OK'], res)
self.requestClient.deleteRequest(res['Value'])
# otherwise no
else:
self.assertFalse(res['OK'], res)
def myRequest():
"""Create a request and put it to the db"""
request = Request()
request.RequestName = 'myAwesomeRemovalRequest.xml'
request.JobID = 0
request.SourceComponent = "myScript"
remove = Operation()
remove.Type = "RemoveFile"
lfn = "/ilc/user/s/sailer/test.txt"
rmFile = File()
rmFile.LFN = lfn
remove.addFile( rmFile )
request.addOperation( remove )
isValid = RequestValidator().validate( request )
if not isValid['OK']:
raise RuntimeError( "Failover request is not valid: %s" % isValid['Message'] )
else:
print("It is a GOGOGO")
requestClient = ReqClient()
result = requestClient.putRequest( request )
print(result)
def __init__(
self,
requestJSON,
handlersDict,
csPath,
agentName,
standalone=False,
requestClient=None):
"""c'tor
:param self: self reference
:param str requestJSON: request serialized to JSON
:param dict opHandlers: operation handlers
"""
self.request = Request(requestJSON)
# # csPath
self.csPath = csPath
# # agent name
self.agentName = agentName
# # standalone flag
self.standalone = standalone
# # handlers dict
self.handlersDict = handlersDict
# # handlers class def
self.handlers = {}
# # own sublogger
self.log = gLogger.getSubLogger("pid_%s/%s" % (os.getpid(), self.request.RequestName))
# # get shifters info
self.__managersDict = {}
shifterProxies = self.__setupManagerProxies()
if not shifterProxies["OK"]:
self.log.error(shifterProxies["Message"])
# # initialize gMonitor
gMonitor.setComponentType(gMonitor.COMPONENT_AGENT)
gMonitor.setComponentName(self.agentName)
gMonitor.initialize()
# # own gMonitor activities
gMonitor.registerActivity("RequestAtt", "Requests processed",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("RequestFail", "Requests failed",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM)
gMonitor.registerActivity("RequestOK", "Requests done",
"RequestExecutingAgent", "Requests/min", gMonitor.OP_SUM)
if requestClient is None:
self.requestClient = ReqClient()
else:
self.requestClient = requestClient
def __init__( self, requestObject = None, log = None, defaultChecksumType = 'ADLER32' ):
""" Constructor function, can specify request object to instantiate
FailoverTransfer or a new request object is created.
"""
self.log = log
if not self.log:
self.log = gLogger.getSubLogger( "FailoverTransfer" )
self.request = requestObject
if not self.request:
self.request = Request()
self.request.RequestName = 'noname_request'
self.request.SourceComponent = 'FailoverTransfer'
self.defaultChecksumType = defaultChecksumType
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
