def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def export_putRequest(self, requestJSON):
""" put a new request into RequestDB """
requestDict = json.loads(requestJSON)
request = Request(requestDict)
operation = Operation() # # create new operation
operation.Type = "WMSSecureOutputData"
request.insertBefore(operation, request[0])
userDN, userGroup, _ = self.__getOwnerGroupDN('ProductionManager')
request.OwnerDN = userDN
request.OwnerGroup = userGroup
return ReqClient().putRequest(request)
def test07Authorization(self):
""" Test whether request sets on behalf of others are rejected"""
request = Request({"RequestName": "unauthorized"})
request.OwnerDN = 'NotMe'
request.OwnerDN = 'AnotherGroup'
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
res = self.requestClient.putRequest(request)
self.assertFalse(res['OK'], res)
def export_putRequest(self, requestJSON):
""" put a new request into RequestDB """
requestDict = json.loads(requestJSON)
requestName = requestDict.get("RequestID",
requestDict.get('RequestName',
"***UNKNOWN***")) #pylint: disable=unused-variable
request = Request(requestDict)
operation = Operation() # # create new operation
operation.Type = "WMSSecureOutputData"
request.insertBefore(operation, request[0])
userDN, userGroup, userName = self.__getOwnerGroupDN(
'ProductionManager') #pylint: disable=unused-variable
request.OwnerDN = userDN
request.OwnerGroup = userGroup
return ReqClient().putRequest(request)
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
if self.getCSOption("DelayedExternalDeletion", True):
gLogger.info("Setting deletion request")
try:
# We need the hostDN used in order to pass these credentials to the
# SandboxStoreDB..
hostCertLocation, _ = Locations.getHostCertificateAndKeyLocation(
)
hostCert = X509Certificate.X509Certificate()
hostCert.loadFromFile(hostCertLocation)
hostDN = hostCert.getSubjectDN().get("Value")
# use the host authentication to fetch the data
result = self.sandboxDB.getSandboxOwner(
SEName, SEPFN, hostDN, "hosts")
if not result["OK"]:
return result
_owner, ownerDN, ownerGroup = result["Value"]
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % (
SEName, SEPFN, time.time())
request.OwnerDN = ownerDN
request.OwnerGroup = ownerGroup
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile(fileToRemove)
request.addOperation(physicalRemoval)
return ReqClient().putRequest(request)
except Exception as e:
gLogger.exception("Exception while setting deletion request")
return S_ERROR(f"Cannot set deletion request: {e}")
else:
gLogger.info("Deleting external Sandbox")
try:
return StorageElement(SEName).removeFile(SEPFN)
except Exception:
gLogger.exception(
"RM raised an exception while trying to delete a remote sandbox"
)
return S_ERROR(
"RM raised an exception while trying to delete a remote sandbox"
)
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
""" Set removal request with the given credentials
"""
oRequest = Request()
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml'
oRequest.SourceComponent = 'JobCleaningAgent'
removeFile = Operation()
removeFile.Type = 'RemoveFile'
removedFile = File()
removedFile.LFN = lfn
removeFile.addFile( removedFile )
oRequest.addOperation( removeFile )
return ReqClient().putRequest( oRequest )
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
""" Set removal request with the given credentials
"""
oRequest = Request()
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml'
oRequest.SourceComponent = 'JobCleaningAgent'
removeFile = Operation()
removeFile.Type = 'RemoveFile'
removedFile = File()
removedFile.LFN = lfn
removeFile.addFile( removedFile )
oRequest.addOperation( removeFile )
return ReqClient().putRequest( oRequest )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if type( paramDict['InputData'] ) == type( [] ):
files = paramDict['InputData']
elif type( paramDict['InputData'] ) == type( '' ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
oRequest.OwnerDN = owner
oRequest.OwnerGroup = ownerGroup
isValid = gRequestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def test07Authorization(self):
""" Test whether request sets on behalf of others are rejected, unless done with Delegation properties
This test is kind of stupid though, since we do the same thing than the server... not a real test !
"""
request = Request({"RequestName": "unauthorized"})
request.OwnerDN = 'NotMe'
request.OwnerDN = 'AnotherGroup'
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
res = self.requestClient.putRequest(request)
credProperties = getProxyInfo()['Value']['groupProperties']
# If the proxy with which we test has delegation, it should work
if FULL_DELEGATION in credProperties or LIMITED_DELEGATION in credProperties:
self.assertTrue(res['OK'], res)
self.requestClient.deleteRequest(res['Value'])
# otherwise no
else:
self.assertFalse(res['OK'], res)
def test02Authorization(self):
""" Test whether request sets on behalf of others are rejected, unless done with Delegation properties
This test is kind of stupid though, since we do the same thing than the server... not a real test !
"""
request = Request({"RequestName": "unauthorized"})
request.OwnerDN = 'NotMe'
request.OwnerDN = 'AnotherGroup'
op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"})
op += File({"LFN": "/lhcb/user/c/cibak/foo"})
request += op
res = self.requestClient.putRequest(request)
credProperties = getProxyInfo()['Value']['groupProperties']
# If the proxy with which we test has delegation, it should work
if FULL_DELEGATION in credProperties or LIMITED_DELEGATION in credProperties:
self.assertTrue(res['OK'], res)
self.requestClient.deleteRequest(res['Value'])
# otherwise no
else:
self.assertFalse(res['OK'], res)
def __setRemovalRequest(self, lfn, ownerDN, ownerGroup):
"""Set removal request with the given credentials"""
oRequest = Request()
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
oRequest.RequestName = os.path.basename(
lfn).strip() + "_removal_request.xml"
oRequest.SourceComponent = "JobCleaningAgent"
removeFile = Operation()
removeFile.Type = "RemoveFile"
removedFile = File()
removedFile.LFN = lfn
removeFile.addFile(removedFile)
oRequest.addOperation(removeFile)
# put the request with the owner certificate to make sure it's still a valid DN
return ReqClient(useCertificates=True,
delegatedDN=ownerDN,
delegatedGroup=ownerGroup).putRequest(oRequest)
ar_mock.commit.return_value = {'OK': True, 'Value': ''}
jr_mock = MagicMock()
jr_mock.setApplicationStatus.return_value = {'OK': True, 'Value': ''}
jr_mock.generateForwardDISET.return_value = {'OK': True, 'Value': Operation()}
jr_mock.setJobParameter.return_value = {'OK': True, 'Value': 'pippo'}
fr_mock = MagicMock()
fr_mock.getFiles.return_value = {}
fr_mock.setFileStatus.return_value = {'OK': True, 'Value': ''}
fr_mock.commit.return_value = {'OK': True, 'Value': ''}
fr_mock.generateForwardDISET.return_value = {'OK': True, 'Value': Operation()}
rc_mock = Request()
rc_mock.RequestName = 'aRequestName'
rc_mock.OwnerDN = 'pippo'
rc_mock.OwnerGroup = 'pippoGRP'
rOp = Operation()
rOp.Type = 'PutAndRegister'
rOp.TargetSE = 'anSE'
f = File()
f.LFN = '/foo/bar.py'
f.PFN = '/foo/bar.py'
rOp.addFile(f)
rc_mock.addOperation(rOp)
wf_commons = [{'PRODUCTION_ID': prod_id, 'JOB_ID': prod_job_id, 'eventType': '123456789', 'jobType': 'merge',
'configName': 'aConfigName', 'configVersion': 'aConfigVersion', 'outputDataFileMask': '',
'BookkeepingLFNs': 'aa', 'ProductionOutputData': 'ProductionOutputData', 'numberOfEvents': '100',
'JobReport': jr_mock, 'Request': rc_mock, 'AccountingReport': ar_mock, 'FileReport': fr_mock,
def test01fullChain( self ):
put = self.requestClient.putRequest( self.request )
self.assert_( put['OK'] )
self.assertEqual( type( put['Value'] ), long )
reqID = put['Value']
# # summary
ret = RequestDB().getDBSummary()
self.assertEqual( ret,
{ 'OK': True,
'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } },
'Request': { 'Waiting': 1L },
'File': { 'Waiting': 2L} } } )
get = self.requestClient.getRequest( reqID )
self.assert_( get['OK'] )
self.assertEqual( isinstance( get['Value'], Request ), True )
# # summary - the request became "Assigned"
res = RequestDB().getDBSummary()
self.assertEqual( res,
{ 'OK': True,
'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } },
'Request': { 'Assigned': 1L },
'File': { 'Waiting': 2L} } } )
res = self.requestClient.getRequestInfo( reqID )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.getRequestFileStatus( reqID, self.file.LFN )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.getRequestFileStatus( reqID, [self.file.LFN] )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.getDigest( reqID )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.readRequestsForJobs( [123] )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
self.assert_( isinstance( res['Value']['Successful'][123], Request ) )
# Adding new request
request2 = Request()
request2.RequestName = "RequestManagerHandlerTests-2"
request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
request2.OwnerGroup = "dirac_user"
request2.JobID = 456
request2.addOperation( self.operation )
# # update
res = self.requestClient.putRequest( request2 )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
reqID2 = res['Value']
# # get summary again
ret = RequestDB().getDBSummary()
self.assertEqual( ret,
{ 'OK': True,
'Value': { 'Operation': { 'ReplicateAndRegister': {'Waiting': 2L } },
'Request': { 'Waiting': 1L, 'Assigned': 1L },
'File': { 'Waiting': 4L} } } )
res = self.requestClient.getRequestNamesForJobs( [123] )
self.assert_( res['OK'] )
self.assertEqual( res['Value'], {'Successful': {123L:self.request.RequestName}, 'Failed': {}} )
res = self.requestClient.getRequestNamesList()
self.assert_( res['OK'] )
res = self.requestClient.readRequestsForJobs( [123] )
self.assert_( res['OK'] )
self.assert_( isinstance( res['Value']['Successful'][123], Request ) )
# Adding new request
request2 = Request()
request2.RequestName = "RequestManagerHandlerTests-2"
request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
request2.OwnerGroup = "dirac_user"
request2.JobID = 456
request2.addOperation( self.operation )
# # update
res = self.requestClient.putRequest( request2 )
self.assert_( res['OK'] )
# # get summary again
ret = RequestDB().getDBSummary()
self.assertEqual( ret,
{ 'OK': True,
'Value': { 'Operation': { 'ReplicateAndRegister': {'Waiting': 2L } },
'Request': { 'Waiting': 1L, 'Assigned': 1L },
'File': { 'Waiting': 4L} } } )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if not taskDict:
return S_OK({})
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
# Do not remove sorted, we might pop elements in the loop
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
# If there are input files
if paramDict['InputData']:
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
self.log.error( "Error creating request for task", "%s %s" % ( taskID, isValid ) )
# This works because we loop over a copy of the keys !
taskDict.pop( taskID )
continue
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def test01fullChain( self ):
put = self.requestClient.putRequest( self.request )
self.assert_( put['OK'] )
self.assertEqual( type( put['Value'] ), long )
reqID = put['Value']
# # summary
ret = RequestDB().getDBSummary()
self.assertEqual( ret,
{ 'OK': True,
'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } },
'Request': { 'Waiting': 1L },
'File': { 'Waiting': 2L} } } )
get = self.requestClient.getRequest( reqID )
self.assert_( get['OK'] )
self.assertEqual( isinstance( get['Value'], Request ), True )
# # summary - the request became "Assigned"
res = RequestDB().getDBSummary()
self.assertEqual( res,
{ 'OK': True,
'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } },
'Request': { 'Assigned': 1L },
'File': { 'Waiting': 2L} } } )
res = self.requestClient.getRequestInfo( reqID )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.getRequestFileStatus( reqID, self.file.LFN )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.getRequestFileStatus( reqID, [self.file.LFN] )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.getDigest( reqID )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.readRequestsForJobs( [123] )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
self.assert_( isinstance( res['Value']['Successful'][123], Request ) )
# Adding new request
request2 = Request()
request2.RequestName = "RequestManagerHandlerTests-2"
request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
request2.OwnerGroup = "dirac_user"
request2.JobID = 456
request2.addOperation( self.operation )
# # update
res = self.requestClient.putRequest( request2 )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
reqID2 = res['Value']
# # get summary again
ret = RequestDB().getDBSummary()
self.assertEqual( ret,
{ 'OK': True,
'Value': { 'Operation': { 'ReplicateAndRegister': {'Waiting': 2L } },
'Request': { 'Waiting': 1L, 'Assigned': 1L },
'File': { 'Waiting': 4L} } } )
123L: self.request.RequestName
},
'Failed': {}
})
res = self.requestClient.getRequestNamesList()
self.assert_(res['OK'])
res = self.requestClient.readRequestsForJobs([123])
self.assert_(res['OK'])
self.assert_(isinstance(res['Value']['Successful'][123], Request))
# Adding new request
request2 = Request()
request2.RequestName = "RequestManagerHandlerTests-2"
request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
request2.OwnerGroup = "dirac_user"
request2.JobID = 456
request2.addOperation(self.operation)
# # update
res = self.requestClient.putRequest(request2)
self.assert_(res['OK'])
# # get summary again
ret = RequestDB().getDBSummary()
self.assertEqual(
ret, {
'OK': True,
'Value': {
'Operation': {
def prepareTransformationTasks(self,
transBody,
taskDict,
owner='',
ownerGroup='',
ownerDN=''):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if not taskDict:
return S_OK({})
if (not owner) or (not ownerGroup):
res = getProxyInfo(False, False)
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername(owner)
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split(';')
except AttributeError:
pass
# Do not remove sorted, we might pop elements in the loop
for taskID in sorted(taskDict):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
# If there are input files
if paramDict['InputData']:
if isinstance(paramDict['InputData'], list):
files = paramDict['InputData']
elif isinstance(paramDict['InputData'], basestring):
files = paramDict['InputData'].split(';')
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
oRequest.RequestName = _requestName(transID, taskID)
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate(oRequest)
if not isValid['OK']:
self.log.error("Error creating request for task",
"%s %s" % (taskID, isValid))
# This works because we loop over a copy of the keys !
taskDict.pop(taskID)
continue
taskDict[taskID]['TaskObject'] = oRequest
return S_OK(taskDict)