def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def export_putRequest(self, requestJSON):
""" put a new request into RequestDB """
requestDict = json.loads(requestJSON)
request = Request(requestDict)
operation = Operation() # # create new operation
operation.Type = "WMSSecureOutputData"
request.insertBefore(operation, request[0])
userDN, userGroup, _ = self.__getOwnerGroupDN('ProductionManager')
request.OwnerDN = userDN
request.OwnerGroup = userGroup
return ReqClient().putRequest(request)
def export_putRequest(self, requestJSON):
""" put a new request into RequestDB """
requestDict = json.loads(requestJSON)
requestName = requestDict.get("RequestID",
requestDict.get('RequestName',
"***UNKNOWN***")) #pylint: disable=unused-variable
request = Request(requestDict)
operation = Operation() # # create new operation
operation.Type = "WMSSecureOutputData"
request.insertBefore(operation, request[0])
userDN, userGroup, userName = self.__getOwnerGroupDN(
'ProductionManager') #pylint: disable=unused-variable
request.OwnerDN = userDN
request.OwnerGroup = userGroup
return ReqClient().putRequest(request)
def __deleteSandboxFromExternalBackend(self, SEName, SEPFN):
if self.getCSOption("DelayedExternalDeletion", True):
gLogger.info("Setting deletion request")
try:
# We need the hostDN used in order to pass these credentials to the
# SandboxStoreDB..
hostCertLocation, _ = Locations.getHostCertificateAndKeyLocation(
)
hostCert = X509Certificate.X509Certificate()
hostCert.loadFromFile(hostCertLocation)
hostDN = hostCert.getSubjectDN().get("Value")
# use the host authentication to fetch the data
result = self.sandboxDB.getSandboxOwner(
SEName, SEPFN, hostDN, "hosts")
if not result["OK"]:
return result
_owner, ownerDN, ownerGroup = result["Value"]
request = Request()
request.RequestName = "RemoteSBDeletion:%s|%s:%s" % (
SEName, SEPFN, time.time())
request.OwnerDN = ownerDN
request.OwnerGroup = ownerGroup
physicalRemoval = Operation()
physicalRemoval.Type = "PhysicalRemoval"
physicalRemoval.TargetSE = SEName
fileToRemove = File()
fileToRemove.PFN = SEPFN
physicalRemoval.addFile(fileToRemove)
request.addOperation(physicalRemoval)
return ReqClient().putRequest(request)
except Exception as e:
gLogger.exception("Exception while setting deletion request")
return S_ERROR(f"Cannot set deletion request: {e}")
else:
gLogger.info("Deleting external Sandbox")
try:
return StorageElement(SEName).removeFile(SEPFN)
except Exception:
gLogger.exception(
"RM raised an exception while trying to delete a remote sandbox"
)
return S_ERROR(
"RM raised an exception while trying to delete a remote sandbox"
)
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
""" Set removal request with the given credentials
"""
oRequest = Request()
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml'
oRequest.SourceComponent = 'JobCleaningAgent'
removeFile = Operation()
removeFile.Type = 'RemoveFile'
removedFile = File()
removedFile.LFN = lfn
removeFile.addFile( removedFile )
oRequest.addOperation( removeFile )
return ReqClient().putRequest( oRequest )
def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
""" Set removal request with the given credentials
"""
oRequest = Request()
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
oRequest.RequestName = os.path.basename( lfn ).strip() + '_removal_request.xml'
oRequest.SourceComponent = 'JobCleaningAgent'
removeFile = Operation()
removeFile.Type = 'RemoveFile'
removedFile = File()
removedFile.LFN = lfn
removeFile.addFile( removedFile )
oRequest.addOperation( removeFile )
return ReqClient().putRequest( oRequest )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
if paramDict['InputData']:
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
if type( paramDict['InputData'] ) == type( [] ):
files = paramDict['InputData']
elif type( paramDict['InputData'] ) == type( '' ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = str( transID ).zfill( 8 ) + '_' + str( taskID ).zfill( 8 )
oRequest.OwnerDN = owner
oRequest.OwnerGroup = ownerGroup
isValid = gRequestValidator.validate( oRequest )
if not isValid['OK']:
return isValid
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
def __setRemovalRequest(self, lfn, ownerDN, ownerGroup):
"""Set removal request with the given credentials"""
oRequest = Request()
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
oRequest.RequestName = os.path.basename(
lfn).strip() + "_removal_request.xml"
oRequest.SourceComponent = "JobCleaningAgent"
removeFile = Operation()
removeFile.Type = "RemoveFile"
removedFile = File()
removedFile.LFN = lfn
removeFile.addFile(removedFile)
oRequest.addOperation(removeFile)
# put the request with the owner certificate to make sure it's still a valid DN
return ReqClient(useCertificates=True,
delegatedDN=ownerDN,
delegatedGroup=ownerGroup).putRequest(oRequest)
res = self.requestClient.getRequestNamesForJobs( [123] )
self.assert_( res['OK'] )
self.assertEqual( res['Value'], {'Successful': {123L:self.request.RequestName}, 'Failed': {}} )
res = self.requestClient.getRequestNamesList()
self.assert_( res['OK'] )
res = self.requestClient.readRequestsForJobs( [123] )
self.assert_( res['OK'] )
self.assert_( isinstance( res['Value']['Successful'][123], Request ) )
# Adding new request
request2 = Request()
request2.RequestName = "RequestManagerHandlerTests-2"
request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
request2.OwnerGroup = "dirac_user"
request2.JobID = 456
request2.addOperation( self.operation )
# # update
res = self.requestClient.putRequest( request2 )
self.assert_( res['OK'] )
# # get summary again
ret = RequestDB().getDBSummary()
self.assertEqual( ret,
{ 'OK': True,
'Value': { 'Operation': { 'ReplicateAndRegister': {'Waiting': 2L } },
'Request': { 'Waiting': 1L, 'Assigned': 1L },
'File': { 'Waiting': 4L} } } )
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if not taskDict:
return S_OK({})
if ( not owner ) or ( not ownerGroup ):
res = getProxyInfo( False, False )
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername( owner )
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split( ';' )
except AttributeError:
pass
# Do not remove sorted, we might pop elements in the loop
for taskID in sorted( taskDict ):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
# If there are input files
if paramDict['InputData']:
if isinstance( paramDict['InputData'], list ):
files = paramDict['InputData']
elif isinstance( paramDict['InputData'], basestring ):
files = paramDict['InputData'].split( ';' )
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile( trFile )
oRequest.addOperation( transfer )
oRequest.RequestName = _requestName( transID, taskID )
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate( oRequest )
if not isValid['OK']:
self.log.error( "Error creating request for task", "%s %s" % ( taskID, isValid ) )
# This works because we loop over a copy of the keys !
taskDict.pop( taskID )
continue
taskDict[taskID]['TaskObject'] = oRequest
return S_OK( taskDict )
},
'Failed': {}
})
res = self.requestClient.getRequestNamesList()
self.assert_(res['OK'])
res = self.requestClient.readRequestsForJobs([123])
self.assert_(res['OK'])
self.assert_(isinstance(res['Value']['Successful'][123], Request))
# Adding new request
request2 = Request()
request2.RequestName = "RequestManagerHandlerTests-2"
request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
request2.OwnerGroup = "dirac_user"
request2.JobID = 456
request2.addOperation(self.operation)
# # update
res = self.requestClient.putRequest(request2)
self.assert_(res['OK'])
# # get summary again
ret = RequestDB().getDBSummary()
self.assertEqual(
ret, {
'OK': True,
'Value': {
'Operation': {
'ReplicateAndRegister': {
def prepareTransformationTasks(self,
transBody,
taskDict,
owner='',
ownerGroup='',
ownerDN=''):
""" Prepare tasks, given a taskDict, that is created (with some manipulation) by the DB
"""
if not taskDict:
return S_OK({})
if (not owner) or (not ownerGroup):
res = getProxyInfo(False, False)
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
if not ownerDN:
res = getDNForUsername(owner)
if not res['OK']:
return res
ownerDN = res['Value'][0]
requestOperation = 'ReplicateAndRegister'
if transBody:
try:
_requestType, requestOperation = transBody.split(';')
except AttributeError:
pass
# Do not remove sorted, we might pop elements in the loop
for taskID in sorted(taskDict):
paramDict = taskDict[taskID]
transID = paramDict['TransformationID']
oRequest = Request()
transfer = Operation()
transfer.Type = requestOperation
transfer.TargetSE = paramDict['TargetSE']
# If there are input files
if paramDict['InputData']:
if isinstance(paramDict['InputData'], list):
files = paramDict['InputData']
elif isinstance(paramDict['InputData'], basestring):
files = paramDict['InputData'].split(';')
for lfn in files:
trFile = File()
trFile.LFN = lfn
transfer.addFile(trFile)
oRequest.addOperation(transfer)
oRequest.RequestName = _requestName(transID, taskID)
oRequest.OwnerDN = ownerDN
oRequest.OwnerGroup = ownerGroup
isValid = self.requestValidator.validate(oRequest)
if not isValid['OK']:
self.log.error("Error creating request for task",
"%s %s" % (taskID, isValid))
# This works because we loop over a copy of the keys !
taskDict.pop(taskID)
continue
taskDict[taskID]['TaskObject'] = oRequest
return S_OK(taskDict)
def test01fullChain( self ):
put = self.requestClient.putRequest( self.request )
self.assert_( put['OK'] )
self.assertEqual( type( put['Value'] ), long )
reqID = put['Value']
# # summary
ret = RequestDB().getDBSummary()
self.assertEqual( ret,
{ 'OK': True,
'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } },
'Request': { 'Waiting': 1L },
'File': { 'Waiting': 2L} } } )
get = self.requestClient.getRequest( reqID )
self.assert_( get['OK'] )
self.assertEqual( isinstance( get['Value'], Request ), True )
# # summary - the request became "Assigned"
res = RequestDB().getDBSummary()
self.assertEqual( res,
{ 'OK': True,
'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } },
'Request': { 'Assigned': 1L },
'File': { 'Waiting': 2L} } } )
res = self.requestClient.getRequestInfo( reqID )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.getRequestFileStatus( reqID, self.file.LFN )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.getRequestFileStatus( reqID, [self.file.LFN] )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.getDigest( reqID )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.readRequestsForJobs( [123] )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
self.assert_( isinstance( res['Value']['Successful'][123], Request ) )
# Adding new request
request2 = Request()
request2.RequestName = "RequestManagerHandlerTests-2"
request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
request2.OwnerGroup = "dirac_user"
request2.JobID = 456
request2.addOperation( self.operation )
# # update
res = self.requestClient.putRequest( request2 )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
reqID2 = res['Value']
# # get summary again
ret = RequestDB().getDBSummary()
self.assertEqual( ret,
{ 'OK': True,
'Value': { 'Operation': { 'ReplicateAndRegister': {'Waiting': 2L } },
'Request': { 'Waiting': 1L, 'Assigned': 1L },
'File': { 'Waiting': 4L} } } )
def test01fullChain( self ):
put = self.requestClient.putRequest( self.request )
self.assert_( put['OK'] )
self.assertEqual( type( put['Value'] ), long )
reqID = put['Value']
# # summary
ret = RequestDB().getDBSummary()
self.assertEqual( ret,
{ 'OK': True,
'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } },
'Request': { 'Waiting': 1L },
'File': { 'Waiting': 2L} } } )
get = self.requestClient.getRequest( reqID )
self.assert_( get['OK'] )
self.assertEqual( isinstance( get['Value'], Request ), True )
# # summary - the request became "Assigned"
res = RequestDB().getDBSummary()
self.assertEqual( res,
{ 'OK': True,
'Value': { 'Operation': { 'ReplicateAndRegister': { 'Waiting': 1L } },
'Request': { 'Assigned': 1L },
'File': { 'Waiting': 2L} } } )
res = self.requestClient.getRequestInfo( reqID )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.getRequestFileStatus( reqID, self.file.LFN )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.getRequestFileStatus( reqID, [self.file.LFN] )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.getDigest( reqID )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
res = self.requestClient.readRequestsForJobs( [123] )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
self.assert_( isinstance( res['Value']['Successful'][123], Request ) )
# Adding new request
request2 = Request()
request2.RequestName = "RequestManagerHandlerTests-2"
request2.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
request2.OwnerGroup = "dirac_user"
request2.JobID = 456
request2.addOperation( self.operation )
# # update
res = self.requestClient.putRequest( request2 )
self.assertEqual( res['OK'], True, res['Message'] if 'Message' in res else 'OK' )
reqID2 = res['Value']
# # get summary again
ret = RequestDB().getDBSummary()
self.assertEqual( ret,
{ 'OK': True,
'Value': { 'Operation': { 'ReplicateAndRegister': {'Waiting': 2L } },
'Request': { 'Waiting': 1L, 'Assigned': 1L },
'File': { 'Waiting': 4L} } } )
jr_mock = MagicMock()
jr_mock.setApplicationStatus.return_value = {'OK': True, 'Value': ''}
jr_mock.generateForwardDISET.return_value = {'OK': True, 'Value': Operation()}
jr_mock.setJobParameter.return_value = {'OK': True, 'Value': 'pippo'}
fr_mock = MagicMock()
fr_mock.getFiles.return_value = {}
fr_mock.setFileStatus.return_value = {'OK': True, 'Value': ''}
fr_mock.commit.return_value = {'OK': True, 'Value': ''}
fr_mock.generateForwardDISET.return_value = {'OK': True, 'Value': Operation()}
rc_mock = Request()
rc_mock.RequestName = 'aRequestName'
rc_mock.OwnerDN = 'pippo'
rc_mock.OwnerGroup = 'pippoGRP'
rOp = Operation()
rOp.Type = 'PutAndRegister'
rOp.TargetSE = 'anSE'
f = File()
f.LFN = '/foo/bar.py'
f.PFN = '/foo/bar.py'
rOp.addFile(f)
rc_mock.addOperation(rOp)
wf_commons = [{'PRODUCTION_ID': prod_id, 'JOB_ID': prod_job_id, 'eventType': '123456789', 'jobType': 'merge',
'configName': 'aConfigName', 'configVersion': 'aConfigVersion', 'outputDataFileMask': '',
'BookkeepingLFNs': 'aa', 'ProductionOutputData': 'ProductionOutputData', 'numberOfEvents': '100',
'JobReport': jr_mock, 'Request': rc_mock, 'AccountingReport': ar_mock, 'FileReport': fr_mock,
'SystemConfig': 'sys_config', 'runNumber': 'Unknown', 'gaudiSteps': ['someApp_1'],
