def web_getSandbox( self ):
if 'jobID' not in self.request.arguments:
self.finish( {"success":"false", "error":"Maybe you forgot the jobID ?"} )
return
jobID = int( self.request.arguments['jobID'][0] )
sbType = 'Output'
if 'sandbox' in self.request.arguments:
sbType = str( self.request.arguments['sandbox'][0] )
userData = self.getSessionData()
client = SandboxStoreClient( useCertificates = True,
delegatedDN = str( userData["user"]["DN"] ),
delegatedGroup = str( userData["user"]["group"] ),
setup = userData["setup"] )
result = yield self.threadTask( client.downloadSandboxForJob, jobID, sbType, inMemory = True )
if not result['OK']:
self.finish( {"success":"false", "error":"Error: %s" % result['Message']} )
return
if "check" in self.request.arguments:
self.finish( {"success":"true"} )
return
data = result['Value']
fname = "%s_%sSandbox.tar" % ( str( jobID ), sbType )
self.set_header( 'Content-type', 'application/x-tar' )
self.set_header( 'Content-Disposition', 'attachment; filename="%s"' % fname )
self.set_header( 'Content-Length', len( data ) )
self.set_header( 'Cache-Control', "no-cache, no-store, must-revalidate, max-age=0" )
self.set_header( 'Pragma', "no-cache" )
self.finish( data )
def initializeOptimizer( self ):
"""Initialize specific parameters for JobSanityAgent.
"""
#Test control flags N.B. JDL check is mandatory
self.inputDataCheck = self.am_getOption( 'InputDataCheck', 1 )
self.outputDataCheck = self.am_getOption( 'OutputDataCheck', 0 )
self.inputSandboxCheck = self.am_getOption( 'InputSandboxCheck', 1 )
self.platformCheck = self.am_getOption( 'PlatformCheck', 0 )
#Other parameters
self.successStatus = self.am_getOption( 'SuccessfulJobStatus', 'OutputReady' )
self.maxDataPerJob = self.am_getOption( 'MaxInputDataPerJob', 100 )
#Sandbox
self.sandboxClient = SandboxStoreClient( useCertificates = True )
self.log.debug( 'JDL Check ==> Enabled' )
if self.inputDataCheck:
self.log.debug( 'Input Data Check ==> Enabled' )
else:
self.log.debug( 'Input Data Check ==> Disabled' )
if self.outputDataCheck:
self.log.debug( 'Output Data Check ==> Enabled' )
else:
self.log.debug( 'Output Data Check ==> Disabled' )
if self.inputSandboxCheck:
self.log.debug( 'Input Sbox Check ==> Enabled' )
else:
self.log.debug( 'Input Sbox Check ==> Disabled' )
if self.platformCheck:
self.log.debug( 'Platform Check ==> Enabled' )
else:
self.log.debug( 'Platform Check ==> Disabled' )
return S_OK()
def getSandbox(self):
""" Get job sandbox
"""
if 'jobID' not in request.params:
c.error = "Maybe you forgot the jobID ?"
return render("/error.mako")
jobID = int(request.params['jobID'])
sbType = 'Output'
if 'sandbox' in request.params:
sbType = str(request.params['sandbox'])
client = SandboxStoreClient(useCertificates=True,
delegatedDN=str(credentials.getUserDN()),
delegatedGroup=str(
credentials.getSelectedGroup()),
setup=credentials.getSelectedSetup())
result = client.downloadSandboxForJob(jobID, sbType, inMemory=True)
if not result['OK']:
c.error = "Error: %s" % result['Message']
return render("/error.mako")
data = result['Value']
fname = "%s_%sSandbox.tar" % (str(jobID), sbType)
response.headers['Content-type'] = 'application/x-tar'
response.headers[
'Content-Disposition'] = 'attachment; filename="%s"' % fname
response.headers['Content-Length'] = len(data)
return data
def __init__(
self,
jobManagerClient=None,
sbRPCClient=None,
sbTransferClient=None,
useCertificates=False,
timeout=600,
delegatedDN=None,
delegatedGroup=None,
):
"""WMS Client constructor
Here we also initialize the needed clients and connections
"""
self.useCertificates = useCertificates
self.delegatedDN = delegatedDN
self.delegatedGroup = delegatedGroup
self.timeout = timeout
self._jobManager = jobManagerClient
self.operationsHelper = Operations()
self.sandboxClient = None
if sbRPCClient and sbTransferClient:
self.sandboxClient = SandboxStoreClient(
rpcClient=sbRPCClient, transferClient=sbTransferClient, useCertificates=useCertificates
)
def __uploadInputSandbox(self, classAdJob, jobDescriptionObject=None):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries(classAdJob)
realFiles = []
badFiles = []
diskFiles = []
for isFile in inputSandbox:
if not isFile.startswith(("lfn:", "LFN:", "SB:", "%s", "%(")):
realFiles.append(isFile)
stringIOFiles = []
stringIOFilesSize = 0
if jobDescriptionObject is not None:
if isinstance(jobDescriptionObject, StringIO):
stringIOFiles = [jobDescriptionObject]
stringIOFilesSize = len(jobDescriptionObject.getvalue())
gLogger.debug("Size of the stringIOFiles: " + str(stringIOFilesSize))
else:
return S_ERROR(EWMSJDL, "jobDescriptionObject is not a StringIO object")
# Check real files
for isFile in realFiles:
if not os.path.exists(isFile): # we are passing in real files, we expect them to be on disk
badFiles.append(isFile)
gLogger.warn("inputSandbox file/directory " + isFile + " not found. Keep looking for the others")
continue
diskFiles.append(isFile)
diskFilesSize = File.getGlobbedTotalSize(diskFiles)
gLogger.debug("Size of the diskFiles: " + str(diskFilesSize))
totalSize = diskFilesSize + stringIOFilesSize
gLogger.verbose("Total size of the inputSandbox: " + str(totalSize))
okFiles = stringIOFiles + diskFiles
if badFiles:
result = S_ERROR(EWMSJDL, "Input Sandbox is not valid")
result["BadFile"] = badFiles
result["TotalSize"] = totalSize
return result
if okFiles:
if not self.sandboxClient:
self.sandboxClient = SandboxStoreClient(
useCertificates=self.useCertificates,
delegatedDN=self.delegatedDN,
delegatedGroup=self.delegatedGroup,
)
result = self.sandboxClient.uploadFilesAsSandbox(okFiles)
if not result["OK"]:
return result
inputSandbox.append(result["Value"])
classAdJob.insertAttributeVectorString("InputSandbox", inputSandbox)
return S_OK()
def test_uploadFilesAsSandbox(self):
ourSSC = importlib.import_module('DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient')
ourSSC.TransferClient = MagicMock()
ssc = SandboxStoreClient()
fileList = [StringIO.StringIO('try')]
res = ssc.uploadFilesAsSandbox(fileList)
print(res)
def test_uploadFilesAsSandbox(mocker, setUp):
mocker.patch(
"DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient.TransferClient",
return_value=MagicMock())
ssc = SandboxStoreClient()
fileList = [BytesIO(b"try")]
res = ssc.uploadFilesAsSandbox(fileList)
print(res)
def __uploadInputSandbox(self, classAdJob):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
sandboxClient = SandboxStoreClient(
useCertificates=self.useCertificates,
rpcClient=self.sbRPCClient,
transferClient=self.sbTransferClient)
inputSandbox = self.__getInputSandboxEntries(classAdJob)
realFiles = []
badFiles = []
okFiles = []
realFiles = []
for file in inputSandbox:
valid = True
for tag in (
'lfn:', 'LFN:', 'SB:', '%s'
): #in case of parametric input sandbox, there is %s passed, so have to ignore it also
if file.find(tag) == 0:
valid = False
break
if valid:
realFiles.append(file)
#If there are no files, skip!
if not realFiles:
return S_OK()
#Check real files
for file in realFiles:
if not os.path.exists(file):
badFiles.append(file)
print "inputSandbox file/directory " + file + " not found"
continue
okFiles.append(file)
#print "Total size of the inputSandbox: "+str(totalSize)
totalSize = File.getGlobbedTotalSize(okFiles)
if badFiles:
result = S_ERROR('Input Sandbox is not valid')
result['BadFile'] = badFiles
result['TotalSize'] = totalSize
return result
if okFiles:
result = sandboxClient.uploadFilesAsSandbox(okFiles)
if not result['OK']:
return result
inputSandbox.append(result['Value'])
classAdJob.insertAttributeVectorString("InputSandbox",
inputSandbox)
return S_OK()
def __assignSandboxesToJob(self, jobID, classAdJob):
sandboxClient = SandboxStoreClient()
inputSandboxes = self.__getInputSandboxEntries(classAdJob)
sbToAssign = []
for isb in inputSandboxes:
if isb.find("SB:") == 0:
sbToAssign.append(isb)
if sbToAssign:
assignList = [(isb, 'Input') for isb in sbToAssign]
result = sandboxClient.assignSandboxesToJob(jobID, assignList)
if not result['OK']:
return result
return S_OK()
def _getJobSB( self, jid, objName ):
with TmpDir() as tmpDir:
if objName == "outputsandbox":
objName = "Output"
else:
objName = "Input"
result = SandboxStoreClient().downloadSandboxForJob( int( jid ), objName, tmpDir, inMemory = True )
if not result[ 'OK' ]:
msg = result[ 'Message' ]
if msg.find( "No %s sandbox" % objName ) == 0:
return WErr( 404, "No %s sandbox defined for job %s" % ( jid, objName.lower() ) )
return WErr( 500, result[ 'Message' ] )
return WOK( result[ 'Value' ] )
def __init__(self,
jobManagerClient=False,
sbRPCClient=False,
sbTransferClient=False,
useCertificates=False,
timeout=120):
""" WMS Client constructor
"""
self.jobManagerClient = jobManagerClient
self.useCertificates = useCertificates
self.timeout = timeout
self.sandboxClient = SandboxStoreClient(
useCertificates=useCertificates,
rpcClient=sbRPCClient,
transferClient=sbTransferClient)
def uploadSandbox(self, fileData):
with TmpDir() as tmpDir:
fileList = []
for fName in fileData:
for entry in fileData[fName]:
tmpFile = os.path.join(tmpDir, entry.filename)
if tmpFile not in fileList:
fileList.append(tmpFile)
dfd = open(tmpFile, "w")
dfd.write(entry.body)
dfd.close()
sbClient = SandboxStoreClient()
result = sbClient.uploadFilesAsSandbox(fileList)
if not result['OK']:
return WErr(500, result['Message'])
return WOK(result['Value'])
def test_SSCChain( self ):
""" full test of functionalities
"""
ssc = SandboxStoreClient()
smDB = SandboxMetadataDB()
exeScriptLocation = find_all( 'exe-script.py', '.', 'WorkloadManagementSystem' )[0]
fileList = [exeScriptLocation]
res = ssc.uploadFilesAsSandbox( fileList )
self.assert_( res['OK'] )
# SEPFN = res['Value'].split( '|' )[1]
res = ssc.uploadFilesAsSandboxForJob( fileList, 1, 'Input' )
self.assert_( res['OK'] )
# res = ssc.downloadSandboxForJob( 1, 'Input' ) #to run this would need the RSS on
# self.assert_( res['OK'] )
# only ones needing the DB
res = smDB.getUnusedSandboxes()
self.assert_( res['OK'] )
def __init__(self,
jobManagerClient=None,
sbRPCClient=None,
sbTransferClient=None,
useCertificates=False,
timeout=600):
""" WMS Client constructor
Here we also initialize the needed clients and connections
"""
self.useCertificates = useCertificates
self.timeout = timeout
self.jobManager = jobManagerClient
self.sandboxClient = None
if sbRPCClient and sbTransferClient:
self.sandboxClient = SandboxStoreClient(
rpcClient=sbRPCClient,
transferClient=sbTransferClient,
useCertificates=useCertificates)
def test_SSCChain():
"""full test of functionalities"""
ssc = SandboxStoreClient()
smDB = SandboxMetadataDB()
exeScriptLocation = find_all("exe-script.py", "..", "/DIRAC/tests/Integration")[0]
fileList = [exeScriptLocation]
res = ssc.uploadFilesAsSandbox(fileList)
assert res["OK"] is True, res["Message"]
# SEPFN = res['Value'].split( '|' )[1]
res = ssc.uploadFilesAsSandboxForJob(fileList, 1, "Input")
assert res["OK"] is True, res["Message"]
res = ssc.downloadSandboxForJob(1, "Input") # to run this we need the RSS on
print(res) # for debug...
assert res["OK"] is True, res["Message"]
# only ones needing the DB
res = smDB.getUnusedSandboxes()
print(res)
assert res["OK"] is True, res["Message"]
def test_SSCChain(self):
""" full test of functionalities
"""
ssc = SandboxStoreClient()
smDB = SandboxMetadataDB()
exeScriptLocation = find_all('exe-script.py', '..',
'/DIRAC/tests/Integration')[0]
fileList = [exeScriptLocation]
res = ssc.uploadFilesAsSandbox(fileList)
assert res['OK'] is True
# SEPFN = res['Value'].split( '|' )[1]
res = ssc.uploadFilesAsSandboxForJob(fileList, 1, 'Input')
assert res['OK'] is True
res = ssc.downloadSandboxForJob(1,
'Input') # to run this we need the RSS on
print(res) # for debug...
assert res['OK'] is True
# only ones needing the DB
res = smDB.getUnusedSandboxes()
print(res)
assert res['OK'] is True
def removeJobsByStatus( self, condDict, delay = False ):
""" Remove deleted jobs
"""
if delay:
gLogger.verbose( "Removing jobs with %s and older than %s" % ( condDict, delay ) )
result = self.jobDB.selectJobs( condDict, older = delay, limit = self.maxJobsAtOnce )
else:
gLogger.verbose( "Removing jobs with %s " % condDict )
result = self.jobDB.selectJobs( condDict, limit = self.maxJobsAtOnce )
if not result['OK']:
return result
jobList = result['Value']
if len(jobList) > self.maxJobsAtOnce:
jobList = jobList[:self.maxJobsAtOnce]
if not jobList:
return S_OK()
self.log.notice( "Deleting %s jobs for %s" % ( len( jobList ), condDict ) )
count = 0
error_count = 0
result = SandboxStoreClient( useCertificates = True ).unassignJobs( jobList )
if not result[ 'OK' ]:
gLogger.warn( "Cannot unassign jobs to sandboxes", result[ 'Message' ] )
result = self.deleteJobOversizedSandbox( jobList )
if not result[ 'OK' ]:
gLogger.warn( "Cannot schedule removal of oversized sandboxes", result[ 'Message' ] )
return result
failedJobs = result['Value']['Failed']
for job in failedJobs:
jobList.pop( jobList.index( job ) )
# TODO: we should not remove a job if it still has requests in the RequestManager.
# But this logic should go in the client or in the service, and right now no service expose jobDB.removeJobFromDB
if self.jobByJob:
for jobID in jobList:
resultJobDB = self.jobDB.removeJobFromDB( jobID )
resultTQ = self.taskQueueDB.deleteJob( jobID )
resultLogDB = self.jobLoggingDB.deleteJob( jobID )
errorFlag = False
if not resultJobDB['OK']:
gLogger.warn( 'Failed to remove job %d from JobDB' % jobID, result['Message'] )
errorFlag = True
if not resultTQ['OK']:
gLogger.warn( 'Failed to remove job %d from TaskQueueDB' % jobID, result['Message'] )
errorFlag = True
if not resultLogDB['OK']:
gLogger.warn( 'Failed to remove job %d from JobLoggingDB' % jobID, result['Message'] )
errorFlag = True
if errorFlag:
error_count += 1
else:
count += 1
if self.throttlingPeriod:
time.sleep(self.throttlingPeriod)
else:
result = self.jobDB.removeJobFromDB( jobList )
if not result['OK']:
gLogger.error('Failed to delete %d jobs from JobDB' % len(jobList) )
else:
gLogger.info('Deleted %d jobs from JobDB' % len(jobList) )
for jobID in jobList:
resultTQ = self.taskQueueDB.deleteJob( jobID )
if not resultTQ['OK']:
gLogger.warn( 'Failed to remove job %d from TaskQueueDB' % jobID, resultTQ['Message'] )
error_count += 1
else:
count += 1
result = self.jobLoggingDB.deleteJob( jobList )
if not result['OK']:
gLogger.error('Failed to delete %d jobs from JobLoggingDB' % len(jobList) )
else:
gLogger.info('Deleted %d jobs from JobLoggingDB' % len(jobList) )
if count > 0 or error_count > 0 :
gLogger.info( 'Deleted %d jobs from JobDB, %d errors' % ( count, error_count ) )
return S_OK()
def __uploadInputSandbox(self, classAdJob, jobDescriptionObject=None):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries(classAdJob)
realFiles = []
badFiles = []
diskFiles = []
for isFile in inputSandbox:
valid = True
for tag in (
'lfn:', 'LFN:', 'SB:', '%s'
): # in case of parametric input sandbox, there is %s passed, so have to ignore it also
if isFile.find(tag) == 0:
valid = False
break
if valid:
realFiles.append(isFile)
stringIOFiles = []
stringIOFilesSize = 0
if jobDescriptionObject is not None:
if isinstance(jobDescriptionObject, StringIO.StringIO):
stringIOFiles = [jobDescriptionObject]
stringIOFilesSize = len(jobDescriptionObject.buf)
gLogger.debug("Size of the stringIOFiles: " +
str(stringIOFilesSize))
else:
return S_ERROR("jobDescriptionObject is not a StringIO object")
# Check real files
for isFile in realFiles:
if not os.path.exists(
isFile
): # we are passing in real files, we expect them to be on disk
badFiles.append(isFile)
gLogger.warn("inputSandbox file/directory " + isFile +
" not found. Keep looking for the others")
continue
diskFiles.append(isFile)
diskFilesSize = File.getGlobbedTotalSize(diskFiles)
gLogger.debug("Size of the diskFiles: " + str(diskFilesSize))
totalSize = diskFilesSize + stringIOFilesSize
gLogger.verbose("Total size of the inputSandbox: " + str(totalSize))
okFiles = stringIOFiles + diskFiles
if badFiles:
result = S_ERROR('Input Sandbox is not valid')
result['BadFile'] = badFiles
result['TotalSize'] = totalSize
return result
if okFiles:
if not self.sandboxClient:
self.sandboxClient = SandboxStoreClient(
useCertificates=self.useCertificates)
result = self.sandboxClient.uploadFilesAsSandbox(okFiles)
if not result['OK']:
return result
inputSandbox.append(result['Value'])
classAdJob.insertAttributeVectorString("InputSandbox",
inputSandbox)
return S_OK()
def initializeOptimizer(cls):
"""Initialize specific parameters for JobSanityAgent.
"""
cls.sandboxClient = SandboxStoreClient(useCertificates=True, smdb=True)
return S_OK()
def removeJobsByStatus( self, condDict, delay = False ):
""" Remove deleted jobs
"""
if delay:
gLogger.verbose( "Removing jobs with %s and older than %s" % ( condDict, delay ) )
result = self.jobDB.selectJobs( condDict, older = delay, limit = self.maxJobsAtOnce )
else:
gLogger.verbose( "Removing jobs with %s " % condDict )
result = self.jobDB.selectJobs( condDict, limit = self.maxJobsAtOnce )
if not result['OK']:
return result
jobList = result['Value']
if len(jobList) > self.maxJobsAtOnce:
jobList = jobList[:self.maxJobsAtOnce]
if not jobList:
return S_OK()
self.log.notice( "Deleting %s jobs for %s" % ( len( jobList ), condDict ) )
count = 0
error_count = 0
result = SandboxStoreClient( useCertificates = True ).unassignJobs( jobList )
if not result[ 'OK' ]:
gLogger.warn( "Cannot unassign jobs to sandboxes", result[ 'Message' ] )
if self.jobByJob:
for jobID in jobList:
resultJobDB = self.jobDB.removeJobFromDB( jobID )
resultTQ = self.taskQueueDB.deleteJob( jobID )
resultLogDB = self.jobLoggingDB.deleteJob( jobID )
errorFlag = False
if not resultJobDB['OK']:
gLogger.warn( 'Failed to remove job %d from JobDB' % jobID, result['Message'] )
errorFlag = True
if not resultTQ['OK']:
gLogger.warn( 'Failed to remove job %d from TaskQueueDB' % jobID, result['Message'] )
errorFlag = True
if not resultLogDB['OK']:
gLogger.warn( 'Failed to remove job %d from JobLoggingDB' % jobID, result['Message'] )
errorFlag = True
if errorFlag:
error_count += 1
else:
count += 1
if self.throttlingPeriod:
time.sleep(self.throttlingPeriod)
else:
result = self.jobDB.removeJobFromDB( jobList )
if not result['OK']:
gLogger.error('Failed to delete %d jobs from JobDB' % len(jobList) )
else:
gLogger.info('Deleted %d jobs from JobDB' % len(jobList) )
for jobID in jobList:
resultTQ = self.taskQueueDB.deleteJob( jobID )
if not resultTQ['OK']:
gLogger.warn( 'Failed to remove job %d from TaskQueueDB' % jobID, resultTQ['Message'] )
error_count += 1
else:
count += 1
result = self.jobLoggingDB.deleteJob( jobList )
if not result['OK']:
gLogger.error('Failed to delete %d jobs from JobLoggingDB' % len(jobList) )
else:
gLogger.info('Deleted %d jobs from JobLoggingDB' % len(jobList) )
if count > 0 or error_count > 0 :
gLogger.info( 'Deleted %d jobs from JobDB, %d errors' % ( count, error_count ) )
return S_OK()
def removeDeletedJobs(self):
"""Fully remove jobs that are already in status "DELETED", unless there are still requests.
:returns: S_OK/S_ERROR
"""
res = self._getJobsList({"Status": JobStatus.DELETED})
if not res["OK"]:
return res
jobList = res["Value"]
if not jobList:
self.log.info("No jobs to remove")
return S_OK()
self.log.info("Unassigning sandboxes from soon to be deleted jobs",
"(%d)" % len(jobList))
result = SandboxStoreClient(useCertificates=True).unassignJobs(jobList)
if not result["OK"]:
self.log.error("Cannot unassign jobs to sandboxes",
result["Message"])
return result
self.log.info("Attempting to remove deleted jobs",
"(%d)" % len(jobList))
# remove from jobList those that have still Operations to do in RMS
reqClient = ReqClient()
res = reqClient.getRequestIDsForJobs(jobList)
if not res["OK"]:
return res
if res["Value"]["Successful"]:
notFinal = set()
# Check whether these requests are in a final status
for job, reqID in res["Value"]["Successful"].items():
# If not, remove job from list to remove
if reqClient.getRequestStatus(reqID).get(
"Value") not in Request.FINAL_STATES:
# Keep that job
notFinal.add(job)
else:
# Remove the request, if failed, keep the job
res1 = reqClient.deleteRequest(reqID)
if not res1["OK"]:
notFinal.add(job)
if notFinal:
self.log.info(
"Some jobs won't be removed, as still having Requests not in final status",
"(n=%d)" % len(notFinal))
jobList = list(set(jobList) - notFinal)
if not jobList:
return S_OK()
ownerJobsDict = self._getOwnerJobsDict(jobList)
fail = False
for owner, jobsList in ownerJobsDict.items():
ownerDN = owner.split(";")[0]
ownerGroup = owner.split(";")[1]
self.log.verbose(
"Attempting to remove jobs",
"(n=%d) for %s : %s" % (len(jobsList), ownerDN, ownerGroup))
wmsClient = WMSClient(useCertificates=True,
delegatedDN=ownerDN,
delegatedGroup=ownerGroup)
result = wmsClient.removeJob(jobsList)
if not result["OK"]:
self.log.error(
"Could not remove jobs",
"for %s : %s (n=%d) : %s" %
(ownerDN, ownerGroup, len(jobsList), result["Message"]),
)
fail = True
if fail:
return S_ERROR()
return S_OK()
