def uploadSandbox( fileList ):
tmpDir = tempfile.mkdtemp( prefix = "upload.", dir = gWorkDir )
fileList = []
for fileName in fileList:
fDir = os.path.dirname( fileName )
if fDir:
fDir = os.path.join( tmpDir, fDir )
if not os.path.isdir( fDir ):
try:
os.makedirs( fDir )
except:
gLogger.exception( "Could not create temporal dir %s" % fDir )
bottle.abort( 500 )
absFile = os.path.join( tmpDir, fileName )
fileList.append( absFile )
ofd = reqFiles[ fileName ]
dfd = open( absFile, "w" )
dBuf = ofd.read( 524288 )
while dBug:
dfd.write( dBuf )
dBuf = ofd.read( 524288 )
dfd.close()
dBuf.close()
sbClient = SandboxStoreClient( useCertificates = True, delegatedDN = gOAData.userDN,
delegatedGroup = gOAData.userGroup )
result = sbClient.uploadFilesAsSandbox( fileList )
shutil.rmtree( tmpDir )
return result
def test_uploadFilesAsSandbox(self):
ourSSC = importlib.import_module('DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient')
ourSSC.TransferClient = MagicMock()
ssc = SandboxStoreClient()
fileList = [StringIO.StringIO('try')]
res = ssc.uploadFilesAsSandbox(fileList)
print(res)
def test_uploadFilesAsSandbox( self ):
ourSSC = importlib.import_module( 'DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient' )
ourSSC.TransferClient = MagicMock()
ssc = SandboxStoreClient()
fileList = [StringIO.StringIO( 'try' )]
res = ssc.uploadFilesAsSandbox( fileList )
print res
def test_uploadFilesAsSandbox(mocker, setUp):
mocker.patch(
"DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient.TransferClient",
return_value=MagicMock())
ssc = SandboxStoreClient()
fileList = [BytesIO(b"try")]
res = ssc.uploadFilesAsSandbox(fileList)
print(res)
def __uploadInputSandbox(self, classAdJob):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
sandboxClient = SandboxStoreClient(
useCertificates=self.useCertificates, rpcClient=self.sbRPCClient, transferClient=self.sbTransferClient
)
inputSandbox = self.__getInputSandboxEntries(classAdJob)
realFiles = []
badFiles = []
okFiles = []
realFiles = []
for file in inputSandbox:
valid = True
for tag in (
"lfn:",
"LFN:",
"SB:",
"%s",
): # in case of parametric input sandbox, there is %s passed, so have to ignore it also
if file.find(tag) == 0:
valid = False
break
if valid:
realFiles.append(file)
# If there are no files, skip!
if not realFiles:
return S_OK()
# Check real files
for file in realFiles:
if not os.path.exists(file):
badFiles.append(file)
print "inputSandbox file/directory " + file + " not found"
continue
okFiles.append(file)
# print "Total size of the inputSandbox: "+str(totalSize)
totalSize = File.getGlobbedTotalSize(okFiles)
if badFiles:
result = S_ERROR("Input Sandbox is not valid")
result["BadFile"] = badFiles
result["TotalSize"] = totalSize
return result
if okFiles:
result = sandboxClient.uploadFilesAsSandbox(okFiles)
if not result["OK"]:
return result
inputSandbox.append(result["Value"])
classAdJob.insertAttributeVectorString("InputSandbox", inputSandbox)
return S_OK()
def __uploadInputSandbox(self, classAdJob):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
sandboxClient = SandboxStoreClient(
useCertificates=self.useCertificates,
rpcClient=self.sbRPCClient,
transferClient=self.sbTransferClient)
inputSandbox = self.__getInputSandboxEntries(classAdJob)
realFiles = []
badFiles = []
okFiles = []
realFiles = []
for file in inputSandbox:
valid = True
for tag in (
'lfn:', 'LFN:', 'SB:', '%s'
): #in case of parametric input sandbox, there is %s passed, so have to ignore it also
if file.find(tag) == 0:
valid = False
break
if valid:
realFiles.append(file)
#If there are no files, skip!
if not realFiles:
return S_OK()
#Check real files
for file in realFiles:
if not os.path.exists(file):
badFiles.append(file)
print "inputSandbox file/directory " + file + " not found"
continue
okFiles.append(file)
#print "Total size of the inputSandbox: "+str(totalSize)
totalSize = File.getGlobbedTotalSize(okFiles)
if badFiles:
result = S_ERROR('Input Sandbox is not valid')
result['BadFile'] = badFiles
result['TotalSize'] = totalSize
return result
if okFiles:
result = sandboxClient.uploadFilesAsSandbox(okFiles)
if not result['OK']:
return result
inputSandbox.append(result['Value'])
classAdJob.insertAttributeVectorString("InputSandbox",
inputSandbox)
return S_OK()
def uploadSandbox(self, fileData):
with TmpDir() as tmpDir:
fileList = []
for fName in fileData:
for entry in fileData[fName]:
tmpFile = os.path.join(tmpDir, entry.filename)
if tmpFile not in fileList:
fileList.append(tmpFile)
dfd = open(tmpFile, "w")
dfd.write(entry.body)
dfd.close()
sbClient = SandboxStoreClient()
result = sbClient.uploadFilesAsSandbox(fileList)
if not result['OK']:
return WErr(500, result['Message'])
return WOK(result['Value'])
def uploadSandbox( self, fileData ):
with TmpDir() as tmpDir:
fileList = []
for fName in fileData:
for entry in fileData[ fName ]:
tmpFile = os.path.join( tmpDir, entry.filename )
if tmpFile not in fileList:
fileList.append( tmpFile )
dfd = open( tmpFile, "w" )
dfd.write( entry.body )
dfd.close()
sbClient = SandboxStoreClient()
result = sbClient.uploadFilesAsSandbox( fileList )
if not result[ 'OK' ]:
return WErr( 500, result[ 'Message' ] )
return WOK( result[ 'Value' ] )
def test_SSCChain( self ):
""" full test of functionalities
"""
ssc = SandboxStoreClient()
smDB = SandboxMetadataDB()
exeScriptLocation = find_all( 'exe-script.py', '.', 'WorkloadManagementSystem' )[0]
fileList = [exeScriptLocation]
res = ssc.uploadFilesAsSandbox( fileList )
self.assert_( res['OK'] )
# SEPFN = res['Value'].split( '|' )[1]
res = ssc.uploadFilesAsSandboxForJob( fileList, 1, 'Input' )
self.assert_( res['OK'] )
# res = ssc.downloadSandboxForJob( 1, 'Input' ) #to run this would need the RSS on
# self.assert_( res['OK'] )
# only ones needing the DB
res = smDB.getUnusedSandboxes()
self.assert_( res['OK'] )
def test_SSCChain():
"""full test of functionalities"""
ssc = SandboxStoreClient()
smDB = SandboxMetadataDB()
exeScriptLocation = find_all("exe-script.py", "..", "/DIRAC/tests/Integration")[0]
fileList = [exeScriptLocation]
res = ssc.uploadFilesAsSandbox(fileList)
assert res["OK"] is True, res["Message"]
# SEPFN = res['Value'].split( '|' )[1]
res = ssc.uploadFilesAsSandboxForJob(fileList, 1, "Input")
assert res["OK"] is True, res["Message"]
res = ssc.downloadSandboxForJob(1, "Input") # to run this we need the RSS on
print(res) # for debug...
assert res["OK"] is True, res["Message"]
# only ones needing the DB
res = smDB.getUnusedSandboxes()
print(res)
assert res["OK"] is True, res["Message"]
def test_SSCChain(self):
""" full test of functionalities
"""
ssc = SandboxStoreClient()
smDB = SandboxMetadataDB()
exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0]
fileList = [exeScriptLocation]
res = ssc.uploadFilesAsSandbox(fileList)
assert res['OK'] is True
# SEPFN = res['Value'].split( '|' )[1]
res = ssc.uploadFilesAsSandboxForJob(fileList, 1, 'Input')
assert res['OK'] is True
res = ssc.downloadSandboxForJob(1, 'Input') # to run this we need the RSS on
print res # for debug...
assert res['OK'] is True
# only ones needing the DB
res = smDB.getUnusedSandboxes()
print res
assert res['OK'] is True
def test_SSCChain(self):
""" full test of functionalities
"""
ssc = SandboxStoreClient()
smDB = SandboxMetadataDB()
exeScriptLocation = find_all('exe-script.py', '..',
'/DIRAC/tests/Integration')[0]
fileList = [exeScriptLocation]
res = ssc.uploadFilesAsSandbox(fileList)
assert res['OK'] is True
# SEPFN = res['Value'].split( '|' )[1]
res = ssc.uploadFilesAsSandboxForJob(fileList, 1, 'Input')
assert res['OK'] is True
res = ssc.downloadSandboxForJob(1,
'Input') # to run this we need the RSS on
print(res) # for debug...
assert res['OK'] is True
# only ones needing the DB
res = smDB.getUnusedSandboxes()
print(res)
assert res['OK'] is True
class WMSClient( object ):
def __init__( self, jobManagerClient = None, sbRPCClient = None, sbTransferClient = None,
useCertificates = False, timeout = 600 ):
""" WMS Client constructor
Here we also initialize the needed clients and connections
"""
self.useCertificates = useCertificates
self.timeout = timeout
self.jobManager = jobManagerClient
self.sandboxClient = None
if sbRPCClient and sbTransferClient:
self.sandboxClient = SandboxStoreClient( rpcClient = sbRPCClient,
transferClient = sbTransferClient,
useCertificates = useCertificates )
###############################################################################
def __getInputSandboxEntries( self, classAdJob ):
if classAdJob.lookupAttribute( "InputSandbox" ):
inputSandbox = classAdJob.get_expression( "InputSandbox" )
inputSandbox = inputSandbox.replace( '","', "\n" )
inputSandbox = inputSandbox.replace( '{', "" )
inputSandbox = inputSandbox.replace( '}', "" )
inputSandbox = inputSandbox.replace( '"', "" )
inputSandbox = inputSandbox.replace( ',', "" )
inputSandbox = inputSandbox.split()
else:
inputSandbox = []
return inputSandbox
def __uploadInputSandbox( self, classAdJob ):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries( classAdJob )
badFiles = []
okFiles = []
realFiles = []
for isFile in inputSandbox:
valid = True
for tag in ( 'lfn:', 'LFN:', 'SB:', '%s' ): # in case of parametric input sandbox, there is %s passed, so have to ignore it also
if isFile.find( tag ) == 0:
valid = False
break
if valid:
realFiles.append( isFile )
# If there are no files, skip!
if not realFiles:
return S_OK()
# Check real files
for isFile in realFiles:
if not os.path.exists( isFile ):
badFiles.append( isFile )
gLogger.warn( "inputSandbox file/directory " + isFile + " not found. Keep looking for the others" )
continue
okFiles.append( isFile )
totalSize = File.getGlobbedTotalSize( okFiles )
gLogger.verbose( "Total size of the inputSandbox: " + str( totalSize ) )
if badFiles:
result = S_ERROR( 'Input Sandbox is not valid' )
result['BadFile'] = badFiles
result['TotalSize'] = totalSize
return result
if okFiles:
if not self.sandboxClient:
self.sandboxClient = SandboxStoreClient( useCertificates = self.useCertificates )
result = self.sandboxClient.uploadFilesAsSandbox( okFiles )
if not result[ 'OK' ]:
return result
inputSandbox.append( result[ 'Value' ] )
classAdJob.insertAttributeVectorString( "InputSandbox", inputSandbox )
return S_OK()
def submitJob( self, jdl ):
""" Submit one job specified by its JDL to WMS
"""
if os.path.exists( jdl ):
fic = open ( jdl, "r" )
jdlString = fic.read()
fic.close()
else:
# If file JDL does not exist, assume that the JDL is passed as a string
jdlString = jdl
# Check the validity of the input JDL
jdlString = jdlString.strip()
if jdlString.find( "[" ) != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd( jdlString )
if not classAdJob.isOK():
return S_ERROR( 'Invalid job JDL' )
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox( classAdJob )
if not result['OK']:
return result
# Submit the job now and get the new job ID
if not self.jobManager:
self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
useCertificates = self.useCertificates,
timeout = self.timeout )
result = self.jobManager.submitJob( classAdJob.asJDL() )
if 'requireProxyUpload' in result and result['requireProxyUpload']:
gLogger.warn( "Need to upload the proxy" )
return result
def killJob( self, jobID ):
""" Kill running job.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
useCertificates = self.useCertificates,
timeout = self.timeout )
return self.jobManager.killJob( jobID )
def deleteJob( self, jobID ):
""" Delete job(s) from the WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
useCertificates = self.useCertificates,
timeout = self.timeout )
return self.jobManager.deleteJob( jobID )
def rescheduleJob( self, jobID ):
""" Reschedule job(s) in WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
useCertificates = self.useCertificates,
timeout = self.timeout )
return self.jobManager.rescheduleJob( jobID )
def resetJob( self, jobID ):
""" Reset job(s) in WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
useCertificates = self.useCertificates,
timeout = self.timeout )
return self.jobManager.resetJob( jobID )
class WMSClient(object):
"""Class exposing the following jobs methods:
submit
kill
delete
remove
reschedule
reset
"""
def __init__(
self,
jobManagerClient=None,
sbRPCClient=None,
sbTransferClient=None,
useCertificates=False,
timeout=600,
delegatedDN=None,
delegatedGroup=None,
):
"""WMS Client constructor
Here we also initialize the needed clients and connections
"""
self.useCertificates = useCertificates
self.delegatedDN = delegatedDN
self.delegatedGroup = delegatedGroup
self.timeout = timeout
self._jobManager = jobManagerClient
self.operationsHelper = Operations()
self.sandboxClient = None
if sbRPCClient and sbTransferClient:
self.sandboxClient = SandboxStoreClient(
rpcClient=sbRPCClient, transferClient=sbTransferClient, useCertificates=useCertificates
)
@property
def jobManager(self):
if not self._jobManager:
self._jobManager = JobManagerClient(
useCertificates=self.useCertificates,
delegatedDN=self.delegatedDN,
delegatedGroup=self.delegatedGroup,
timeout=self.timeout,
)
return self._jobManager
def __getInputSandboxEntries(self, classAdJob):
if classAdJob.lookupAttribute("InputSandbox"):
inputSandbox = classAdJob.get_expression("InputSandbox")
inputSandbox = inputSandbox.replace('","', "\n")
inputSandbox = inputSandbox.replace("{", "")
inputSandbox = inputSandbox.replace("}", "")
inputSandbox = inputSandbox.replace('"', "")
inputSandbox = inputSandbox.replace(",", "")
inputSandbox = inputSandbox.split()
else:
inputSandbox = []
return inputSandbox
def __uploadInputSandbox(self, classAdJob, jobDescriptionObject=None):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries(classAdJob)
realFiles = []
badFiles = []
diskFiles = []
for isFile in inputSandbox:
if not isFile.startswith(("lfn:", "LFN:", "SB:", "%s", "%(")):
realFiles.append(isFile)
stringIOFiles = []
stringIOFilesSize = 0
if jobDescriptionObject is not None:
if isinstance(jobDescriptionObject, StringIO):
stringIOFiles = [jobDescriptionObject]
stringIOFilesSize = len(jobDescriptionObject.getvalue())
gLogger.debug("Size of the stringIOFiles: " + str(stringIOFilesSize))
else:
return S_ERROR(EWMSJDL, "jobDescriptionObject is not a StringIO object")
# Check real files
for isFile in realFiles:
if not os.path.exists(isFile): # we are passing in real files, we expect them to be on disk
badFiles.append(isFile)
gLogger.warn("inputSandbox file/directory " + isFile + " not found. Keep looking for the others")
continue
diskFiles.append(isFile)
diskFilesSize = File.getGlobbedTotalSize(diskFiles)
gLogger.debug("Size of the diskFiles: " + str(diskFilesSize))
totalSize = diskFilesSize + stringIOFilesSize
gLogger.verbose("Total size of the inputSandbox: " + str(totalSize))
okFiles = stringIOFiles + diskFiles
if badFiles:
result = S_ERROR(EWMSJDL, "Input Sandbox is not valid")
result["BadFile"] = badFiles
result["TotalSize"] = totalSize
return result
if okFiles:
if not self.sandboxClient:
self.sandboxClient = SandboxStoreClient(
useCertificates=self.useCertificates,
delegatedDN=self.delegatedDN,
delegatedGroup=self.delegatedGroup,
)
result = self.sandboxClient.uploadFilesAsSandbox(okFiles)
if not result["OK"]:
return result
inputSandbox.append(result["Value"])
classAdJob.insertAttributeVectorString("InputSandbox", inputSandbox)
return S_OK()
def submitJob(self, jdl, jobDescriptionObject=None):
"""Submit one job specified by its JDL to WMS.
The JDL may actually be the desciption of a parametric job,
resulting in multiple DIRAC jobs submitted to the DIRAC WMS
"""
if os.path.exists(jdl):
with open(jdl, "r") as fic:
jdlString = fic.read()
else:
# If file JDL does not exist, assume that the JDL is passed as a string
jdlString = jdl
jdlString = jdlString.strip()
gLogger.debug("Submitting JDL", jdlString)
# Strip of comments in the jdl string
newJdlList = []
for line in jdlString.split("\n"):
if not line.strip().startswith("#"):
newJdlList.append(line)
jdlString = "\n".join(newJdlList)
# Check the validity of the input JDL
if jdlString.find("[") != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd(jdlString)
if not classAdJob.isOK():
return S_ERROR(EWMSJDL, "Invalid job JDL")
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox(classAdJob, jobDescriptionObject)
if not result["OK"]:
return result
# Submit the job now and get the new job ID
result = getParameterVectorLength(classAdJob)
if not result["OK"]:
return result
nJobs = result["Value"]
result = self.jobManager.submitJob(classAdJob.asJDL())
if nJobs:
gLogger.debug("Applying transactional job submission")
# The server applies transactional bulk submission, we should confirm the jobs
if result["OK"]:
jobIDList = result["Value"]
if len(jobIDList) == nJobs:
# Confirm the submitted jobs
confirmed = False
for _attempt in range(3):
result = self.jobManager.confirmBulkSubmission(jobIDList)
if result["OK"]:
confirmed = True
break
time.sleep(1)
if not confirmed:
# The bulk submission failed, try to remove the created jobs
resultDelete = self.jobManager.removeJob(jobIDList)
error = "Job submission failed to confirm bulk transaction"
if not resultDelete["OK"]:
error += "; removal of created jobs failed"
return S_ERROR(EWMSSUBM, error)
else:
return S_ERROR(EWMSSUBM, "The number of submitted jobs does not match job description")
if result.get("requireProxyUpload"):
gLogger.warn("Need to upload the proxy")
return result
def killJob(self, jobID):
"""Kill running job.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
return self.jobManager.killJob(jobID)
def deleteJob(self, jobID):
"""Delete job(s) (set their status to DELETED) from the WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
return self.jobManager.deleteJob(jobID)
def removeJob(self, jobID):
"""Fully remove job(s) from the WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
return self.jobManager.removeJob(jobID)
def rescheduleJob(self, jobID):
"""Reschedule job(s) in WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
return self.jobManager.rescheduleJob(jobID)
def resetJob(self, jobID):
"""Reset job(s) in WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
return self.jobManager.resetJob(jobID)
class WMSClient:
def __init__( self, jobManagerClient = False, sbRPCClient = False, sbTransferClient = False,
useCertificates = False, timeout = 120 ):
""" WMS Client constructor
"""
self.jobManagerClient = jobManagerClient
self.useCertificates = useCertificates
self.timeout = timeout
self.sandboxClient = SandboxStoreClient( useCertificates = useCertificates, rpcClient = sbRPCClient,
transferClient = sbTransferClient )
###############################################################################
def __getInputSandboxEntries( self, classAdJob ):
if classAdJob.lookupAttribute( "InputSandbox" ):
inputSandbox = classAdJob.get_expression( "InputSandbox" )
inputSandbox = inputSandbox.replace( '","', "\n" )
inputSandbox = inputSandbox.replace( '{', "" )
inputSandbox = inputSandbox.replace( '}', "" )
inputSandbox = inputSandbox.replace( '"', "" )
inputSandbox = inputSandbox.replace( ',', "" )
inputSandbox = inputSandbox.split()
else:
inputSandbox = []
return inputSandbox
# This are the NEW methods
def __uploadInputSandbox( self, classAdJob ):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries( classAdJob )
realFiles = []
badFiles = []
okFiles = []
realFiles = []
for file in inputSandbox:
valid = True
for tag in ( 'lfn:', 'LFN:', 'SB:', '%s' ): # in case of parametric input sandbox, there is %s passed, so have to ignore it also
if file.find( tag ) == 0:
valid = False
break
if valid:
realFiles.append( file )
# If there are no files, skip!
if not realFiles:
return S_OK()
# Check real files
for file in realFiles:
if not os.path.exists( file ):
badFiles.append( file )
print "inputSandbox file/directory " + file + " not found"
continue
okFiles.append( file )
# print "Total size of the inputSandbox: "+str(totalSize)
totalSize = File.getGlobbedTotalSize( okFiles )
if badFiles:
result = S_ERROR( 'Input Sandbox is not valid' )
result['BadFile'] = badFiles
result['TotalSize'] = totalSize
return result
if okFiles:
result = self.sandboxClient.uploadFilesAsSandbox( okFiles )
if not result[ 'OK' ]:
return result
inputSandbox.append( result[ 'Value' ] )
classAdJob.insertAttributeVectorString( "InputSandbox", inputSandbox )
return S_OK()
def __assignSandboxesToJob( self, jobID, classAdJob ):
sandboxClient = SandboxStoreClient()
inputSandboxes = self.__getInputSandboxEntries( classAdJob )
sbToAssign = []
for isb in inputSandboxes:
if isb.find( "SB:" ) == 0:
sbToAssign.append( isb )
if sbToAssign:
assignList = [ ( isb, 'Input' ) for isb in sbToAssign ]
result = sandboxClient.assignSandboxesToJob( jobID, assignList )
if not result[ 'OK' ]:
return result
return S_OK()
def submitJob( self, jdl ):
""" Submit one job specified by its JDL to WMS
"""
if not self.jobManagerClient:
jobManager = RPCClient( 'WorkloadManagement/JobManager', useCertificates = self.useCertificates,
timeout = self.timeout )
else:
jobManager = self.jobManagerClient
if os.path.exists( jdl ):
fic = open ( jdl, "r" )
jdlString = fic.read()
fic.close()
else:
# If file JDL does not exist, assume that the JDL is
# passed as a string
jdlString = jdl
# Check the validity of the input JDL
jdlString = jdlString.strip()
if jdlString.find( "[" ) != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd( jdlString )
if not classAdJob.isOK():
return S_ERROR( 'Invalid job JDL' )
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox( classAdJob )
if not result['OK']:
return result
# Submit the job now and get the new job ID
result = jobManager.submitJob( classAdJob.asJDL() )
if not result['OK']:
return result
jobID = result['Value']
if 'requireProxyUpload' in result and result[ 'requireProxyUpload' ]:
# TODO: We should notify the user to upload a proxy with proxy-upload
pass
# print "Sandbox uploading"
return S_OK( jobID )
# This is the OLD method
def __checkInputSandbox( self, classAdJob ):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries( classAdJob )
if inputSandbox:
ok = 1
# print inputSandbox
# Check the Input Sandbox files
totalSize = 0
for file in inputSandbox:
if file.find( 'lfn:' ) != 0 and file.find( 'LFN:' ) != 0 and file.find( "SB:" ) != 0:
if not os.path.exists( file ):
badfile = file
print "inputSandbox file/directory " + file + " not found"
ok = 0
else:
if os.path.isdir( file ):
comm = 'du -b -s ' + file
status, out = commands.getstatusoutput( comm )
try:
dirSize = int( out.split()[0] )
except Exception, x:
print "Input Sandbox directory name", file, "is not valid !"
print str( x )
badfile = file
ok = 0
totalSize = totalSize + dirSize
else:
totalSize = int( os.stat( file )[6] ) + totalSize
# print "Total size of the inputSandbox: "+str(totalSize)
if not ok:
result = S_ERROR( 'Input Sandbox is not valid' )
result['BadFile'] = file
result['TotalSize'] = totalSize
return result
result = S_OK()
result['InputSandbox'] = inputSandbox
result['TotalSize'] = totalSize
return result
class WMSClient(object):
def __init__(self,
jobManagerClient=None,
sbRPCClient=None,
sbTransferClient=None,
useCertificates=False,
timeout=600):
""" WMS Client constructor
Here we also initialize the needed clients and connections
"""
self.useCertificates = useCertificates
self.timeout = timeout
self.jobManager = jobManagerClient
self.sandboxClient = None
if sbRPCClient and sbTransferClient:
self.sandboxClient = SandboxStoreClient(
rpcClient=sbRPCClient,
transferClient=sbTransferClient,
useCertificates=useCertificates)
###############################################################################
def __getInputSandboxEntries(self, classAdJob):
if classAdJob.lookupAttribute("InputSandbox"):
inputSandbox = classAdJob.get_expression("InputSandbox")
inputSandbox = inputSandbox.replace('","', "\n")
inputSandbox = inputSandbox.replace('{', "")
inputSandbox = inputSandbox.replace('}', "")
inputSandbox = inputSandbox.replace('"', "")
inputSandbox = inputSandbox.replace(',', "")
inputSandbox = inputSandbox.split()
else:
inputSandbox = []
return inputSandbox
def __uploadInputSandbox(self, classAdJob, jobDescriptionObject=None):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries(classAdJob)
realFiles = []
badFiles = []
diskFiles = []
for isFile in inputSandbox:
if not isFile.startswith(('lfn:', 'LFN:', 'SB:', '%s', '%(')):
realFiles.append(isFile)
stringIOFiles = []
stringIOFilesSize = 0
if jobDescriptionObject is not None:
if isinstance(jobDescriptionObject, StringIO.StringIO):
stringIOFiles = [jobDescriptionObject]
stringIOFilesSize = len(jobDescriptionObject.buf)
gLogger.debug("Size of the stringIOFiles: " +
str(stringIOFilesSize))
else:
return S_ERROR("jobDescriptionObject is not a StringIO object")
# Check real files
for isFile in realFiles:
if not os.path.exists(
isFile
): # we are passing in real files, we expect them to be on disk
badFiles.append(isFile)
gLogger.warn("inputSandbox file/directory " + isFile +
" not found. Keep looking for the others")
continue
diskFiles.append(isFile)
diskFilesSize = File.getGlobbedTotalSize(diskFiles)
gLogger.debug("Size of the diskFiles: " + str(diskFilesSize))
totalSize = diskFilesSize + stringIOFilesSize
gLogger.verbose("Total size of the inputSandbox: " + str(totalSize))
okFiles = stringIOFiles + diskFiles
if badFiles:
result = S_ERROR('Input Sandbox is not valid')
result['BadFile'] = badFiles
result['TotalSize'] = totalSize
return result
if okFiles:
if not self.sandboxClient:
self.sandboxClient = SandboxStoreClient(
useCertificates=self.useCertificates)
result = self.sandboxClient.uploadFilesAsSandbox(okFiles)
if not result['OK']:
return result
inputSandbox.append(result['Value'])
classAdJob.insertAttributeVectorString("InputSandbox",
inputSandbox)
return S_OK()
def submitJob(self, jdl, jobDescriptionObject=None):
""" Submit one job specified by its JDL to WMS
"""
if os.path.exists(jdl):
fic = open(jdl, "r")
jdlString = fic.read()
fic.close()
else:
# If file JDL does not exist, assume that the JDL is passed as a string
jdlString = jdl
jdlString = jdlString.strip()
# Strip of comments in the jdl string
newJdlList = []
for line in jdlString.split('\n'):
if not line.strip().startswith('#'):
newJdlList.append(line)
jdlString = '\n'.join(newJdlList)
# Check the validity of the input JDL
if jdlString.find("[") != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd(jdlString)
if not classAdJob.isOK():
return S_ERROR('Invalid job JDL')
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox(classAdJob, jobDescriptionObject)
if not result['OK']:
return result
# Submit the job now and get the new job ID
if not self.jobManager:
self.jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
result = self.jobManager.submitJob(classAdJob.asJDL())
if 'requireProxyUpload' in result and result['requireProxyUpload']:
gLogger.warn("Need to upload the proxy")
return result
def killJob(self, jobID):
""" Kill running job.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
return self.jobManager.killJob(jobID)
def deleteJob(self, jobID):
""" Delete job(s) from the WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
return self.jobManager.deleteJob(jobID)
def rescheduleJob(self, jobID):
""" Reschedule job(s) in WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
return self.jobManager.rescheduleJob(jobID)
def resetJob(self, jobID):
""" Reset job(s) in WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
return self.jobManager.resetJob(jobID)
class WMSClient(object):
def __init__(self,
jobManagerClient=None,
sbRPCClient=None,
sbTransferClient=None,
useCertificates=False,
timeout=600):
""" WMS Client constructor
Here we also initialize the needed clients and connections
"""
self.useCertificates = useCertificates
self.timeout = timeout
self.jobManager = jobManagerClient
self.sandboxClient = None
if sbRPCClient and sbTransferClient:
self.sandboxClient = SandboxStoreClient(
rpcClient=sbRPCClient,
transferClient=sbTransferClient,
useCertificates=useCertificates)
###############################################################################
def __getInputSandboxEntries(self, classAdJob):
if classAdJob.lookupAttribute("InputSandbox"):
inputSandbox = classAdJob.get_expression("InputSandbox")
inputSandbox = inputSandbox.replace('","', "\n")
inputSandbox = inputSandbox.replace('{', "")
inputSandbox = inputSandbox.replace('}', "")
inputSandbox = inputSandbox.replace('"', "")
inputSandbox = inputSandbox.replace(',', "")
inputSandbox = inputSandbox.split()
else:
inputSandbox = []
return inputSandbox
def __uploadInputSandbox(self, classAdJob):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries(classAdJob)
badFiles = []
okFiles = []
realFiles = []
for isFile in inputSandbox:
valid = True
for tag in (
'lfn:', 'LFN:', 'SB:', '%s'
): # in case of parametric input sandbox, there is %s passed, so have to ignore it also
if isFile.find(tag) == 0:
valid = False
break
if valid:
realFiles.append(isFile)
# If there are no files, skip!
if not realFiles:
return S_OK()
# Check real files
for isFile in realFiles:
if not os.path.exists(isFile):
badFiles.append(isFile)
gLogger.warn("inputSandbox file/directory " + isFile +
" not found. Keep looking for the others")
continue
okFiles.append(isFile)
totalSize = File.getGlobbedTotalSize(okFiles)
gLogger.verbose("Total size of the inputSandbox: " + str(totalSize))
if badFiles:
result = S_ERROR('Input Sandbox is not valid')
result['BadFile'] = badFiles
result['TotalSize'] = totalSize
return result
if okFiles:
if not self.sandboxClient:
self.sandboxClient = SandboxStoreClient(
useCertificates=self.useCertificates)
result = self.sandboxClient.uploadFilesAsSandbox(okFiles)
if not result['OK']:
return result
inputSandbox.append(result['Value'])
classAdJob.insertAttributeVectorString("InputSandbox",
inputSandbox)
return S_OK()
def submitJob(self, jdl):
""" Submit one job specified by its JDL to WMS
"""
if os.path.exists(jdl):
fic = open(jdl, "r")
jdlString = fic.read()
fic.close()
else:
# If file JDL does not exist, assume that the JDL is passed as a string
jdlString = jdl
# Check the validity of the input JDL
jdlString = jdlString.strip()
if jdlString.find("[") != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd(jdlString)
if not classAdJob.isOK():
return S_ERROR('Invalid job JDL')
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox(classAdJob)
if not result['OK']:
return result
# Submit the job now and get the new job ID
if not self.jobManager:
self.jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
result = self.jobManager.submitJob(classAdJob.asJDL())
if 'requireProxyUpload' in result and result['requireProxyUpload']:
gLogger.warn("Need to upload the proxy")
return result
def killJob(self, jobID):
""" Kill running job.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
return self.jobManager.killJob(jobID)
def deleteJob(self, jobID):
""" Delete job(s) from the WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
return self.jobManager.deleteJob(jobID)
def rescheduleJob(self, jobID):
""" Reschedule job(s) in WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
return self.jobManager.rescheduleJob(jobID)
def resetJob(self, jobID):
""" Reset job(s) in WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
return self.jobManager.resetJob(jobID)
class WMSClient:
def __init__(self,
jobManagerClient=False,
sbRPCClient=False,
sbTransferClient=False,
useCertificates=False,
timeout=120):
""" WMS Client constructor
"""
self.jobManagerClient = jobManagerClient
self.useCertificates = useCertificates
self.timeout = timeout
self.sandboxClient = SandboxStoreClient(
useCertificates=useCertificates,
rpcClient=sbRPCClient,
transferClient=sbTransferClient)
###############################################################################
def __getInputSandboxEntries(self, classAdJob):
if classAdJob.lookupAttribute("InputSandbox"):
inputSandbox = classAdJob.get_expression("InputSandbox")
inputSandbox = inputSandbox.replace('","', "\n")
inputSandbox = inputSandbox.replace('{', "")
inputSandbox = inputSandbox.replace('}', "")
inputSandbox = inputSandbox.replace('"', "")
inputSandbox = inputSandbox.replace(',', "")
inputSandbox = inputSandbox.split()
else:
inputSandbox = []
return inputSandbox
# This are the NEW methods
def __uploadInputSandbox(self, classAdJob):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries(classAdJob)
realFiles = []
badFiles = []
okFiles = []
realFiles = []
for file in inputSandbox:
valid = True
for tag in (
'lfn:', 'LFN:', 'SB:', '%s'
): # in case of parametric input sandbox, there is %s passed, so have to ignore it also
if file.find(tag) == 0:
valid = False
break
if valid:
realFiles.append(file)
# If there are no files, skip!
if not realFiles:
return S_OK()
# Check real files
for file in realFiles:
if not os.path.exists(file):
badFiles.append(file)
print "inputSandbox file/directory " + file + " not found"
continue
okFiles.append(file)
# print "Total size of the inputSandbox: "+str(totalSize)
totalSize = File.getGlobbedTotalSize(okFiles)
if badFiles:
result = S_ERROR('Input Sandbox is not valid')
result['BadFile'] = badFiles
result['TotalSize'] = totalSize
return result
if okFiles:
result = self.sandboxClient.uploadFilesAsSandbox(okFiles)
if not result['OK']:
return result
inputSandbox.append(result['Value'])
classAdJob.insertAttributeVectorString("InputSandbox",
inputSandbox)
return S_OK()
def __assignSandboxesToJob(self, jobID, classAdJob):
sandboxClient = SandboxStoreClient()
inputSandboxes = self.__getInputSandboxEntries(classAdJob)
sbToAssign = []
for isb in inputSandboxes:
if isb.find("SB:") == 0:
sbToAssign.append(isb)
if sbToAssign:
assignList = [(isb, 'Input') for isb in sbToAssign]
result = sandboxClient.assignSandboxesToJob(jobID, assignList)
if not result['OK']:
return result
return S_OK()
def submitJob(self, jdl):
""" Submit one job specified by its JDL to WMS
"""
if not self.jobManagerClient:
jobManager = RPCClient('WorkloadManagement/JobManager',
useCertificates=self.useCertificates,
timeout=self.timeout)
else:
jobManager = self.jobManagerClient
if os.path.exists(jdl):
fic = open(jdl, "r")
jdlString = fic.read()
fic.close()
else:
# If file JDL does not exist, assume that the JDL is
# passed as a string
jdlString = jdl
# Check the validity of the input JDL
jdlString = jdlString.strip()
if jdlString.find("[") != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd(jdlString)
if not classAdJob.isOK():
return S_ERROR('Invalid job JDL')
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox(classAdJob)
if not result['OK']:
return result
# Submit the job now and get the new job ID
result = jobManager.submitJob(classAdJob.asJDL())
if not result['OK']:
return result
jobID = result['Value']
if 'requireProxyUpload' in result and result['requireProxyUpload']:
# TODO: We should notify the user to upload a proxy with proxy-upload
pass
# print "Sandbox uploading"
return S_OK(jobID)
# This is the OLD method
def __checkInputSandbox(self, classAdJob):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries(classAdJob)
if inputSandbox:
ok = 1
# print inputSandbox
# Check the Input Sandbox files
totalSize = 0
for file in inputSandbox:
if file.find('lfn:') != 0 and file.find(
'LFN:') != 0 and file.find("SB:") != 0:
if not os.path.exists(file):
badfile = file
print "inputSandbox file/directory " + file + " not found"
ok = 0
else:
if os.path.isdir(file):
comm = 'du -b -s ' + file
status, out = commands.getstatusoutput(comm)
try:
dirSize = int(out.split()[0])
except Exception, x:
print "Input Sandbox directory name", file, "is not valid !"
print str(x)
badfile = file
ok = 0
totalSize = totalSize + dirSize
else:
totalSize = int(os.stat(file)[6]) + totalSize
# print "Total size of the inputSandbox: "+str(totalSize)
if not ok:
result = S_ERROR('Input Sandbox is not valid')
result['BadFile'] = file
result['TotalSize'] = totalSize
return result
result = S_OK()
result['InputSandbox'] = inputSandbox
result['TotalSize'] = totalSize
return result
class WMSClient( object ):
def __init__( self, jobManagerClient = None, sbRPCClient = None, sbTransferClient = None,
useCertificates = False, timeout = 600 ):
""" WMS Client constructor
Here we also initialize the needed clients and connections
"""
self.useCertificates = useCertificates
self.timeout = timeout
self.jobManager = jobManagerClient
self.sandboxClient = None
if sbRPCClient and sbTransferClient:
self.sandboxClient = SandboxStoreClient( rpcClient = sbRPCClient,
transferClient = sbTransferClient,
useCertificates = useCertificates )
###############################################################################
def __getInputSandboxEntries( self, classAdJob ):
if classAdJob.lookupAttribute( "InputSandbox" ):
inputSandbox = classAdJob.get_expression( "InputSandbox" )
inputSandbox = inputSandbox.replace( '","', "\n" )
inputSandbox = inputSandbox.replace( '{', "" )
inputSandbox = inputSandbox.replace( '}', "" )
inputSandbox = inputSandbox.replace( '"', "" )
inputSandbox = inputSandbox.replace( ',', "" )
inputSandbox = inputSandbox.split()
else:
inputSandbox = []
return inputSandbox
def __uploadInputSandbox( self, classAdJob, jobDescriptionObject = None ):
"""Checks the validity of the job Input Sandbox.
The function returns the list of Input Sandbox files.
The total volume of the input sandbox is evaluated
"""
inputSandbox = self.__getInputSandboxEntries( classAdJob )
realFiles = []
badFiles = []
diskFiles = []
for isFile in inputSandbox:
if not isFile.startswith( ( 'lfn:', 'LFN:', 'SB:', '%s', '%(' ) ):
realFiles.append( isFile )
stringIOFiles = []
stringIOFilesSize = 0
if jobDescriptionObject is not None:
if isinstance( jobDescriptionObject, StringIO.StringIO ):
stringIOFiles = [jobDescriptionObject]
stringIOFilesSize = len( jobDescriptionObject.buf )
gLogger.debug( "Size of the stringIOFiles: " + str( stringIOFilesSize ) )
else:
return S_ERROR( "jobDescriptionObject is not a StringIO object" )
# Check real files
for isFile in realFiles:
if not os.path.exists( isFile ): # we are passing in real files, we expect them to be on disk
badFiles.append( isFile )
gLogger.warn( "inputSandbox file/directory " + isFile + " not found. Keep looking for the others" )
continue
diskFiles.append( isFile )
diskFilesSize = File.getGlobbedTotalSize( diskFiles )
gLogger.debug( "Size of the diskFiles: " + str( diskFilesSize ) )
totalSize = diskFilesSize + stringIOFilesSize
gLogger.verbose( "Total size of the inputSandbox: " + str( totalSize ) )
okFiles = stringIOFiles + diskFiles
if badFiles:
result = S_ERROR( 'Input Sandbox is not valid' )
result['BadFile'] = badFiles
result['TotalSize'] = totalSize
return result
if okFiles:
if not self.sandboxClient:
self.sandboxClient = SandboxStoreClient( useCertificates = self.useCertificates )
result = self.sandboxClient.uploadFilesAsSandbox( okFiles )
if not result[ 'OK' ]:
return result
inputSandbox.append( result[ 'Value' ] )
classAdJob.insertAttributeVectorString( "InputSandbox", inputSandbox )
return S_OK()
def submitJob( self, jdl, jobDescriptionObject = None ):
""" Submit one job specified by its JDL to WMS
"""
if os.path.exists( jdl ):
fic = open ( jdl, "r" )
jdlString = fic.read()
fic.close()
else:
# If file JDL does not exist, assume that the JDL is passed as a string
jdlString = jdl
jdlString = jdlString.strip()
# Strip of comments in the jdl string
newJdlList = []
for line in jdlString.split('\n'):
if not line.strip().startswith( '#' ):
newJdlList.append( line )
jdlString = '\n'.join( newJdlList )
# Check the validity of the input JDL
if jdlString.find( "[" ) != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd( jdlString )
if not classAdJob.isOK():
return S_ERROR( 'Invalid job JDL' )
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox( classAdJob, jobDescriptionObject )
if not result['OK']:
return result
# Submit the job now and get the new job ID
if not self.jobManager:
self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
useCertificates = self.useCertificates,
timeout = self.timeout )
result = self.jobManager.submitJob( classAdJob.asJDL() )
if 'requireProxyUpload' in result and result['requireProxyUpload']:
gLogger.warn( "Need to upload the proxy" )
return result
def killJob( self, jobID ):
""" Kill running job.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
useCertificates = self.useCertificates,
timeout = self.timeout )
return self.jobManager.killJob( jobID )
def deleteJob( self, jobID ):
""" Delete job(s) from the WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
useCertificates = self.useCertificates,
timeout = self.timeout )
return self.jobManager.deleteJob( jobID )
def rescheduleJob( self, jobID ):
""" Reschedule job(s) in WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
useCertificates = self.useCertificates,
timeout = self.timeout )
return self.jobManager.rescheduleJob( jobID )
def resetJob( self, jobID ):
""" Reset job(s) in WMS Job database.
jobID can be an integer representing a single DIRAC job ID or a list of IDs
"""
if not self.jobManager:
self.jobManager = RPCClient( 'WorkloadManagement/JobManager',
useCertificates = self.useCertificates,
timeout = self.timeout )
return self.jobManager.resetJob( jobID )