def test_NoParameters(self):
clad = ClassAd( TEST_JDL_NO_PARAMETERS )
result = getParameterVectorLength( clad )
self.assertTrue( result['OK'] )
nParam = result['Value']
self.assertTrue( nParam is None )
def test_NoParameters(self):
clad = ClassAd(TEST_JDL_NO_PARAMETERS)
result = getParameterVectorLength(clad)
self.assertTrue(result['OK'])
nParam = result['Value']
self.assertTrue(nParam is None)
def test_Simple(self):
clad = ClassAd(TEST_JDL_SIMPLE)
result = getParameterVectorLength(clad)
self.assertTrue(result['OK'])
nParam = result['Value']
self.assertEqual(nParam, 3)
result = generateParametricJobs(clad)
self.assertTrue(result['OK'])
jobDescList = result['Value']
self.assertEqual(nParam, len(jobDescList))
# Check the definition of the 2nd job
jobClassAd = ClassAd(jobDescList[1])
self.assertEqual(jobClassAd.getAttributeString('Arguments'), 'b')
self.assertEqual(jobClassAd.getAttributeString('JobName'), 'Test_1')
def test_SimpleBunch(self):
clad = ClassAd(TEST_JDL_SIMPLE_BUNCH)
result = getParameterVectorLength(clad)
self.assertTrue(result["OK"])
nParam = result["Value"]
self.assertEqual(nParam, 3)
result = generateParametricJobs(clad)
self.assertTrue(result["OK"])
jobDescList = result["Value"]
self.assertEqual(nParam, len(jobDescList))
# Check the definition of the 2nd job
jobClassAd = ClassAd(jobDescList[1])
self.assertEqual(jobClassAd.getAttributeString("Arguments"), "5")
self.assertEqual(jobClassAd.getAttributeString("JobName"), "Test_1")
def test_SimpleBunch(self):
clad = ClassAd( TEST_JDL_SIMPLE_BUNCH )
result = getParameterVectorLength( clad )
self.assertTrue( result['OK'] )
nParam = result['Value']
self.assertEqual( nParam, 3 )
result = generateParametricJobs( clad )
self.assertTrue(result['OK'])
jobDescList = result['Value']
self.assertEqual( nParam, len( jobDescList ) )
# Check the definition of the 2nd job
jobClassAd = ClassAd( jobDescList[1] )
self.assertEqual( jobClassAd.getAttributeString( 'Arguments' ), '5' )
self.assertEqual( jobClassAd.getAttributeString( 'JobName' ), 'Test_1' )
def submitJob(self, jdl, jobDescriptionObject=None):
"""Submit one job specified by its JDL to WMS.
The JDL may actually be the desciption of a parametric job,
resulting in multiple DIRAC jobs submitted to the DIRAC WMS
"""
if os.path.exists(jdl):
with open(jdl, "r") as fic:
jdlString = fic.read()
else:
# If file JDL does not exist, assume that the JDL is passed as a string
jdlString = jdl
jdlString = jdlString.strip()
gLogger.debug("Submitting JDL", jdlString)
# Strip of comments in the jdl string
newJdlList = []
for line in jdlString.split("\n"):
if not line.strip().startswith("#"):
newJdlList.append(line)
jdlString = "\n".join(newJdlList)
# Check the validity of the input JDL
if jdlString.find("[") != 0:
jdlString = "[%s]" % jdlString
classAdJob = ClassAd(jdlString)
if not classAdJob.isOK():
return S_ERROR(EWMSJDL, "Invalid job JDL")
# Check the size and the contents of the input sandbox
result = self.__uploadInputSandbox(classAdJob, jobDescriptionObject)
if not result["OK"]:
return result
# Submit the job now and get the new job ID
result = getParameterVectorLength(classAdJob)
if not result["OK"]:
return result
nJobs = result["Value"]
result = self.jobManager.submitJob(classAdJob.asJDL())
if nJobs:
gLogger.debug("Applying transactional job submission")
# The server applies transactional bulk submission, we should confirm the jobs
if result["OK"]:
jobIDList = result["Value"]
if len(jobIDList) == nJobs:
# Confirm the submitted jobs
confirmed = False
for _attempt in range(3):
result = self.jobManager.confirmBulkSubmission(jobIDList)
if result["OK"]:
confirmed = True
break
time.sleep(1)
if not confirmed:
# The bulk submission failed, try to remove the created jobs
resultDelete = self.jobManager.removeJob(jobIDList)
error = "Job submission failed to confirm bulk transaction"
if not resultDelete["OK"]:
error += "; removal of created jobs failed"
return S_ERROR(EWMSSUBM, error)
else:
return S_ERROR(EWMSSUBM, "The number of submitted jobs does not match job description")
if result.get("requireProxyUpload"):
gLogger.warn("Need to upload the proxy")
return result
def export_submitJob(self, jobDesc):
""" Submit a job to DIRAC WMS.
The job can be a single job, or a parametric job.
If it is a parametric job, then the parameters will need to be unpacked.
:param str jobDesc: job description JDL (of a single or parametric job)
:return: S_OK/S_ERROR, a list of newly created job IDs in case of S_OK.
"""
if self.peerUsesLimitedProxy:
return S_ERROR(EWMSSUBM, "Can't submit using a limited proxy")
# Check job submission permission
result = self.jobPolicy.getJobPolicy()
if not result['OK']:
return S_ERROR(EWMSSUBM, 'Failed to get job policies')
policyDict = result['Value']
if not policyDict[RIGHT_SUBMIT]:
return S_ERROR(EWMSSUBM, 'Job submission not authorized')
# jobDesc is JDL for now
jobDesc = jobDesc.strip()
if jobDesc[0] != "[":
jobDesc = "[%s" % jobDesc
if jobDesc[-1] != "]":
jobDesc = "%s]" % jobDesc
# Check if the job is a parametric one
jobClassAd = ClassAd(jobDesc)
result = getParameterVectorLength(jobClassAd)
if not result['OK']:
return result
nJobs = result['Value']
parametricJob = False
if nJobs > 0:
# if we are here, then jobDesc was the description of a parametric job. So we start unpacking
parametricJob = True
if nJobs > self.maxParametricJobs:
return S_ERROR(EWMSJDL, "Number of parametric jobs exceeds the limit of %d" % self.maxParametricJobs)
result = generateParametricJobs(jobClassAd)
if not result['OK']:
return result
jobDescList = result['Value']
else:
# if we are here, then jobDesc was the description of a single job.
jobDescList = [jobDesc]
jobIDList = []
if parametricJob:
initialStatus = 'Submitting'
initialMinorStatus = 'Bulk transaction confirmation'
else:
initialStatus = 'Received'
initialMinorStatus = 'Job accepted'
for jobDescription in jobDescList: # jobDescList because there might be a list generated by a parametric job
result = gJobDB.insertNewJobIntoDB(jobDescription,
self.owner,
self.ownerDN,
self.ownerGroup,
self.diracSetup,
initialStatus=initialStatus,
initialMinorStatus=initialMinorStatus)
if not result['OK']:
return result
jobID = result['JobID']
gLogger.info('Job %s added to the JobDB for %s/%s' % (jobID, self.ownerDN, self.ownerGroup))
gJobLoggingDB.addLoggingRecord(jobID, result['Status'], result['MinorStatus'], source='JobManager')
jobIDList.append(jobID)
# Set persistency flag
retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup)
if 'Value' not in retVal or not retVal['Value']:
gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True)
if parametricJob:
result = S_OK(jobIDList)
else:
result = S_OK(jobIDList[0])
result['JobID'] = result['Value']
result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired()
return result
def test_MultiBad(self):
clad = ClassAd(TEST_JDL_MULTI_BAD)
result = getParameterVectorLength(clad)
self.assertTrue(not result['OK'])
def test_MultiBad(self):
clad = ClassAd( TEST_JDL_MULTI_BAD )
result = getParameterVectorLength( clad )
self.assertTrue( not result['OK'] )
def export_submitJob(self, jobDesc):
""" Submit a job to DIRAC WMS.
The job can be a single job, or a parametric job.
If it is a parametric job, then the parameters will need to be unpacked.
:param str jobDesc: job description JDL (of a single or parametric job)
:return: S_OK/S_ERROR, a list of newly created job IDs in case of S_OK.
"""
if self.peerUsesLimitedProxy:
return S_ERROR(EWMSSUBM, "Can't submit using a limited proxy")
# Check job submission permission
result = self.jobPolicy.getJobPolicy()
if not result['OK']:
return S_ERROR(EWMSSUBM, 'Failed to get job policies')
policyDict = result['Value']
if not policyDict[RIGHT_SUBMIT]:
return S_ERROR(EWMSSUBM, 'Job submission not authorized')
# jobDesc is JDL for now
jobDesc = jobDesc.strip()
if jobDesc[0] != "[":
jobDesc = "[%s" % jobDesc
if jobDesc[-1] != "]":
jobDesc = "%s]" % jobDesc
# Check if the job is a parametric one
jobClassAd = ClassAd(jobDesc)
result = getParameterVectorLength(jobClassAd)
if not result['OK']:
gLogger.error("Issue with getParameterVectorLength:", result['Message'])
return result
nJobs = result['Value']
parametricJob = False
if nJobs > 0:
# if we are here, then jobDesc was the description of a parametric job. So we start unpacking
parametricJob = True
if nJobs > self.maxParametricJobs:
gLogger.error("Maximum of parametric jobs exceeded:",
"limit %d smaller than number of jobs %d" % (self.maxParametricJobs, nJobs))
return S_ERROR(EWMSJDL, "Number of parametric jobs exceeds the limit of %d" % self.maxParametricJobs)
result = generateParametricJobs(jobClassAd)
if not result['OK']:
return result
jobDescList = result['Value']
else:
# if we are here, then jobDesc was the description of a single job.
jobDescList = [jobDesc]
jobIDList = []
if parametricJob:
initialStatus = 'Submitting'
initialMinorStatus = 'Bulk transaction confirmation'
else:
initialStatus = 'Received'
initialMinorStatus = 'Job accepted'
for jobDescription in jobDescList: # jobDescList because there might be a list generated by a parametric job
result = gJobDB.insertNewJobIntoDB(jobDescription,
self.owner,
self.ownerDN,
self.ownerGroup,
self.diracSetup,
initialStatus=initialStatus,
initialMinorStatus=initialMinorStatus)
if not result['OK']:
return result
jobID = result['JobID']
gLogger.info('Job %s added to the JobDB for %s/%s' % (jobID, self.ownerDN, self.ownerGroup))
gJobLoggingDB.addLoggingRecord(jobID, result['Status'], result['MinorStatus'], source='JobManager')
jobIDList.append(jobID)
# Set persistency flag
retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup)
if 'Value' not in retVal or not retVal['Value']:
gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True)
if parametricJob:
result = S_OK(jobIDList)
else:
result = S_OK(jobIDList[0])
result['JobID'] = result['Value']
result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired()
# Ensure non-parametric jobs (i.e. non-bulk) get sent to optimizer immediately
if not parametricJob:
self.__sendJobsToOptimizationMind(jobIDList)
return result
def export_submitJob(self, jobDesc):
"""Submit a job to DIRAC WMS.
The job can be a single job, or a parametric job.
If it is a parametric job, then the parameters will need to be unpacked.
:param str jobDesc: job description JDL (of a single or parametric job)
:return: S_OK/S_ERROR, a list of newly created job IDs in case of S_OK.
"""
if self.peerUsesLimitedProxy:
return S_ERROR(EWMSSUBM, "Can't submit using a limited proxy")
# Check job submission permission
result = self.jobPolicy.getJobPolicy()
if not result["OK"]:
return S_ERROR(EWMSSUBM, "Failed to get job policies")
policyDict = result["Value"]
if not policyDict[RIGHT_SUBMIT]:
return S_ERROR(EWMSSUBM, "Job submission not authorized")
# jobDesc is JDL for now
jobDesc = jobDesc.strip()
if jobDesc[0] != "[":
jobDesc = "[%s" % jobDesc
if jobDesc[-1] != "]":
jobDesc = "%s]" % jobDesc
# Check if the job is a parametric one
jobClassAd = ClassAd(jobDesc)
result = getParameterVectorLength(jobClassAd)
if not result["OK"]:
self.log.error("Issue with getParameterVectorLength",
result["Message"])
return result
nJobs = result["Value"]
parametricJob = False
if nJobs is not None and nJobs > 0:
# if we are here, then jobDesc was the description of a parametric job. So we start unpacking
parametricJob = True
if nJobs > self.maxParametricJobs:
self.log.error(
"Maximum of parametric jobs exceeded:",
"limit %d smaller than number of jobs %d" %
(self.maxParametricJobs, nJobs),
)
return S_ERROR(
EWMSJDL,
"Number of parametric jobs exceeds the limit of %d" %
self.maxParametricJobs)
result = generateParametricJobs(jobClassAd)
if not result["OK"]:
return result
jobDescList = result["Value"]
else:
# if we are here, then jobDesc was the description of a single job.
jobDescList = [jobDesc]
jobIDList = []
if parametricJob:
initialStatus = JobStatus.SUBMITTING
initialMinorStatus = "Bulk transaction confirmation"
else:
initialStatus = JobStatus.RECEIVED
initialMinorStatus = "Job accepted"
for jobDescription in jobDescList: # jobDescList because there might be a list generated by a parametric job
result = self.jobDB.insertNewJobIntoDB(
jobDescription,
self.owner,
self.ownerDN,
self.ownerGroup,
self.diracSetup,
initialStatus=initialStatus,
initialMinorStatus=initialMinorStatus,
)
if not result["OK"]:
return result
jobID = result["JobID"]
self.log.info('Job added to the JobDB", "%s for %s/%s' %
(jobID, self.ownerDN, self.ownerGroup))
self.jobLoggingDB.addLoggingRecord(jobID,
result["Status"],
result["MinorStatus"],
source="JobManager")
jobIDList.append(jobID)
# Set persistency flag
retVal = gProxyManager.getUserPersistence(self.ownerDN,
self.ownerGroup)
if "Value" not in retVal or not retVal["Value"]:
gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True)
if parametricJob:
result = S_OK(jobIDList)
else:
result = S_OK(jobIDList[0])
result["JobID"] = result["Value"]
result["requireProxyUpload"] = self.__checkIfProxyUploadIsRequired()
# Ensure non-parametric jobs (i.e. non-bulk) get sent to optimizer immediately
if not parametricJob:
self.__sendJobsToOptimizationMind(jobIDList)
return result
def export_submitJob( self, jobDesc ):
""" Submit a single job to DIRAC WMS
"""
if self.peerUsesLimitedProxy:
return S_ERROR( "Can't submit using a limited proxy! (bad boy!)" )
# Check job submission permission
result = self.jobPolicy.getJobPolicy()
if not result['OK']:
return S_ERROR( 'Failed to get job policies' )
policyDict = result['Value']
if not policyDict[ RIGHT_SUBMIT ]:
return S_ERROR( 'Job submission not authorized' )
#jobDesc is JDL for now
jobDesc = jobDesc.strip()
if jobDesc[0] != "[":
jobDesc = "[%s" % jobDesc
if jobDesc[-1] != "]":
jobDesc = "%s]" % jobDesc
# Check if the job is a parametric one
jobClassAd = ClassAd( jobDesc )
result = getParameterVectorLength( jobClassAd )
if not result['OK']:
return result
nJobs = result['Value']
parametricJob = False
if nJobs > 0:
parametricJob = True
if nJobs > self.maxParametricJobs:
return S_ERROR( EWMSJDL, "Number of parametric jobs exceeds the limit of %d" % self.maxParametricJobs )
result = generateParametricJobs( jobClassAd )
if not result['OK']:
return result
jobDescList = result['Value']
else:
jobDescList = [ jobDesc ]
jobIDList = []
for jobDescription in jobDescList:
result = gJobDB.insertNewJobIntoDB( jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup )
if not result['OK']:
return result
jobID = result['JobID']
gLogger.info( 'Job %s added to the JobDB for %s/%s' % ( jobID, self.ownerDN, self.ownerGroup ) )
gJobLoggingDB.addLoggingRecord( jobID, result['Status'], result['MinorStatus'], source = 'JobManager' )
jobIDList.append( jobID )
#Set persistency flag
retVal = gProxyManager.getUserPersistence( self.ownerDN, self.ownerGroup )
if 'Value' not in retVal or not retVal[ 'Value' ]:
gProxyManager.setPersistency( self.ownerDN, self.ownerGroup, True )
if parametricJob:
result = S_OK( jobIDList )
else:
result = S_OK( jobIDList[0] )
result['JobID'] = result['Value']
result[ 'requireProxyUpload' ] = self.__checkIfProxyUploadIsRequired()
self.__sendJobsToOptimizationMind( jobIDList )
return result
