def test_SimpleProgression(self): clad = ClassAd(TEST_JDL_SIMPLE_PROGRESSION) nParam = getNumberOfParameters(clad) self.assertEqual(nParam, 3) result = generateParametricJobs(clad) self.assert_(result['OK']) jobDescList = result['Value'] self.assertEqual(nParam, len(jobDescList)) # Check the definition of the 2nd job jobClassAd = ClassAd(jobDescList[1]) self.assertEqual(jobClassAd.getAttributeString('Arguments'), '3') self.assertEqual(jobClassAd.getAttributeString('JobName'), 'Test_1')
def test_SimpleProgression(self): clad = ClassAd( TEST_JDL_SIMPLE_PROGRESSION ) nParam = getNumberOfParameters( clad ) self.assertEqual( nParam, 3 ) result = generateParametricJobs( clad ) self.assert_( result['OK'] ) jobDescList = result['Value'] self.assertEqual( nParam, len( jobDescList ) ) # Check the definition of the 2nd job jobClassAd = ClassAd( jobDescList[1] ) self.assertEqual( jobClassAd.getAttributeString( 'Arguments' ), '3' ) self.assertEqual( jobClassAd.getAttributeString( 'JobName' ), 'Test_1' )
def test_Simple(self): clad = ClassAd(TEST_JDL_SIMPLE) result = getParameterVectorLength(clad) self.assertTrue(result['OK']) nParam = result['Value'] self.assertEqual(nParam, 3) result = generateParametricJobs(clad) self.assertTrue(result['OK']) jobDescList = result['Value'] self.assertEqual(nParam, len(jobDescList)) # Check the definition of the 2nd job jobClassAd = ClassAd(jobDescList[1]) self.assertEqual(jobClassAd.getAttributeString('Arguments'), 'b') self.assertEqual(jobClassAd.getAttributeString('JobName'), 'Test_1')
def test_SimpleBunch(self): clad = ClassAd(TEST_JDL_SIMPLE_BUNCH) result = getParameterVectorLength(clad) self.assertTrue(result["OK"]) nParam = result["Value"] self.assertEqual(nParam, 3) result = generateParametricJobs(clad) self.assertTrue(result["OK"]) jobDescList = result["Value"] self.assertEqual(nParam, len(jobDescList)) # Check the definition of the 2nd job jobClassAd = ClassAd(jobDescList[1]) self.assertEqual(jobClassAd.getAttributeString("Arguments"), "5") self.assertEqual(jobClassAd.getAttributeString("JobName"), "Test_1")
def test_SimpleBunch(self): clad = ClassAd( TEST_JDL_SIMPLE_BUNCH ) result = getParameterVectorLength( clad ) self.assertTrue( result['OK'] ) nParam = result['Value'] self.assertEqual( nParam, 3 ) result = generateParametricJobs( clad ) self.assertTrue(result['OK']) jobDescList = result['Value'] self.assertEqual( nParam, len( jobDescList ) ) # Check the definition of the 2nd job jobClassAd = ClassAd( jobDescList[1] ) self.assertEqual( jobClassAd.getAttributeString( 'Arguments' ), '5' ) self.assertEqual( jobClassAd.getAttributeString( 'JobName' ), 'Test_1' )
def export_submitJob(self, jobDesc): """ Submit a single job to DIRAC WMS """ if self.peerUsesLimitedProxy: return S_ERROR("Can't submit using a limited proxy! (bad boy!)") # Check job submission permission result = self.jobPolicy.getJobPolicy() if not result['OK']: return S_ERROR('Failed to get job policies') policyDict = result['Value'] if not policyDict[RIGHT_SUBMIT]: return S_ERROR('Job submission not authorized') #jobDesc is JDL for now jobDesc = jobDesc.strip() if jobDesc[0] != "[": jobDesc = "[%s" % jobDesc if jobDesc[-1] != "]": jobDesc = "%s]" % jobDesc # Check if the job is a parametric one jobClassAd = ClassAd(jobDesc) nParameters = getNumberOfParameters(jobClassAd) parametricJob = False if nParameters > 0: parametricJob = True result = generateParametricJobs(jobClassAd) if not result['OK']: return result jobDescList = result['Value'] else: jobDescList = [jobDesc] jobIDList = [] for jobDescription in jobDescList: result = gJobDB.insertNewJobIntoDB(jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup) if not result['OK']: return result jobID = result['JobID'] gLogger.info('Job %s added to the JobDB for %s/%s' % (jobID, self.ownerDN, self.ownerGroup)) gJobLoggingDB.addLoggingRecord(jobID, result['Status'], result['MinorStatus'], source='JobManager') jobIDList.append(jobID) #Set persistency flag retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup) if 'Value' not in retVal or not retVal['Value']: gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True) if parametricJob: result = S_OK(jobIDList) else: result = S_OK(jobIDList[0]) result['JobID'] = result['Value'] result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired() self.__sendJobsToOptimizationMind(jobIDList) return result
def export_submitJob(self, jobDesc): """ Submit a job to DIRAC WMS. The job can be a single job, or a parametric job. If it is a parametric job, then the parameters will need to be unpacked. :param str jobDesc: job description JDL (of a single or parametric job) :return: S_OK/S_ERROR, a list of newly created job IDs in case of S_OK. """ if self.peerUsesLimitedProxy: return S_ERROR(EWMSSUBM, "Can't submit using a limited proxy") # Check job submission permission result = self.jobPolicy.getJobPolicy() if not result['OK']: return S_ERROR(EWMSSUBM, 'Failed to get job policies') policyDict = result['Value'] if not policyDict[RIGHT_SUBMIT]: return S_ERROR(EWMSSUBM, 'Job submission not authorized') # jobDesc is JDL for now jobDesc = jobDesc.strip() if jobDesc[0] != "[": jobDesc = "[%s" % jobDesc if jobDesc[-1] != "]": jobDesc = "%s]" % jobDesc # Check if the job is a parametric one jobClassAd = ClassAd(jobDesc) result = getParameterVectorLength(jobClassAd) if not result['OK']: return result nJobs = result['Value'] parametricJob = False if nJobs > 0: # if we are here, then jobDesc was the description of a parametric job. So we start unpacking parametricJob = True if nJobs > self.maxParametricJobs: return S_ERROR(EWMSJDL, "Number of parametric jobs exceeds the limit of %d" % self.maxParametricJobs) result = generateParametricJobs(jobClassAd) if not result['OK']: return result jobDescList = result['Value'] else: # if we are here, then jobDesc was the description of a single job. jobDescList = [jobDesc] jobIDList = [] if parametricJob: initialStatus = 'Submitting' initialMinorStatus = 'Bulk transaction confirmation' else: initialStatus = 'Received' initialMinorStatus = 'Job accepted' for jobDescription in jobDescList: # jobDescList because there might be a list generated by a parametric job result = gJobDB.insertNewJobIntoDB(jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup, initialStatus=initialStatus, initialMinorStatus=initialMinorStatus) if not result['OK']: return result jobID = result['JobID'] gLogger.info('Job %s added to the JobDB for %s/%s' % (jobID, self.ownerDN, self.ownerGroup)) gJobLoggingDB.addLoggingRecord(jobID, result['Status'], result['MinorStatus'], source='JobManager') jobIDList.append(jobID) # Set persistency flag retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup) if 'Value' not in retVal or not retVal['Value']: gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True) if parametricJob: result = S_OK(jobIDList) else: result = S_OK(jobIDList[0]) result['JobID'] = result['Value'] result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired() return result
def export_submitJob(self, jobDesc): """ Submit a single job to DIRAC WMS """ if self.peerUsesLimitedProxy: return S_ERROR("Can't submit using a limited proxy! (bad boy!)") # Check job submission permission result = self.jobPolicy.getJobPolicy() if not result["OK"]: return S_ERROR("Failed to get job policies") policyDict = result["Value"] if not policyDict[RIGHT_SUBMIT]: return S_ERROR("Job submission not authorized") # jobDesc is JDL for now jobDesc = jobDesc.strip() if jobDesc[0] != "[": jobDesc = "[%s" % jobDesc if jobDesc[-1] != "]": jobDesc = "%s]" % jobDesc # Check if the job is a parametric one jobClassAd = ClassAd(jobDesc) nParameters = getNumberOfParameters(jobClassAd) parametricJob = False if nParameters > 0: parametricJob = True result = generateParametricJobs(jobClassAd) if not result["OK"]: return result jobDescList = result["Value"] else: jobDescList = [jobDesc] jobIDList = [] for jobDescription in jobDescList: result = gJobDB.insertNewJobIntoDB( jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup ) if not result["OK"]: return result jobID = result["JobID"] gLogger.info("Job %s added to the JobDB for %s/%s" % (jobID, self.ownerDN, self.ownerGroup)) gJobLoggingDB.addLoggingRecord(jobID, result["Status"], result["MinorStatus"], source="JobManager") jobIDList.append(jobID) # Set persistency flag retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup) if "Value" not in retVal or not retVal["Value"]: gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True) if parametricJob: result = S_OK(jobIDList) else: result = S_OK(jobIDList[0]) result["JobID"] = result["Value"] result["requireProxyUpload"] = self.__checkIfProxyUploadIsRequired() self.__sendJobsToOptimizationMind(jobIDList) return result
def export_submitJob(self, jobDesc): """ Submit a job to DIRAC WMS. The job can be a single job, or a parametric job. If it is a parametric job, then the parameters will need to be unpacked. :param str jobDesc: job description JDL (of a single or parametric job) :return: S_OK/S_ERROR, a list of newly created job IDs in case of S_OK. """ if self.peerUsesLimitedProxy: return S_ERROR(EWMSSUBM, "Can't submit using a limited proxy") # Check job submission permission result = self.jobPolicy.getJobPolicy() if not result['OK']: return S_ERROR(EWMSSUBM, 'Failed to get job policies') policyDict = result['Value'] if not policyDict[RIGHT_SUBMIT]: return S_ERROR(EWMSSUBM, 'Job submission not authorized') # jobDesc is JDL for now jobDesc = jobDesc.strip() if jobDesc[0] != "[": jobDesc = "[%s" % jobDesc if jobDesc[-1] != "]": jobDesc = "%s]" % jobDesc # Check if the job is a parametric one jobClassAd = ClassAd(jobDesc) result = getParameterVectorLength(jobClassAd) if not result['OK']: gLogger.error("Issue with getParameterVectorLength:", result['Message']) return result nJobs = result['Value'] parametricJob = False if nJobs > 0: # if we are here, then jobDesc was the description of a parametric job. So we start unpacking parametricJob = True if nJobs > self.maxParametricJobs: gLogger.error("Maximum of parametric jobs exceeded:", "limit %d smaller than number of jobs %d" % (self.maxParametricJobs, nJobs)) return S_ERROR(EWMSJDL, "Number of parametric jobs exceeds the limit of %d" % self.maxParametricJobs) result = generateParametricJobs(jobClassAd) if not result['OK']: return result jobDescList = result['Value'] else: # if we are here, then jobDesc was the description of a single job. jobDescList = [jobDesc] jobIDList = [] if parametricJob: initialStatus = 'Submitting' initialMinorStatus = 'Bulk transaction confirmation' else: initialStatus = 'Received' initialMinorStatus = 'Job accepted' for jobDescription in jobDescList: # jobDescList because there might be a list generated by a parametric job result = gJobDB.insertNewJobIntoDB(jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup, initialStatus=initialStatus, initialMinorStatus=initialMinorStatus) if not result['OK']: return result jobID = result['JobID'] gLogger.info('Job %s added to the JobDB for %s/%s' % (jobID, self.ownerDN, self.ownerGroup)) gJobLoggingDB.addLoggingRecord(jobID, result['Status'], result['MinorStatus'], source='JobManager') jobIDList.append(jobID) # Set persistency flag retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup) if 'Value' not in retVal or not retVal['Value']: gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True) if parametricJob: result = S_OK(jobIDList) else: result = S_OK(jobIDList[0]) result['JobID'] = result['Value'] result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired() # Ensure non-parametric jobs (i.e. non-bulk) get sent to optimizer immediately if not parametricJob: self.__sendJobsToOptimizationMind(jobIDList) return result
def export_submitJob(self, jobDesc): """Submit a job to DIRAC WMS. The job can be a single job, or a parametric job. If it is a parametric job, then the parameters will need to be unpacked. :param str jobDesc: job description JDL (of a single or parametric job) :return: S_OK/S_ERROR, a list of newly created job IDs in case of S_OK. """ if self.peerUsesLimitedProxy: return S_ERROR(EWMSSUBM, "Can't submit using a limited proxy") # Check job submission permission result = self.jobPolicy.getJobPolicy() if not result["OK"]: return S_ERROR(EWMSSUBM, "Failed to get job policies") policyDict = result["Value"] if not policyDict[RIGHT_SUBMIT]: return S_ERROR(EWMSSUBM, "Job submission not authorized") # jobDesc is JDL for now jobDesc = jobDesc.strip() if jobDesc[0] != "[": jobDesc = "[%s" % jobDesc if jobDesc[-1] != "]": jobDesc = "%s]" % jobDesc # Check if the job is a parametric one jobClassAd = ClassAd(jobDesc) result = getParameterVectorLength(jobClassAd) if not result["OK"]: self.log.error("Issue with getParameterVectorLength", result["Message"]) return result nJobs = result["Value"] parametricJob = False if nJobs is not None and nJobs > 0: # if we are here, then jobDesc was the description of a parametric job. So we start unpacking parametricJob = True if nJobs > self.maxParametricJobs: self.log.error( "Maximum of parametric jobs exceeded:", "limit %d smaller than number of jobs %d" % (self.maxParametricJobs, nJobs), ) return S_ERROR( EWMSJDL, "Number of parametric jobs exceeds the limit of %d" % self.maxParametricJobs) result = generateParametricJobs(jobClassAd) if not result["OK"]: return result jobDescList = result["Value"] else: # if we are here, then jobDesc was the description of a single job. jobDescList = [jobDesc] jobIDList = [] if parametricJob: initialStatus = JobStatus.SUBMITTING initialMinorStatus = "Bulk transaction confirmation" else: initialStatus = JobStatus.RECEIVED initialMinorStatus = "Job accepted" for jobDescription in jobDescList: # jobDescList because there might be a list generated by a parametric job result = self.jobDB.insertNewJobIntoDB( jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup, initialStatus=initialStatus, initialMinorStatus=initialMinorStatus, ) if not result["OK"]: return result jobID = result["JobID"] self.log.info('Job added to the JobDB", "%s for %s/%s' % (jobID, self.ownerDN, self.ownerGroup)) self.jobLoggingDB.addLoggingRecord(jobID, result["Status"], result["MinorStatus"], source="JobManager") jobIDList.append(jobID) # Set persistency flag retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup) if "Value" not in retVal or not retVal["Value"]: gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True) if parametricJob: result = S_OK(jobIDList) else: result = S_OK(jobIDList[0]) result["JobID"] = result["Value"] result["requireProxyUpload"] = self.__checkIfProxyUploadIsRequired() # Ensure non-parametric jobs (i.e. non-bulk) get sent to optimizer immediately if not parametricJob: self.__sendJobsToOptimizationMind(jobIDList) return result