Ejemplo n.º 1
0
    def test_SimpleProgression(self):

        clad = ClassAd(TEST_JDL_SIMPLE_PROGRESSION)
        nParam = getNumberOfParameters(clad)

        self.assertEqual(nParam, 3)

        result = generateParametricJobs(clad)
        self.assert_(result['OK'])

        jobDescList = result['Value']
        self.assertEqual(nParam, len(jobDescList))

        # Check the definition of the 2nd job
        jobClassAd = ClassAd(jobDescList[1])
        self.assertEqual(jobClassAd.getAttributeString('Arguments'), '3')
        self.assertEqual(jobClassAd.getAttributeString('JobName'), 'Test_1')
Ejemplo n.º 2
0
  def test_SimpleProgression(self):

    clad = ClassAd( TEST_JDL_SIMPLE_PROGRESSION )
    nParam = getNumberOfParameters( clad )

    self.assertEqual( nParam, 3 )

    result = generateParametricJobs( clad )
    self.assert_( result['OK'] )

    jobDescList = result['Value']
    self.assertEqual( nParam, len( jobDescList ) )

    # Check the definition of the 2nd job
    jobClassAd = ClassAd( jobDescList[1] )
    self.assertEqual( jobClassAd.getAttributeString( 'Arguments' ), '3' )
    self.assertEqual( jobClassAd.getAttributeString( 'JobName' ), 'Test_1' )
Ejemplo n.º 3
0
    def test_Simple(self):

        clad = ClassAd(TEST_JDL_SIMPLE)
        result = getParameterVectorLength(clad)
        self.assertTrue(result['OK'])
        nParam = result['Value']

        self.assertEqual(nParam, 3)

        result = generateParametricJobs(clad)
        self.assertTrue(result['OK'])

        jobDescList = result['Value']
        self.assertEqual(nParam, len(jobDescList))

        # Check the definition of the 2nd job
        jobClassAd = ClassAd(jobDescList[1])
        self.assertEqual(jobClassAd.getAttributeString('Arguments'), 'b')
        self.assertEqual(jobClassAd.getAttributeString('JobName'), 'Test_1')
Ejemplo n.º 4
0
    def test_SimpleBunch(self):

        clad = ClassAd(TEST_JDL_SIMPLE_BUNCH)
        result = getParameterVectorLength(clad)
        self.assertTrue(result["OK"])
        nParam = result["Value"]

        self.assertEqual(nParam, 3)

        result = generateParametricJobs(clad)
        self.assertTrue(result["OK"])

        jobDescList = result["Value"]
        self.assertEqual(nParam, len(jobDescList))

        # Check the definition of the 2nd job
        jobClassAd = ClassAd(jobDescList[1])
        self.assertEqual(jobClassAd.getAttributeString("Arguments"), "5")
        self.assertEqual(jobClassAd.getAttributeString("JobName"), "Test_1")
Ejemplo n.º 5
0
  def test_SimpleBunch(self):

    clad = ClassAd( TEST_JDL_SIMPLE_BUNCH )
    result = getParameterVectorLength( clad )
    self.assertTrue( result['OK'] )
    nParam = result['Value']

    self.assertEqual( nParam, 3 )

    result = generateParametricJobs( clad )
    self.assertTrue(result['OK'])

    jobDescList = result['Value']
    self.assertEqual( nParam, len( jobDescList ) )

    # Check the definition of the 2nd job
    jobClassAd = ClassAd( jobDescList[1] )
    self.assertEqual( jobClassAd.getAttributeString( 'Arguments' ), '5' )
    self.assertEqual( jobClassAd.getAttributeString( 'JobName' ), 'Test_1' )
Ejemplo n.º 6
0
    def export_submitJob(self, jobDesc):
        """ Submit a single job to DIRAC WMS
    """

        if self.peerUsesLimitedProxy:
            return S_ERROR("Can't submit using a limited proxy! (bad boy!)")

        # Check job submission permission
        result = self.jobPolicy.getJobPolicy()
        if not result['OK']:
            return S_ERROR('Failed to get job policies')
        policyDict = result['Value']
        if not policyDict[RIGHT_SUBMIT]:
            return S_ERROR('Job submission not authorized')

        #jobDesc is JDL for now
        jobDesc = jobDesc.strip()
        if jobDesc[0] != "[":
            jobDesc = "[%s" % jobDesc
        if jobDesc[-1] != "]":
            jobDesc = "%s]" % jobDesc

        # Check if the job is a parametric one
        jobClassAd = ClassAd(jobDesc)
        nParameters = getNumberOfParameters(jobClassAd)
        parametricJob = False
        if nParameters > 0:
            parametricJob = True
            result = generateParametricJobs(jobClassAd)
            if not result['OK']:
                return result
            jobDescList = result['Value']
        else:
            jobDescList = [jobDesc]

        jobIDList = []
        for jobDescription in jobDescList:
            result = gJobDB.insertNewJobIntoDB(jobDescription, self.owner,
                                               self.ownerDN, self.ownerGroup,
                                               self.diracSetup)
            if not result['OK']:
                return result

            jobID = result['JobID']
            gLogger.info('Job %s added to the JobDB for %s/%s' %
                         (jobID, self.ownerDN, self.ownerGroup))

            gJobLoggingDB.addLoggingRecord(jobID,
                                           result['Status'],
                                           result['MinorStatus'],
                                           source='JobManager')

            jobIDList.append(jobID)

        #Set persistency flag
        retVal = gProxyManager.getUserPersistence(self.ownerDN,
                                                  self.ownerGroup)
        if 'Value' not in retVal or not retVal['Value']:
            gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True)

        if parametricJob:
            result = S_OK(jobIDList)
        else:
            result = S_OK(jobIDList[0])

        result['JobID'] = result['Value']
        result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired()
        self.__sendJobsToOptimizationMind(jobIDList)
        return result
Ejemplo n.º 7
0
  def export_submitJob(self, jobDesc):
    """ Submit a job to DIRAC WMS.
        The job can be a single job, or a parametric job.
        If it is a parametric job, then the parameters will need to be unpacked.

        :param str jobDesc: job description JDL (of a single or parametric job)
        :return: S_OK/S_ERROR, a list of newly created job IDs in case of S_OK.
    """

    if self.peerUsesLimitedProxy:
      return S_ERROR(EWMSSUBM, "Can't submit using a limited proxy")

    # Check job submission permission
    result = self.jobPolicy.getJobPolicy()
    if not result['OK']:
      return S_ERROR(EWMSSUBM, 'Failed to get job policies')
    policyDict = result['Value']
    if not policyDict[RIGHT_SUBMIT]:
      return S_ERROR(EWMSSUBM, 'Job submission not authorized')

    # jobDesc is JDL for now
    jobDesc = jobDesc.strip()
    if jobDesc[0] != "[":
      jobDesc = "[%s" % jobDesc
    if jobDesc[-1] != "]":
      jobDesc = "%s]" % jobDesc

    # Check if the job is a parametric one
    jobClassAd = ClassAd(jobDesc)
    result = getParameterVectorLength(jobClassAd)
    if not result['OK']:
      return result
    nJobs = result['Value']
    parametricJob = False
    if nJobs > 0:
      # if we are here, then jobDesc was the description of a parametric job. So we start unpacking
      parametricJob = True
      if nJobs > self.maxParametricJobs:
        return S_ERROR(EWMSJDL, "Number of parametric jobs exceeds the limit of %d" % self.maxParametricJobs)
      result = generateParametricJobs(jobClassAd)
      if not result['OK']:
        return result
      jobDescList = result['Value']
    else:
      # if we are here, then jobDesc was the description of a single job.
      jobDescList = [jobDesc]

    jobIDList = []

    if parametricJob:
      initialStatus = 'Submitting'
      initialMinorStatus = 'Bulk transaction confirmation'
    else:
      initialStatus = 'Received'
      initialMinorStatus = 'Job accepted'

    for jobDescription in jobDescList:  # jobDescList because there might be a list generated by a parametric job
      result = gJobDB.insertNewJobIntoDB(jobDescription,
                                         self.owner,
                                         self.ownerDN,
                                         self.ownerGroup,
                                         self.diracSetup,
                                         initialStatus=initialStatus,
                                         initialMinorStatus=initialMinorStatus)
      if not result['OK']:
        return result

      jobID = result['JobID']
      gLogger.info('Job %s added to the JobDB for %s/%s' % (jobID, self.ownerDN, self.ownerGroup))

      gJobLoggingDB.addLoggingRecord(jobID, result['Status'], result['MinorStatus'], source='JobManager')

      jobIDList.append(jobID)

    # Set persistency flag
    retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup)
    if 'Value' not in retVal or not retVal['Value']:
      gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True)

    if parametricJob:
      result = S_OK(jobIDList)
    else:
      result = S_OK(jobIDList[0])

    result['JobID'] = result['Value']
    result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired()
    return result
Ejemplo n.º 8
0
    def export_submitJob(self, jobDesc):
        """ Submit a single job to DIRAC WMS
    """

        if self.peerUsesLimitedProxy:
            return S_ERROR("Can't submit using a limited proxy! (bad boy!)")

        # Check job submission permission
        result = self.jobPolicy.getJobPolicy()
        if not result["OK"]:
            return S_ERROR("Failed to get job policies")
        policyDict = result["Value"]
        if not policyDict[RIGHT_SUBMIT]:
            return S_ERROR("Job submission not authorized")

        # jobDesc is JDL for now
        jobDesc = jobDesc.strip()
        if jobDesc[0] != "[":
            jobDesc = "[%s" % jobDesc
        if jobDesc[-1] != "]":
            jobDesc = "%s]" % jobDesc

        # Check if the job is a parametric one
        jobClassAd = ClassAd(jobDesc)
        nParameters = getNumberOfParameters(jobClassAd)
        parametricJob = False
        if nParameters > 0:
            parametricJob = True
            result = generateParametricJobs(jobClassAd)
            if not result["OK"]:
                return result
            jobDescList = result["Value"]
        else:
            jobDescList = [jobDesc]

        jobIDList = []
        for jobDescription in jobDescList:
            result = gJobDB.insertNewJobIntoDB(
                jobDescription, self.owner, self.ownerDN, self.ownerGroup, self.diracSetup
            )
            if not result["OK"]:
                return result

            jobID = result["JobID"]
            gLogger.info("Job %s added to the JobDB for %s/%s" % (jobID, self.ownerDN, self.ownerGroup))

            gJobLoggingDB.addLoggingRecord(jobID, result["Status"], result["MinorStatus"], source="JobManager")

            jobIDList.append(jobID)

        # Set persistency flag
        retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup)
        if "Value" not in retVal or not retVal["Value"]:
            gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True)

        if parametricJob:
            result = S_OK(jobIDList)
        else:
            result = S_OK(jobIDList[0])

        result["JobID"] = result["Value"]
        result["requireProxyUpload"] = self.__checkIfProxyUploadIsRequired()
        self.__sendJobsToOptimizationMind(jobIDList)
        return result
Ejemplo n.º 9
0
  def export_submitJob(self, jobDesc):
    """ Submit a job to DIRAC WMS.
        The job can be a single job, or a parametric job.
        If it is a parametric job, then the parameters will need to be unpacked.

        :param str jobDesc: job description JDL (of a single or parametric job)
        :return: S_OK/S_ERROR, a list of newly created job IDs in case of S_OK.
    """

    if self.peerUsesLimitedProxy:
      return S_ERROR(EWMSSUBM, "Can't submit using a limited proxy")

    # Check job submission permission
    result = self.jobPolicy.getJobPolicy()
    if not result['OK']:
      return S_ERROR(EWMSSUBM, 'Failed to get job policies')
    policyDict = result['Value']
    if not policyDict[RIGHT_SUBMIT]:
      return S_ERROR(EWMSSUBM, 'Job submission not authorized')

    # jobDesc is JDL for now
    jobDesc = jobDesc.strip()
    if jobDesc[0] != "[":
      jobDesc = "[%s" % jobDesc
    if jobDesc[-1] != "]":
      jobDesc = "%s]" % jobDesc

    # Check if the job is a parametric one
    jobClassAd = ClassAd(jobDesc)
    result = getParameterVectorLength(jobClassAd)
    if not result['OK']:
      gLogger.error("Issue with getParameterVectorLength:", result['Message'])
      return result
    nJobs = result['Value']
    parametricJob = False
    if nJobs > 0:
      # if we are here, then jobDesc was the description of a parametric job. So we start unpacking
      parametricJob = True
      if nJobs > self.maxParametricJobs:
        gLogger.error("Maximum of parametric jobs exceeded:",
                      "limit %d smaller than number of jobs %d" % (self.maxParametricJobs, nJobs))
        return S_ERROR(EWMSJDL, "Number of parametric jobs exceeds the limit of %d" % self.maxParametricJobs)
      result = generateParametricJobs(jobClassAd)
      if not result['OK']:
        return result
      jobDescList = result['Value']
    else:
      # if we are here, then jobDesc was the description of a single job.
      jobDescList = [jobDesc]

    jobIDList = []

    if parametricJob:
      initialStatus = 'Submitting'
      initialMinorStatus = 'Bulk transaction confirmation'
    else:
      initialStatus = 'Received'
      initialMinorStatus = 'Job accepted'

    for jobDescription in jobDescList:  # jobDescList because there might be a list generated by a parametric job
      result = gJobDB.insertNewJobIntoDB(jobDescription,
                                         self.owner,
                                         self.ownerDN,
                                         self.ownerGroup,
                                         self.diracSetup,
                                         initialStatus=initialStatus,
                                         initialMinorStatus=initialMinorStatus)
      if not result['OK']:
        return result

      jobID = result['JobID']
      gLogger.info('Job %s added to the JobDB for %s/%s' % (jobID, self.ownerDN, self.ownerGroup))

      gJobLoggingDB.addLoggingRecord(jobID, result['Status'], result['MinorStatus'], source='JobManager')

      jobIDList.append(jobID)

    # Set persistency flag
    retVal = gProxyManager.getUserPersistence(self.ownerDN, self.ownerGroup)
    if 'Value' not in retVal or not retVal['Value']:
      gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True)

    if parametricJob:
      result = S_OK(jobIDList)
    else:
      result = S_OK(jobIDList[0])

    result['JobID'] = result['Value']
    result['requireProxyUpload'] = self.__checkIfProxyUploadIsRequired()
    # Ensure non-parametric jobs (i.e. non-bulk) get sent to optimizer immediately
    if not parametricJob:
      self.__sendJobsToOptimizationMind(jobIDList)
    return result
Ejemplo n.º 10
0
    def export_submitJob(self, jobDesc):
        """Submit a job to DIRAC WMS.
        The job can be a single job, or a parametric job.
        If it is a parametric job, then the parameters will need to be unpacked.

        :param str jobDesc: job description JDL (of a single or parametric job)
        :return: S_OK/S_ERROR, a list of newly created job IDs in case of S_OK.
        """

        if self.peerUsesLimitedProxy:
            return S_ERROR(EWMSSUBM, "Can't submit using a limited proxy")

        # Check job submission permission
        result = self.jobPolicy.getJobPolicy()
        if not result["OK"]:
            return S_ERROR(EWMSSUBM, "Failed to get job policies")
        policyDict = result["Value"]
        if not policyDict[RIGHT_SUBMIT]:
            return S_ERROR(EWMSSUBM, "Job submission not authorized")

        # jobDesc is JDL for now
        jobDesc = jobDesc.strip()
        if jobDesc[0] != "[":
            jobDesc = "[%s" % jobDesc
        if jobDesc[-1] != "]":
            jobDesc = "%s]" % jobDesc

        # Check if the job is a parametric one
        jobClassAd = ClassAd(jobDesc)
        result = getParameterVectorLength(jobClassAd)
        if not result["OK"]:
            self.log.error("Issue with getParameterVectorLength",
                           result["Message"])
            return result
        nJobs = result["Value"]
        parametricJob = False
        if nJobs is not None and nJobs > 0:
            # if we are here, then jobDesc was the description of a parametric job. So we start unpacking
            parametricJob = True
            if nJobs > self.maxParametricJobs:
                self.log.error(
                    "Maximum of parametric jobs exceeded:",
                    "limit %d smaller than number of jobs %d" %
                    (self.maxParametricJobs, nJobs),
                )
                return S_ERROR(
                    EWMSJDL,
                    "Number of parametric jobs exceeds the limit of %d" %
                    self.maxParametricJobs)
            result = generateParametricJobs(jobClassAd)
            if not result["OK"]:
                return result
            jobDescList = result["Value"]
        else:
            # if we are here, then jobDesc was the description of a single job.
            jobDescList = [jobDesc]

        jobIDList = []

        if parametricJob:
            initialStatus = JobStatus.SUBMITTING
            initialMinorStatus = "Bulk transaction confirmation"
        else:
            initialStatus = JobStatus.RECEIVED
            initialMinorStatus = "Job accepted"

        for jobDescription in jobDescList:  # jobDescList because there might be a list generated by a parametric job
            result = self.jobDB.insertNewJobIntoDB(
                jobDescription,
                self.owner,
                self.ownerDN,
                self.ownerGroup,
                self.diracSetup,
                initialStatus=initialStatus,
                initialMinorStatus=initialMinorStatus,
            )
            if not result["OK"]:
                return result

            jobID = result["JobID"]
            self.log.info('Job added to the JobDB", "%s for %s/%s' %
                          (jobID, self.ownerDN, self.ownerGroup))

            self.jobLoggingDB.addLoggingRecord(jobID,
                                               result["Status"],
                                               result["MinorStatus"],
                                               source="JobManager")

            jobIDList.append(jobID)

        # Set persistency flag
        retVal = gProxyManager.getUserPersistence(self.ownerDN,
                                                  self.ownerGroup)
        if "Value" not in retVal or not retVal["Value"]:
            gProxyManager.setPersistency(self.ownerDN, self.ownerGroup, True)

        if parametricJob:
            result = S_OK(jobIDList)
        else:
            result = S_OK(jobIDList[0])

        result["JobID"] = result["Value"]
        result["requireProxyUpload"] = self.__checkIfProxyUploadIsRequired()
        # Ensure non-parametric jobs (i.e. non-bulk) get sent to optimizer immediately
        if not parametricJob:
            self.__sendJobsToOptimizationMind(jobIDList)
        return result