Ejemplo n.º 1
0
def test_basicJob():
    job = Job()

    job.setOwner('ownerName')
    job.setOwnerGroup('ownerGroup')
    job.setName('jobName')
    job.setJobGroup('jobGroup')
    job.setExecutable('someExe')
    job.setType('jobType')
    job.setDestination('ANY')

    xml = job._toXML()

    try:
        with open('./DIRAC/Interfaces/API/test/testWF.xml') as fd:
            expected = fd.read()
    except IOError:
        with open('./Interfaces/API/test/testWF.xml') as fd:
            expected = fd.read()

    assert xml == expected

    try:
        with open('./DIRAC/Interfaces/API/test/testWFSIO.jdl') as fd:
            expected = fd.read()
    except IOError:
        with open('./Interfaces/API/test/testWFSIO.jdl') as fd:
            expected = fd.read()

    jdlSIO = job._toJDL(jobDescriptionObject=StringIO.StringIO(job._toXML()))
    assert jdlSIO == expected
Ejemplo n.º 2
0
def test_basicJob():
  job = Job()

  job.setOwner('ownerName')
  job.setOwnerGroup('ownerGroup')
  job.setName('jobName')
  job.setJobGroup('jobGroup')
  job.setExecutable('someExe')
  job.setType('jobType')
  job.setDestination('ANY')

  xml = job._toXML()

  try:
    with open('./DIRAC/Interfaces/API/test/testWF.xml') as fd:
      expected = fd.read()
  except IOError:
    with open('./Interfaces/API/test/testWF.xml') as fd:
      expected = fd.read()

  assert xml == expected

  try:
    with open('./DIRAC/Interfaces/API/test/testWFSIO.jdl') as fd:
      expected = fd.read()
  except IOError:
    with open('./Interfaces/API/test/testWFSIO.jdl') as fd:
      expected = fd.read()

  jdlSIO = job._toJDL(jobDescriptionObject=StringIO.StringIO(job._toXML()))
  assert jdlSIO == expected
Ejemplo n.º 3
0
  def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''):
    if (not owner) or (not ownerGroup):
      res = getProxyInfo(False,False)
      if not res['OK']:
        return res
      proxyInfo = res['Value']
      owner = proxyInfo['username']
      ownerGroup = proxyInfo['group']

    oJob = Job(transBody)
    for taskNumber in sortList(taskDict.keys()):
      paramsDict = taskDict[taskNumber]
      transID = paramsDict['TransformationID']
      self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup))
      oJob.setOwner(owner)
      oJob.setOwnerGroup(ownerGroup)
      transGroup = str(transID).zfill(8)
      self.log.verbose('Adding default transformation group of %s' % (transGroup))
      oJob.setJobGroup(transGroup)
      constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8)
      self.log.verbose('Setting task name to %s' % constructedName)
      oJob.setName(constructedName)
      oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8))
      oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8))
      inputData = None
      for paramName,paramValue in paramsDict.items():
        self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue))
        if paramName=='InputData':
          if paramValue:
            self.log.verbose('Setting input data to %s' %paramValue)
            oJob.setInputData(paramValue)
        elif paramName=='Site':
          if paramValue:
            self.log.verbose('Setting allocated site to: %s' %(paramValue))
            oJob.setDestination(paramValue)
        elif paramValue:
          self.log.verbose('Setting %s to %s' % (paramName,paramValue))
          oJob._addJDLParameter(paramName,paramValue)

      hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])]
      if int(transID) in hospitalTrans:
        hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch')
        hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[])
        oJob.setType('Hospital')
        oJob.setDestination(hospitalSite)
        oJob.setInputDataPolicy('download',dataScheduling=False)
        if hospitalCEs:
          oJob._addJDLParameter('GridRequiredCEs',hospitalCEs)        
      taskDict[taskNumber]['TaskObject'] = '' 
      res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData})
      if not res ['OK']:
        self.log.error("Failed to generate output data",res['Message'])
        continue
      for name,output in res['Value'].items():
        oJob._addJDLParameter(name,string.join(output,';'))
      taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML())
    return S_OK(taskDict)
Ejemplo n.º 4
0
  def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''):
    if (not owner) or (not ownerGroup):
      res = getProxyInfo(False,False)
      if not res['OK']:
        return res
      proxyInfo = res['Value']
      owner = proxyInfo['username']
      ownerGroup = proxyInfo['group']

    oJob = Job(transBody)
    for taskNumber in sortList(taskDict.keys()):
      paramsDict = taskDict[taskNumber]
      transID = paramsDict['TransformationID']
      self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup))
      oJob.setOwner(owner)
      oJob.setOwnerGroup(ownerGroup)
      transGroup = str(transID).zfill(8)
      self.log.verbose('Adding default transformation group of %s' % (transGroup))
      oJob.setJobGroup(transGroup)
      constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8)
      self.log.verbose('Setting task name to %s' % constructedName)
      oJob.setName(constructedName)
      oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8))
      oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8))
      inputData = None
      for paramName,paramValue in paramsDict.items():
        self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue))
        if paramName=='InputData':
          if paramValue:
            self.log.verbose('Setting input data to %s' %paramValue)
            oJob.setInputData(paramValue)
        elif paramName=='Site':
          if paramValue:
            self.log.verbose('Setting allocated site to: %s' %(paramValue))
            oJob.setDestination(paramValue)
        elif paramValue:
          self.log.verbose('Setting %s to %s' % (paramName,paramValue))
          oJob._addJDLParameter(paramName,paramValue)

      hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])]
      if int(transID) in hospitalTrans:
        hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch')
        hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[])
        oJob.setType('Hospital')
        oJob.setDestination(hospitalSite)
        oJob.setInputDataPolicy('download',dataScheduling=False)
        if hospitalCEs:
          oJob._addJDLParameter('GridRequiredCEs',hospitalCEs)        
      taskDict[taskNumber]['TaskObject'] = '' 
      res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData})
      if not res ['OK']:
        self.log.error("Failed to generate output data",res['Message'])
        continue
      for name,output in res['Value'].items():
        oJob._addJDLParameter(name,string.join(output,';'))
      taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML())
    return S_OK(taskDict)
Ejemplo n.º 5
0
def test_basicJob():
    job = Job()

    job.setOwner("ownerName")
    job.setOwnerGroup("ownerGroup")
    job.setName("jobName")
    job.setJobGroup("jobGroup")
    job.setExecutable("someExe")
    job.setType("jobType")
    job.setDestination("ANY")

    xml = job._toXML()

    with open(join(dirname(__file__), "testWF.xml")) as fd:
        expected = fd.read()

    assert xml == expected

    with open(join(dirname(__file__), "testWFSIO.jdl")) as fd:
        expected = fd.read()

    jdlSIO = job._toJDL(jobDescriptionObject=StringIO(job._toXML()))
    assert jdlSIO == expected
Ejemplo n.º 6
0
 def basicTest(self):
     j = Job()
     j.setCPUTime(50000)
     j.setExecutable(
         '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py'
     )
     # j.setExecutable('/bin/echo hello')
     j.setOwner('paterson')
     j.setType('test')
     j.setName('MyJobName')
     #j.setAncestorDepth(1)
     j.setInputSandbox([
         '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts',
         '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts'
     ])
     j.setOutputSandbox(['firstfile.txt', 'anotherfile.root'])
     j.setInputData([
         '/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst',
         '/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst'
     ])
     j.setOutputData(['my.dst', 'myfile.log'])
     j.setDestination('LCG.CERN.ch')
     j.setPlatform('LCG')
     j.setSystemConfig('x86_64-slc5-gcc43-opt')
     j.setSoftwareTags(['VO-lhcb-Brunel-v30r17', 'VO-lhcb-Boole-v12r10'])
     #print j._toJDL()
     #print j.printObj()
     xml = j._toXML()
     testFile = 'jobDescription.xml'
     if os.path.exists(testFile):
         os.remove(testFile)
     xmlfile = open(testFile, 'w')
     xmlfile.write(xml)
     xmlfile.close()
     print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow'
     print j.createCode()
     print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow'
     j.execute()
     print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file'
     workflow = fromXMLFile(testFile)
     code = workflow.createCode()
     print code
     workflow.execute()
Ejemplo n.º 7
0
 def basicTest(self):
   j = Job()
   j.setCPUTime(50000)
   j.setExecutable('/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py')
  # j.setExecutable('/bin/echo hello')
   j.setOwner('paterson')
   j.setType('test')
   j.setName('MyJobName')
   #j.setAncestorDepth(1)
   j.setInputSandbox(['/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts','/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts'])
   j.setOutputSandbox(['firstfile.txt','anotherfile.root'])
   j.setInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst',
                   '/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst'])
   j.setOutputData(['my.dst','myfile.log'])
   j.setDestination('LCG.CERN.ch')
   j.setPlatform('LCG')
   j.setSystemConfig('x86_64-slc5-gcc43-opt')
   j.setSoftwareTags(['VO-lhcb-Brunel-v30r17','VO-lhcb-Boole-v12r10'])
   #print j._toJDL()
   #print j.printObj()
   xml = j._toXML()
   testFile = 'jobDescription.xml'
   if os.path.exists(testFile):
     os.remove(testFile)
   xmlfile = open(testFile,'w')
   xmlfile.write(xml)
   xmlfile.close()
   print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow'
   print j.createCode()
   print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow'
   j.execute()
   print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file'
   workflow = fromXMLFile(testFile)
   code = workflow.createCode()
   print code
   workflow.execute()
Ejemplo n.º 8
0
from DIRAC.Core.Base.Script import parseCommandLine
parseCommandLine()

from DIRAC.tests.Utilities.utils import find_all
from DIRAC.Interfaces.API.Job import Job


# With a script that returns 0
j = Job()
scriptSHLocation = find_all( 'script-OK.sh', '..', '/DIRAC/WorkloadManagementSystem/JobWrapper' )[0]
j.setExecutable('sh %s' %scriptSHLocation)

jobXMLFile = 'jobDescription-OK.xml'
with open( jobXMLFile, 'w+' ) as fd:
  fd.write( j._toXML() )

# With a script that returns 111
j = Job()
scriptSHLocation = find_all( 'script.sh', '..', '/DIRAC/WorkloadManagementSystem/JobWrapper' )[0]
j.setExecutable('sh %s' %scriptSHLocation)

jobXMLFile = 'jobDescription-FAIL.xml'
with open( jobXMLFile, 'w+' ) as fd:
  fd.write( j._toXML() )

# With a script that returns 1502
j = Job()
scriptSHLocation = find_all( 'script-RESC.sh', '..', '/DIRAC/WorkloadManagementSystem/JobWrapper' )[0]
j.setExecutable('sh %s' %scriptSHLocation)