def test_basicJob(): job = Job() job.setOwner('ownerName') job.setOwnerGroup('ownerGroup') job.setName('jobName') job.setJobGroup('jobGroup') job.setExecutable('someExe') job.setType('jobType') job.setDestination('ANY') xml = job._toXML() try: with open('./DIRAC/Interfaces/API/test/testWF.xml') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWF.xml') as fd: expected = fd.read() assert xml == expected try: with open('./DIRAC/Interfaces/API/test/testWFSIO.jdl') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWFSIO.jdl') as fd: expected = fd.read() jdlSIO = job._toJDL(jobDescriptionObject=StringIO.StringIO(job._toXML())) assert jdlSIO == expected
def test_basicJob(): job = Job() job.setOwner('ownerName') job.setOwnerGroup('ownerGroup') job.setName('jobName') job.setJobGroup('jobGroup') job.setExecutable('someExe') job.setType('jobType') job.setDestination('ANY') xml = job._toXML() try: with open('./DIRAC/Interfaces/API/test/testWF.xml') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWF.xml') as fd: expected = fd.read() assert xml == expected try: with open('./DIRAC/Interfaces/API/test/testWFSIO.jdl') as fd: expected = fd.read() except IOError: with open('./Interfaces/API/test/testWFSIO.jdl') as fd: expected = fd.read() jdlSIO = job._toJDL(jobDescriptionObject=StringIO.StringIO(job._toXML())) assert jdlSIO == expected
def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''): if (not owner) or (not ownerGroup): res = getProxyInfo(False,False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] oJob = Job(transBody) for taskNumber in sortList(taskDict.keys()): paramsDict = taskDict[taskNumber] transID = paramsDict['TransformationID'] self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup)) oJob.setOwner(owner) oJob.setOwnerGroup(ownerGroup) transGroup = str(transID).zfill(8) self.log.verbose('Adding default transformation group of %s' % (transGroup)) oJob.setJobGroup(transGroup) constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8) self.log.verbose('Setting task name to %s' % constructedName) oJob.setName(constructedName) oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8)) oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8)) inputData = None for paramName,paramValue in paramsDict.items(): self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue)) if paramName=='InputData': if paramValue: self.log.verbose('Setting input data to %s' %paramValue) oJob.setInputData(paramValue) elif paramName=='Site': if paramValue: self.log.verbose('Setting allocated site to: %s' %(paramValue)) oJob.setDestination(paramValue) elif paramValue: self.log.verbose('Setting %s to %s' % (paramName,paramValue)) oJob._addJDLParameter(paramName,paramValue) hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])] if int(transID) in hospitalTrans: hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch') hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[]) oJob.setType('Hospital') oJob.setDestination(hospitalSite) oJob.setInputDataPolicy('download',dataScheduling=False) if hospitalCEs: oJob._addJDLParameter('GridRequiredCEs',hospitalCEs) taskDict[taskNumber]['TaskObject'] = '' res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData}) if not res ['OK']: self.log.error("Failed to generate output data",res['Message']) continue for name,output in res['Value'].items(): oJob._addJDLParameter(name,string.join(output,';')) taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML()) return S_OK(taskDict)
def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''): if (not owner) or (not ownerGroup): res = getProxyInfo(False,False) if not res['OK']: return res proxyInfo = res['Value'] owner = proxyInfo['username'] ownerGroup = proxyInfo['group'] oJob = Job(transBody) for taskNumber in sortList(taskDict.keys()): paramsDict = taskDict[taskNumber] transID = paramsDict['TransformationID'] self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup)) oJob.setOwner(owner) oJob.setOwnerGroup(ownerGroup) transGroup = str(transID).zfill(8) self.log.verbose('Adding default transformation group of %s' % (transGroup)) oJob.setJobGroup(transGroup) constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8) self.log.verbose('Setting task name to %s' % constructedName) oJob.setName(constructedName) oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8)) oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8)) inputData = None for paramName,paramValue in paramsDict.items(): self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue)) if paramName=='InputData': if paramValue: self.log.verbose('Setting input data to %s' %paramValue) oJob.setInputData(paramValue) elif paramName=='Site': if paramValue: self.log.verbose('Setting allocated site to: %s' %(paramValue)) oJob.setDestination(paramValue) elif paramValue: self.log.verbose('Setting %s to %s' % (paramName,paramValue)) oJob._addJDLParameter(paramName,paramValue) hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])] if int(transID) in hospitalTrans: hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch') hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[]) oJob.setType('Hospital') oJob.setDestination(hospitalSite) oJob.setInputDataPolicy('download',dataScheduling=False) if hospitalCEs: oJob._addJDLParameter('GridRequiredCEs',hospitalCEs) taskDict[taskNumber]['TaskObject'] = '' res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData}) if not res ['OK']: self.log.error("Failed to generate output data",res['Message']) continue for name,output in res['Value'].items(): oJob._addJDLParameter(name,string.join(output,';')) taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML()) return S_OK(taskDict)
def test_basicJob(): job = Job() job.setOwner("ownerName") job.setOwnerGroup("ownerGroup") job.setName("jobName") job.setJobGroup("jobGroup") job.setExecutable("someExe") job.setType("jobType") job.setDestination("ANY") xml = job._toXML() with open(join(dirname(__file__), "testWF.xml")) as fd: expected = fd.read() assert xml == expected with open(join(dirname(__file__), "testWFSIO.jdl")) as fd: expected = fd.read() jdlSIO = job._toJDL(jobDescriptionObject=StringIO(job._toXML())) assert jdlSIO == expected
def basicTest(self): j = Job() j.setCPUTime(50000) j.setExecutable( '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py' ) # j.setExecutable('/bin/echo hello') j.setOwner('paterson') j.setType('test') j.setName('MyJobName') #j.setAncestorDepth(1) j.setInputSandbox([ '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts', '/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts' ]) j.setOutputSandbox(['firstfile.txt', 'anotherfile.root']) j.setInputData([ '/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst', '/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst' ]) j.setOutputData(['my.dst', 'myfile.log']) j.setDestination('LCG.CERN.ch') j.setPlatform('LCG') j.setSystemConfig('x86_64-slc5-gcc43-opt') j.setSoftwareTags(['VO-lhcb-Brunel-v30r17', 'VO-lhcb-Boole-v12r10']) #print j._toJDL() #print j.printObj() xml = j._toXML() testFile = 'jobDescription.xml' if os.path.exists(testFile): os.remove(testFile) xmlfile = open(testFile, 'w') xmlfile.write(xml) xmlfile.close() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow' print j.createCode() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow' j.execute() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file' workflow = fromXMLFile(testFile) code = workflow.createCode() print code workflow.execute()
def basicTest(self): j = Job() j.setCPUTime(50000) j.setExecutable('/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py') # j.setExecutable('/bin/echo hello') j.setOwner('paterson') j.setType('test') j.setName('MyJobName') #j.setAncestorDepth(1) j.setInputSandbox(['/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts','/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts']) j.setOutputSandbox(['firstfile.txt','anotherfile.root']) j.setInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst', '/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst']) j.setOutputData(['my.dst','myfile.log']) j.setDestination('LCG.CERN.ch') j.setPlatform('LCG') j.setSystemConfig('x86_64-slc5-gcc43-opt') j.setSoftwareTags(['VO-lhcb-Brunel-v30r17','VO-lhcb-Boole-v12r10']) #print j._toJDL() #print j.printObj() xml = j._toXML() testFile = 'jobDescription.xml' if os.path.exists(testFile): os.remove(testFile) xmlfile = open(testFile,'w') xmlfile.write(xml) xmlfile.close() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow' print j.createCode() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow' j.execute() print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file' workflow = fromXMLFile(testFile) code = workflow.createCode() print code workflow.execute()
from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() from DIRAC.tests.Utilities.utils import find_all from DIRAC.Interfaces.API.Job import Job # With a script that returns 0 j = Job() scriptSHLocation = find_all( 'script-OK.sh', '..', '/DIRAC/WorkloadManagementSystem/JobWrapper' )[0] j.setExecutable('sh %s' %scriptSHLocation) jobXMLFile = 'jobDescription-OK.xml' with open( jobXMLFile, 'w+' ) as fd: fd.write( j._toXML() ) # With a script that returns 111 j = Job() scriptSHLocation = find_all( 'script.sh', '..', '/DIRAC/WorkloadManagementSystem/JobWrapper' )[0] j.setExecutable('sh %s' %scriptSHLocation) jobXMLFile = 'jobDescription-FAIL.xml' with open( jobXMLFile, 'w+' ) as fd: fd.write( j._toXML() ) # With a script that returns 1502 j = Job() scriptSHLocation = find_all( 'script-RESC.sh', '..', '/DIRAC/WorkloadManagementSystem/JobWrapper' )[0] j.setExecutable('sh %s' %scriptSHLocation)