def test_basicJob():
job = Job()
job.setOwner('ownerName')
job.setOwnerGroup('ownerGroup')
job.setName('jobName')
job.setJobGroup('jobGroup')
job.setExecutable('someExe')
job.setType('jobType')
job.setDestination('ANY')
xml = job._toXML()
try:
with open('./DIRAC/Interfaces/API/test/testWF.xml') as fd:
expected = fd.read()
except IOError:
with open('./Interfaces/API/test/testWF.xml') as fd:
expected = fd.read()
assert xml == expected
try:
with open('./DIRAC/Interfaces/API/test/testWFSIO.jdl') as fd:
expected = fd.read()
except IOError:
with open('./Interfaces/API/test/testWFSIO.jdl') as fd:
expected = fd.read()
jdlSIO = job._toJDL(jobDescriptionObject=StringIO.StringIO(job._toXML()))
assert jdlSIO == expected
def test_execute( self ):
job = Job()
job.setName( "helloWorld-test" )
job.setExecutable( find_all( "helloWorld.py", '.', 'Integration' )[0],
arguments = "This is an argument",
logFile = "aLogFileForTest.txt" ,
parameters=[('executable', 'string', '', "Executable Script"),
('arguments', 'string', '', 'Arguments for executable Script'),
( 'applicationLog', 'string', '', "Log file name" ),
( 'someCustomOne', 'string', '', "boh" )],
paramValues = [( 'someCustomOne', 'aCustomValue' )] )
job.setBannedSites( ['LCG.SiteA.com', 'DIRAC.SiteB.org'] )
job.setOwner( 'ownerName' )
job.setOwnerGroup( 'ownerGroup' )
job.setName( 'jobName' )
job.setJobGroup( 'jobGroup' )
job.setType( 'jobType' )
job.setDestination( 'DIRAC.someSite.ch' )
job.setCPUTime( 12345 )
job.setLogLevel( 'DEBUG' )
res = job.runLocal( self.d )
self.assertTrue( res['OK'] )
def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''):
if (not owner) or (not ownerGroup):
res = getProxyInfo(False,False)
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
oJob = Job(transBody)
for taskNumber in sortList(taskDict.keys()):
paramsDict = taskDict[taskNumber]
transID = paramsDict['TransformationID']
self.log.verbose('Setting job owner:group to %s:%s' % (owner,ownerGroup))
oJob.setOwner(owner)
oJob.setOwnerGroup(ownerGroup)
transGroup = str(transID).zfill(8)
self.log.verbose('Adding default transformation group of %s' % (transGroup))
oJob.setJobGroup(transGroup)
constructedName = str(transID).zfill(8)+'_'+str(taskNumber).zfill(8)
self.log.verbose('Setting task name to %s' % constructedName)
oJob.setName(constructedName)
oJob._setParamValue('PRODUCTION_ID',str(transID).zfill(8))
oJob._setParamValue('JOB_ID',str(taskNumber).zfill(8))
inputData = None
for paramName,paramValue in paramsDict.items():
self.log.verbose('TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %(transID,taskNumber,paramName,paramValue))
if paramName=='InputData':
if paramValue:
self.log.verbose('Setting input data to %s' %paramValue)
oJob.setInputData(paramValue)
elif paramName=='Site':
if paramValue:
self.log.verbose('Setting allocated site to: %s' %(paramValue))
oJob.setDestination(paramValue)
elif paramValue:
self.log.verbose('Setting %s to %s' % (paramName,paramValue))
oJob._addJDLParameter(paramName,paramValue)
hospitalTrans = [int(x) for x in gConfig.getValue("/Operations/Hospital/Transformations",[])]
if int(transID) in hospitalTrans:
hospitalSite = gConfig.getValue("/Operations/Hospital/HospitalSite",'DIRAC.JobDebugger.ch')
hospitalCEs = gConfig.getValue("/Operations/Hospital/HospitalCEs",[])
oJob.setType('Hospital')
oJob.setDestination(hospitalSite)
oJob.setInputDataPolicy('download',dataScheduling=False)
if hospitalCEs:
oJob._addJDLParameter('GridRequiredCEs',hospitalCEs)
taskDict[taskNumber]['TaskObject'] = ''
res = self.getOutputData({'Job':oJob._toXML(),'TransformationID':transID,'TaskID':taskNumber,'InputData':inputData})
if not res ['OK']:
self.log.error("Failed to generate output data",res['Message'])
continue
for name,output in res['Value'].items():
oJob._addJDLParameter(name,string.join(output,';'))
taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML())
return S_OK(taskDict)
def test_execute(self):
job = Job()
job._siteSet = {'DIRAC.someSite.ch'}
job.setName("helloWorld-test")
job.setExecutable(self.helloWorld,
arguments="This is an argument",
logFile="aLogFileForTest.txt",
parameters=[('executable', 'string', '',
"Executable Script"),
('arguments', 'string', '',
'Arguments for executable Script'),
('applicationLog', 'string', '',
"Log file name"),
('someCustomOne', 'string', '', "boh")],
paramValues=[('someCustomOne', 'aCustomValue')])
job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org'])
job.setOwner('ownerName')
job.setOwnerGroup('ownerGroup')
job.setName('jobName')
job.setJobGroup('jobGroup')
job.setType('jobType')
job.setDestination('DIRAC.someSite.ch')
job.setCPUTime(12345)
job.setLogLevel('DEBUG')
try:
# This is the standard location in Jenkins
job.setInputSandbox(
find_all('pilot.cfg',
os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
job.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
job.setConfigArgs('pilot.cfg')
res = job.runLocal(self.d)
self.assertTrue(res['OK'])
def basicTest(self):
j = Job()
j.setCPUTime(50000)
j.setExecutable('/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py')
# j.setExecutable('/bin/echo hello')
j.setOwner('paterson')
j.setType('test')
j.setName('MyJobName')
#j.setAncestorDepth(1)
j.setInputSandbox(['/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts','/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts'])
j.setOutputSandbox(['firstfile.txt','anotherfile.root'])
j.setInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst',
'/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst'])
j.setOutputData(['my.dst','myfile.log'])
j.setDestination('LCG.CERN.ch')
j.setPlatform('LCG')
j.setSystemConfig('x86_64-slc5-gcc43-opt')
j.setSoftwareTags(['VO-lhcb-Brunel-v30r17','VO-lhcb-Boole-v12r10'])
#print j._toJDL()
#print j.printObj()
xml = j._toXML()
testFile = 'jobDescription.xml'
if os.path.exists(testFile):
os.remove(testFile)
xmlfile = open(testFile,'w')
xmlfile.write(xml)
xmlfile.close()
print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow'
print j.createCode()
print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow'
j.execute()
print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file'
workflow = fromXMLFile(testFile)
code = workflow.createCode()
print code
workflow.execute()
def submitTS():
########################################
# Modify here with your dirac username
owner = 'user02'
########################################
########################################
# Job description
########################################
job = Job()
job.setName('build mandelbrot')
job.setOutputSandbox( ['*log'] )
job.setType('DataReprocessing')
## define the job workflow in 3 steps
# job step1: setup software
job.setExecutable('git clone https://github.com/bregeon/mandel4ts.git')
# job step2: run mandelbrot build image
job.setExecutable('./mandel4ts/build_merged_img.py')
outputPath = os.path.join('/vo.france-grilles.fr/user',owner[0],owner,'mandelbrot/images/final')
outputPattern = 'merged_image.bmp'
outputSE = 'DIRAC-USER'
outputMetadata = json.dumps( {"application":"mandelbrot","image_format":"bmp", "image_width":7680, "image_height":4200, "owner":owner} )
# job step3: upload data and set metadata
job.setExecutable( './mandel4ts/dirac-add-files.py', arguments = "%s '%s' %s '%s'" % (outputPath, outputPattern, outputSE, outputMetadata ) )
# job step4: mark input files as done with the FailoverRequest (and a few other things)
job.setExecutable('/bin/ls -l', modulesList=['Script', 'FailoverRequest'])
########################################
# Transformation definition
########################################
t = Transformation()
t.setTransformationName( owner+'_step3' )
t.setType( "DataReprocessing" )
t.setDescription( "Merge mandelbrot images production" )
t.setLongDescription( "Merge mandelbrot images production" )
t.setGroupSize( 3 ) # group input files
# set the job workflow to the transformation
t.setBody ( job.workflow.toXML() )
# define input data by metadata query
inputMetaquery = {"application":"mandelbrot","image_format":"ascii", "image_width":7680, "image_height":1400, "owner":owner}
t.setInputMetaQuery(inputMetaquery)
########################################
# Transformation submission
########################################
res = t.addTransformation()
if not res['OK']:
print(res['Message'])
DIRAC.exit( -1 )
t.setStatus( "Active" )
t.setAgentType( "Automatic" )
return res
if switch == "UseFilter":
UseFilter = True if switchValue.lower() == "true" else False
# Let's first create the prodJobuction
prodJobType = 'Merge'
transName = 'testProduction_' + str(int(time.time()))
desc = 'just test'
prodJob = Job()
prodJob._addParameter(prodJob.workflow, 'PRODUCTION_ID', 'string', '00012345', 'ProductionID')
prodJob._addParameter(prodJob.workflow, 'JOB_ID', 'string', '00006789', 'ProductionJobID')
prodJob._addParameter(prodJob.workflow, 'eventType', 'string', 'TestEventType', 'Event Type of the prodJobuction')
prodJob._addParameter(prodJob.workflow, 'numberOfEvents', 'string', '-1', 'Number of events requested')
prodJob._addParameter(prodJob.workflow, 'ProcessingType', 'JDL', str('Test'), 'ProductionGroupOrType')
prodJob._addParameter(prodJob.workflow, 'Priority', 'JDL', str(9), 'UserPriority')
prodJob.setType(prodJobType)
prodJob.workflow.setName(transName)
prodJob.workflow.setDescrShort(desc)
prodJob.workflow.setDescription(desc)
prodJob.setCPUTime(86400)
prodJob.setInputDataPolicy('Download')
prodJob.setExecutable('/bin/ls', '-l')
# Let's submit the prodJobuction now
#result = prodJob.create()
name = prodJob.workflow.getName()
name = name.replace('/', '').replace('\\', '')
prodJob.workflow.toXMLFile(name)
print 'Workflow XML file name is: %s' % name
def prepareTransformationTasks(self,
transBody,
taskDict,
owner='',
ownerGroup=''):
if (not owner) or (not ownerGroup):
res = getProxyInfo(False, False)
if not res['OK']:
return res
proxyInfo = res['Value']
owner = proxyInfo['username']
ownerGroup = proxyInfo['group']
oJob = Job(transBody)
for taskNumber in sortList(taskDict.keys()):
paramsDict = taskDict[taskNumber]
transID = paramsDict['TransformationID']
self.log.verbose('Setting job owner:group to %s:%s' %
(owner, ownerGroup))
oJob.setOwner(owner)
oJob.setOwnerGroup(ownerGroup)
transGroup = str(transID).zfill(8)
self.log.verbose('Adding default transformation group of %s' %
(transGroup))
oJob.setJobGroup(transGroup)
constructedName = str(transID).zfill(8) + '_' + str(
taskNumber).zfill(8)
self.log.verbose('Setting task name to %s' % constructedName)
oJob.setName(constructedName)
oJob._setParamValue('PRODUCTION_ID', str(transID).zfill(8))
oJob._setParamValue('JOB_ID', str(taskNumber).zfill(8))
inputData = None
for paramName, paramValue in paramsDict.items():
self.log.verbose(
'TransID: %s, TaskID: %s, ParamName: %s, ParamValue: %s' %
(transID, taskNumber, paramName, paramValue))
if paramName == 'InputData':
if paramValue:
self.log.verbose('Setting input data to %s' %
paramValue)
oJob.setInputData(paramValue)
elif paramName == 'Site':
if paramValue:
self.log.verbose('Setting allocated site to: %s' %
(paramValue))
oJob.setDestination(paramValue)
elif paramValue:
self.log.verbose('Setting %s to %s' %
(paramName, paramValue))
oJob._addJDLParameter(paramName, paramValue)
hospitalTrans = [
int(x) for x in gConfig.getValue(
"/Operations/Hospital/Transformations", [])
]
if int(transID) in hospitalTrans:
hospitalSite = gConfig.getValue(
"/Operations/Hospital/HospitalSite",
'DIRAC.JobDebugger.ch')
hospitalCEs = gConfig.getValue(
"/Operations/Hospital/HospitalCEs", [])
oJob.setType('Hospital')
oJob.setDestination(hospitalSite)
oJob.setInputDataPolicy('download', dataScheduling=False)
if hospitalCEs:
oJob._addJDLParameter('GridRequiredCEs', hospitalCEs)
taskDict[taskNumber]['TaskObject'] = ''
res = self.getOutputData({
'Job': oJob._toXML(),
'TransformationID': transID,
'TaskID': taskNumber,
'InputData': inputData
})
if not res['OK']:
self.log.error("Failed to generate output data",
res['Message'])
continue
for name, output in res['Value'].items():
oJob._addJDLParameter(name, string.join(output, ';'))
taskDict[taskNumber]['TaskObject'] = Job(oJob._toXML())
return S_OK(taskDict)
if switch[1] == 'True':
UseFilter = True
#Let's first create the prodJobuction
prodJobType = 'Merge'
transName = 'testProduction_' + str(int(time.time()))
desc = 'just test'
prodJob = Job()
prodJob._addParameter( prodJob.workflow, 'PRODUCTION_ID', 'string', '00012345', 'ProductionID' )
prodJob._addParameter( prodJob.workflow, 'JOB_ID', 'string', '00006789', 'ProductionJobID' )
prodJob._addParameter( prodJob.workflow, 'eventType', 'string', 'TestEventType', 'Event Type of the prodJobuction' )
prodJob._addParameter( prodJob.workflow, 'numberOfEvents', 'string', '-1', 'Number of events requested' )
prodJob._addParameter( prodJob.workflow, 'ProcessingType', 'JDL', str( 'Test' ), 'ProductionGroupOrType' )
prodJob._addParameter( prodJob.workflow, 'Priority', 'JDL', str( 9 ), 'UserPriority' )
prodJob.setType( prodJobType )
prodJob.workflow.setName(transName)
prodJob.workflow.setDescrShort( desc )
prodJob.workflow.setDescription( desc )
prodJob.setCPUTime( 86400 )
prodJob.setInputDataPolicy( 'Download' )
prodJob.setExecutable('/bin/ls', '-l')
#Let's submit the prodJobuction now
#result = prodJob.create()
name = prodJob.workflow.getName()
name = name.replace( '/', '' ).replace( '\\', '' )
prodJob.workflow.toXMLFile( name )
print 'Workflow XML file name is: %s' % name
def submitTS():
########################################
# Modify here with your dirac username
owner = 'user02'
########################################
########################################
# Job description
########################################
job = Job()
job.setName('mandelbrot raw')
job.setOutputSandbox(['*log'])
job.setType('MCSimulation')
# this is so that the JOB_ID within the transformation can be evaluated on the fly in the job application, see below
job.workflow.addParameter(
Parameter("JOB_ID", "000000", "string", "", "", True, False,
"Initialize JOB_ID"))
## define the job workflow in 3 steps
# job step1: setup software
job.setExecutable('git clone https://github.com/bregeon/mandel4ts.git')
# job step2: run mandelbrot application
# note how the JOB_ID (within the transformation) is passed as an argument and will be evaluated on the fly
job.setExecutable('./mandel4ts/mandelbrot.py',
arguments="-P 0.0005 -M 1000 -L @{JOB_ID} -N 200")
outputPath = os.path.join('/vo.france-grilles.fr/user', owner[0], owner,
'ts_mandelbrot/images/raw')
outputPattern = 'data_*txt'
outputSE = 'DIRAC-USER'
outputMetadata = json.dumps({
"application": "mandelbrot",
"image_format": "ascii",
"image_width": 7680,
"image_height": 200,
"owner": owner
})
# job step3: upload data and set metadata
# pilot.cfg in arguments is necessary with pilot 3
job.setExecutable('./mandel4ts/dirac-add-files.py',
arguments="pilot.cfg %s '%s' %s '%s'" %
(outputPath, outputPattern, outputSE, outputMetadata))
# job step4: mark input files as done with the FailoverRequest (and a few other things)
job.setExecutable('/bin/ls -l', modulesList=['Script', 'FailoverRequest'])
########################################
# Transformation definition
########################################
t = Transformation()
t.setTransformationName(owner + '_step1')
t.setType("MCSimulation")
t.setDescription("Mandelbrot images production")
t.setLongDescription("Mandelbrot images production")
# set the job workflow to the transformation
t.setBody(job.workflow.toXML())
########################################
# Transformation submission
########################################
res = t.addTransformation()
if not res['OK']:
print(res['Message'])
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
return res
set the job name
'''
import DIRAC
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.Job import Job
j = Job()
j.setExecutable('ls')
j.setName('API')
j.setType("")
dirac = Dirac()
result = dirac.submit(j)
errorList = []
if result['OK']:
print 'JobID = %s' % ( result['Value'] )
else:
errorList.append( ( j, result['Message'] ) )
for error in errorList:
print "ERROR %s: %s" % error