def softinstall( args = None ) :
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.Job import Job
if (len(args)!=2):
Script.showHelp()
version = args[0]
site = args[1]
if version not in ['prod-2_21122012','prod-2_08032013']:
Script.gLogger.error('Version not valid')
Script.showHelp()
j = Job()
j.setInputSandbox( ['cta-swinstall.py','SoftwareInstallation.py','CODE'] )
j.setExecutable('./cta-swinstall.py', version)
j.setDestination([site])
j.setName('SoftInstall')
j.setCPUTime(100000)
Script.gLogger.info( j._toJDL() )
Dirac().submit( j )
def helloWorldJob():
job = Job()
job.setName("helloWorld")
exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0]
job.setInputSandbox(exeScriptLocation)
job.setExecutable(exeScriptLocation, "", "helloWorld.log")
return job
def __submit( self, site, CE, vo ):
"""
set the job and submit.
"""
job = Job()
job.setName( self.testType )
job.setJobGroup( 'CE-Test' )
job.setExecutable( self.executable )
job.setInputSandbox( '%s/%s' % ( self.__scriptPath, self.executable ) )
if site and not CE:
job.setDestination( site )
if CE:
job.setDestinationCE( CE )
LOCK.acquire()
proxyPath = BESUtils.getProxyByVO( 'zhangxm', vo )
if not proxyPath[ 'OK' ]:
LOCK.release()
return proxyPath
proxyPath = proxyPath[ 'Value' ]
oldProxy = os.environ.get( 'X509_USER_PROXY' )
os.environ[ 'X509_USER_PROXY' ] = proxyPath
result = self.dirac.submit( job )
if oldProxy is None:
del os.environ[ 'X509_USER_PROXY' ]
else:
os.environ[ 'X509_USER_PROXY' ] = oldProxy
LOCK.release()
return result
def test_SimpleParametricJob():
job = Job()
job.setExecutable('myExec')
job.setLogLevel('DEBUG')
parList = [1, 2, 3]
job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'],
['/lhcb/data/data3', '/lhcb/data/data4'],
['/lhcb/data/data5', '/lhcb/data/data6']]
job.setParameterSequence('InputData', inputDataList, addToWorkflow=True)
jdl = job._toJDL()
try:
with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd:
expected = fd.read()
except IOError:
with open('./Interfaces/API/test/testWF.jdl') as fd:
expected = fd.read()
assert jdl == expected
clad = ClassAd('[' + jdl + ']')
arguments = clad.getAttributeString('Arguments')
job_id = clad.getAttributeString('JOB_ID')
inputData = clad.getAttributeString('InputData')
assert job_id == '%(JOB_ID)s'
assert inputData == '%(InputData)s'
assert 'jobDescription.xml' in arguments
assert '-o LogLevel=DEBUG' in arguments
assert '-p JOB_ID=%(JOB_ID)s' in arguments
assert '-p InputData=%(InputData)s' in arguments
def __submit(self, site, CE, vo):
"""
set the job and submit.
"""
job = Job()
job.setName(self.testType)
job.setJobGroup('CE-Test')
job.setExecutable(self.executable)
job.setInputSandbox('%s/%s' % (self.__scriptPath, self.executable))
if site and not CE:
job.setDestination(site)
if CE:
job.setDestinationCE(CE)
LOCK.acquire()
proxyPath = BESUtils.getProxyByVO('zhangxm', vo)
if not proxyPath['OK']:
LOCK.release()
return proxyPath
proxyPath = proxyPath['Value']
oldProxy = os.environ.get('X509_USER_PROXY')
os.environ['X509_USER_PROXY'] = proxyPath
result = self.dirac.submitJob(job)
if oldProxy is None:
del os.environ['X509_USER_PROXY']
else:
os.environ['X509_USER_PROXY'] = oldProxy
LOCK.release()
return result
def test_basicJob():
job = Job()
job.setOwner('ownerName')
job.setOwnerGroup('ownerGroup')
job.setName('jobName')
job.setJobGroup('jobGroup')
job.setExecutable('someExe')
job.setType('jobType')
job.setDestination('ANY')
xml = job._toXML()
try:
with open('./DIRAC/Interfaces/API/test/testWF.xml') as fd:
expected = fd.read()
except IOError:
with open('./Interfaces/API/test/testWF.xml') as fd:
expected = fd.read()
assert xml == expected
try:
with open('./DIRAC/Interfaces/API/test/testWFSIO.jdl') as fd:
expected = fd.read()
except IOError:
with open('./Interfaces/API/test/testWFSIO.jdl') as fd:
expected = fd.read()
jdlSIO = job._toJDL(jobDescriptionObject=StringIO.StringIO(job._toXML()))
assert jdlSIO == expected
def test_min2max4(self):
""" this tests executes a job that requires 2 to 4 processors
"""
j = Job()
j.setName("MP-test-min2max4")
j.setExecutable(self.mpExeFlex, arguments='2')
j.setInputSandbox(
find_all('mpTest-flexible.py', rootPath,
'DIRAC/tests/Utilities')[0])
j.setNumberOfProcessors(
minNumberOfProcessors=2,
maxNumberOfProcessors=4) # This requires 2 to 4 processors
j.setLogLevel('DEBUG')
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all('pilot.cfg',
os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
j.setConfigArgs('pilot.cfg')
res = j.runLocal(self.d)
self.assertTrue(res['OK'])
def test_execute( self ):
job = Job()
job.setName( "helloWorld-test" )
job.setExecutable( find_all( "helloWorld.py", '.', 'Integration' )[0],
arguments = "This is an argument",
logFile = "aLogFileForTest.txt" ,
parameters=[('executable', 'string', '', "Executable Script"),
('arguments', 'string', '', 'Arguments for executable Script'),
( 'applicationLog', 'string', '', "Log file name" ),
( 'someCustomOne', 'string', '', "boh" )],
paramValues = [( 'someCustomOne', 'aCustomValue' )] )
job.setBannedSites( ['LCG.SiteA.com', 'DIRAC.SiteB.org'] )
job.setOwner( 'ownerName' )
job.setOwnerGroup( 'ownerGroup' )
job.setName( 'jobName' )
job.setJobGroup( 'jobGroup' )
job.setType( 'jobType' )
job.setDestination( 'DIRAC.someSite.ch' )
job.setCPUTime( 12345 )
job.setLogLevel( 'DEBUG' )
res = job.runLocal( self.d )
self.assertTrue( res['OK'] )
def helloWorldJob(): job = Job() job.setName( "helloWorld" ) exeScriptLocation = find_all( 'exe-script.py', '.', 'WorkloadManagementSystem' )[0] job.setInputSandbox( exeScriptLocation ) job.setExecutable( exeScriptLocation, "", "helloWorld.log" ) return job
def test_SimpleParametricJob(self):
job = Job()
job.setExecutable('myExec')
job.setLogLevel('DEBUG')
parList = [1, 2, 3]
job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'],
['/lhcb/data/data3', '/lhcb/data/data4'],
['/lhcb/data/data5', '/lhcb/data/data6']]
job.setParameterSequence('InputData',
inputDataList,
addToWorkflow=True)
jdl = job._toJDL()
print jdl
clad = ClassAd('[' + jdl + ']')
arguments = clad.getAttributeString('Arguments')
job_id = clad.getAttributeString('JOB_ID')
inputData = clad.getAttributeString('InputData')
print "arguments", arguments
self.assertEqual(job_id, '%(JOB_ID)s')
self.assertEqual(inputData, '%(InputData)s')
self.assertIn('jobDescription.xml', arguments)
self.assertIn('-o LogLevel=DEBUG', arguments)
self.assertIn('-p JOB_ID=%(JOB_ID)s', arguments)
self.assertIn('-p InputData=%(InputData)s', arguments)
def test_min2(self):
"""this tests executes a job that requires at least 2 processors"""
j = Job()
j.setName("MP-test-min2")
# FIXME: the number of processors should be discovered at runtime using JobParameters.getNumberOfJobProcessors()
# here, and later
j.setExecutable(self.mpExeFlex, arguments="2")
j.setInputSandbox(
find_all("mpTest-flexible.py", rootPath,
"DIRAC/tests/Utilities")[0])
j.setNumberOfProcessors(
minNumberOfProcessors=2) # This requires at least 2 processors
j.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
j.setConfigArgs("pilot.cfg")
res = j.runLocal(self.d)
self.assertTrue(res["OK"])
def submitProbeJobs(self, ce):
""" Submit some jobs to the CEs
"""
#need credentials, should be there since the initialize
from DIRAC.Interfaces.API.Dirac import Dirac
d = Dirac()
from DIRAC.Interfaces.API.Job import Job
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
import DIRAC
ops = Operations()
scriptname = ops.getValue("ResourceStatus/SofwareManagementScript", self.script)
j = Job()
j.setDestinationCE(ce)
j.setCPUTime(1000)
j.setName("Probe %s" % ce)
j.setJobGroup("SoftwareProbe")
j.setExecutable("%s/GlastDIRAC/ResourceStatusSystem/Client/%s" % (DIRAC.rootPath, scriptname),
logFile='SoftwareProbe.log')
j.setOutputSandbox('*.log')
res = d.submit(j)
if not res['OK']:
return res
return S_OK()
def helloWorldJob(): job = Job() job.setName( "helloWorld" ) exeScriptLocation = find_all( 'exe-script.py', '.', 'WorkloadManagementSystem' )[0] job.setInputSandbox( exeScriptLocation ) job.setExecutable( exeScriptLocation, "", "helloWorld.log" ) return job
def test_fixed(self):
"""this tests executes a job that requires exactly 4 processors"""
j = Job()
j.setName("MP-test")
j.setExecutable(self.mpExe)
j.setInputSandbox(
find_all("mpTest.py", rootPath, "DIRAC/tests/Utilities")[0])
j.setNumberOfProcessors(
4) # This requires a fixed number of processors
j.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
j.setConfigArgs("pilot.cfg")
res = j.runLocal(self.d)
if multiprocessing.cpu_count() > 1:
self.assertTrue(res["OK"])
else:
self.assertFalse(res["OK"])
def test_basicJob():
job = Job()
job.setOwner('ownerName')
job.setOwnerGroup('ownerGroup')
job.setName('jobName')
job.setJobGroup('jobGroup')
job.setExecutable('someExe')
job.setType('jobType')
job.setDestination('ANY')
xml = job._toXML()
try:
with open('./DIRAC/Interfaces/API/test/testWF.xml') as fd:
expected = fd.read()
except IOError:
with open('./Interfaces/API/test/testWF.xml') as fd:
expected = fd.read()
assert xml == expected
try:
with open('./DIRAC/Interfaces/API/test/testWFSIO.jdl') as fd:
expected = fd.read()
except IOError:
with open('./Interfaces/API/test/testWFSIO.jdl') as fd:
expected = fd.read()
jdlSIO = job._toJDL(jobDescriptionObject=StringIO.StringIO(job._toXML()))
assert jdlSIO == expected
def test_execute( self ):
j = Job()
j.setName( "helloWorld-test" )
j.setExecutable( self.exeScriptLocation )
res = j.runLocal( self.d )
self.assertTrue( res['OK'] )
def parametricJob():
job = Job()
job.setName("parametric_helloWorld_%n")
exeScriptLocation = find_all('exe-script.py', '..', '/DIRAC/tests/Integration')[0]
job.setInputSandbox(exeScriptLocation)
job.setParameterSequence("args", ['one', 'two', 'three'])
job.setParameterSequence("iargs", [1, 2, 3])
job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile='helloWorld_%n.log')
return job
def parametricJob():
job = Job()
job.setName("parametric_helloWorld_%n")
exeScriptLocation = find_all("exe-script.py", "..", "/DIRAC/tests/Integration")[0]
job.setInputSandbox(exeScriptLocation)
job.setParameterSequence("args", ["one", "two", "three"])
job.setParameterSequence("iargs", [1, 2, 3])
job.setExecutable(exeScriptLocation, arguments=": testing %(args)s %(iargs)s", logFile="helloWorld_%n.log")
return job
def base():
job = Job()
job.setName("helloWorld-TEST-TO-Jenkins")
job.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]])
job.setExecutable("exe-script.py", "", "helloWorld.log")
job.setCPUTime(1780)
job.setDestination('DIRAC.Jenkins.ch')
job.setLogLevel('DEBUG')
return job
def test_execute( self ):
""" just testing unix "ls"
"""
job = Job()
job.setName( "ls-test" )
job.setExecutable( "/bin/ls", '-l' )
res = job.runLocal( self.d )
self.assertTrue( res['OK'] )
def base():
job = Job()
job.setName("helloWorld-TEST-TO-Jenkins")
executablePath = find_all("exe-script.py", "..",
"/DIRAC/tests/Workflow/")[0]
job.setInputSandbox([executablePath])
job.setExecutable(executablePath, "", "helloWorld.log")
job.setCPUTime(1780)
job.setDestination("DIRAC.Jenkins.ch")
job.setLogLevel("DEBUG")
return job
def test_execute( self ):
""" this one tests that I can execute a job that requires multi-processing
"""
j = Job()
j.setName( "MP-test" )
j.setExecutable( self.mpExe )
j.setInputSandbox( find_all( 'mpTest.py', '.', 'Utilities' )[0] )
j.setTag( 'MultiProcessor' )
res = j.runLocal( self.d )
self.assertTrue( res['OK'] )
def softinstall(args=None):
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.Job import Job
if (len(args) < 2):
Script.showHelp()
version = args[0]
site = args[1]
if version not in ['18122013', 'v0r7p0']:
Script.gLogger.error('Version not valid')
Script.showHelp()
j = Job()
j.setInputSandbox(['cta-ctools-install.py', 'SoftwareInstallation.py'])
j.setExecutable('./cta-ctools-install.py', version)
j.setDestination([site])
j.setName('ctoolsInstall')
j.setCPUTime(100000)
Script.gLogger.info(j._toJDL())
if site in ['LCG.GRIF.fr', 'LCG.M3PEC.fr']:
if site == 'LCG.GRIF.fr':
ceList = [
'apcce02.in2p3.fr', 'grid36.lal.in2p3.fr',
'lpnhe-cream.in2p3.fr', 'llrcream.in2p3.fr',
'node74.datagrid.cea.fr'
]
elif site == 'LCG.M3PEC.fr':
# ceList = ['ce0.bordeaux.inra.fr','ce0.m3pec.u-bordeaux1.fr']
ceList = ['ce0.bordeaux.inra.fr']
for ce in ceList:
j.setDestinationCE(ce)
name = 'ctoolsInstall' + '_' + ce
j.setName(name)
res = Dirac().submit(j)
print res
DIRAC.exit()
else:
name = 'ctoolsInstall'
j.setName(name)
res = Dirac().submit(j)
print res
def _submitJob(self, result_id, executable, test_name, site_name):
executable = executable.split('&')
j = Job()
j.setExecutable('python', arguments=executable[0] + " " + str(result_id))
sandBox = []
for file_name in executable:
sandBox.append(SAM_TEST_DIR + file_name)
j.setInputSandbox(sandBox)
j.setName(test_name)
j.setJobGroup('sam_test')
j.setDestination(site_name)
result = self.dirac.submit(j)
return result
def do_installonsite(self, argss):
""" Install a release on a grid site :
installonsite tag site
"""
args = argss.split()
if len(args) < 2:
print self.do_installonsite.__doc__
return
tag = args[0]
site = args[1]
#print "Check if the software with the tag '"+tag+"' exists on the rsync server..."
#res = self.client.getSitesForTag(tag)
#if not res['OK']:
#print res['Message']
#return
#print "tag found !"
from DIRAC.Interfaces.API.Dirac import Dirac
d = Dirac()
from DIRAC.Interfaces.API.Job import Job
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
import os
ops = Operations()
scriptname = "InstallSoftware.py"
j = Job()
j.setDestination(site)
j.setCPUTime(1000)
j.setName("Installation " + tag)
j.setExecutable(os.environ['DIRAC'] +
"/GlastDIRAC/ResourceStatusSystem/Client/externals/" +
scriptname,
logFile='SoftwareInstallation.log')
j.setOutputSandbox('*.log')
res = d.submit(j)
if not res['OK']:
print "Could not submit the installation at site %s, message %s" % (
site, res['Message'])
return
print "Job submitted, id = " + str(res['Value'])
print "Add tag :"
res = self.client.addTagAtSite(tag, site)
if not res['OK']:
print "Could not register tag %s at site %s, message %s" % (
tag, site, res['Message'])
return
print "Added %s to %i CEs" % (tag, len(res['Value'][tag]))
def createJob(self):
job = Job()
job.setName(self.__stepName)
job.setOutputSandbox(['*log'])
job.setExecutable('/usr/bin/wget',
arguments='"{0}/{1}"'.format(URL_ROOT,
self.__executable))
job.setExecutable('/bin/chmod',
arguments='+x "{0}"'.format(self.__executable))
arguments = '"{0}" "{1}" "{2}" "{3}" "{4}" "{5}" @{{JOB_ID}}'.format(
self.__softwareVersion, self.__application, self.__outputPath,
self.__outputPattern, self.__outputSE, self.__outputMode)
if self.__extraArgs:
arguments += ' ' + self.__extraArgs
job.setExecutable(self.__executable, arguments=arguments)
# failover for failed jobs
job.setExecutable('/bin/ls -l',
modulesList=['Script', 'FailoverRequest'])
if self.__inputData:
job.setInputData(self.__inputData)
if self.__site:
job.setDestination(self.__site)
if self.__bannedsite:
job.setBannedSites(self.__bannedsite)
job.setOutputSandbox(['app.out', 'app.err', 'Script3_CodeOutput.log'])
self.__job = job
def test_MPJob(proc, minProc, maxProc, expectedProc, expectedMinProc, expectedMaxProc):
job = Job()
job.setExecutable("myExec")
job.setLogLevel("DEBUG")
job.setNumberOfProcessors(proc, minProc, maxProc)
jdl = job._toJDL()
clad = ClassAd("[" + jdl + "]")
processors = clad.getAttributeInt("NumberOfProcessors")
minProcessors = clad.getAttributeInt("MinNumberOfProcessors")
maxProcessors = clad.getAttributeInt("MaxNumberOfProcessors")
assert processors == expectedProc
assert minProcessors == expectedMinProc
assert maxProcessors == expectedMaxProc
def test_MPJob(proc, minProc, maxProc, expectedProc, expectedMinProc,
expectedMaxProc):
job = Job()
job.setExecutable('myExec')
job.setLogLevel('DEBUG')
job.setNumberOfProcessors(proc, minProc, maxProc)
jdl = job._toJDL()
clad = ClassAd('[' + jdl + ']')
processors = clad.getAttributeInt('NumberOfProcessors')
minProcessors = clad.getAttributeInt('MinNumberOfProcessors')
maxProcessors = clad.getAttributeInt('MaxNumberOfProcessors')
assert processors == expectedProc
assert minProcessors == expectedMinProc
assert maxProcessors == expectedMaxProc
def run_test_job(args):
simtel_files = load_files_from_list(args[0])
#simtel_files = ["/vo.cta.in2p3.fr/MC/PROD3/LaPalma/proton/simtel/1260/Data/071xxx/proton_40deg_180deg_run71001___cta-prod3-lapalma3-2147m-LaPalma.simtel.gz",
#"/vo.cta.in2p3.fr/MC/PROD3/LaPalma/proton/simtel/1260/Data/070xxx/proton_40deg_180deg_run70502___cta-prod3-lapalma3-2147m-LaPalma.simtel.gz"]
dirac = Dirac()
j = Job()
j.setCPUTime(500)
j.setInputData(simtel_files[0])
j.setExecutable('echo', 'Hello World!')
j.setName('Hello World')
res = dirac.submit(j)
print('Submission Result: {}'.format(res))
return res
def test_execute(self):
""" this one tests that I can execute a job that requires multi-processing
"""
j = Job()
j.setName("MP-test")
j.setExecutable(self.mpExe)
j.setInputSandbox(find_all('mpTest.py', rootPath, 'DIRAC/tests/Utilities')[0])
j.setTag('MultiProcessor')
j.setLogLevel('DEBUG')
res = j.runLocal(self.d)
if multiprocessing.cpu_count() > 1:
self.assertTrue(res['OK'])
else:
self.assertFalse(res['OK'])
def do_installonsite(self,argss):
""" Install a release on a grid site :
installonsite tag site
"""
args = argss.split()
if len(args)<2:
print self.do_installonsite.__doc__
return
tag = args[0]
site = args[1]
#print "Check if the software with the tag '"+tag+"' exists on the rsync server..."
#res = self.client.getSitesForTag(tag)
#if not res['OK']:
#print res['Message']
#return
#print "tag found !"
from DIRAC.Interfaces.API.Dirac import Dirac
d = Dirac()
from DIRAC.Interfaces.API.Job import Job
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
import os
ops = Operations()
scriptname = "InstallSoftware.py"
j = Job()
j.setDestination(site)
j.setCPUTime(1000)
j.setName("Installation "+tag)
j.setExecutable(os.environ['DIRAC']+"/GlastDIRAC/ResourceStatusSystem/Client/"+scriptname , logFile='SoftwareInstallation.log')
j.setOutputSandbox('*.log')
res = d.submit(j)
if not res['OK']:
print "Could not submit the installation at site %s, message %s"%(site,res['Message'])
return
print "Job submitted, id = "+str(res['Value'])
print "Add tag :"
res = self.client.addTagAtSite(tag,site)
if not res['OK']:
print "Could not register tag %s at site %s, message %s"%(tag,site,res['Message'])
return
print "Added %s to %i CEs"%(tag,len(res['Value'][tag]))
def test_SimpleParametricJob():
job = Job()
job.setExecutable('myExec')
job.setLogLevel('DEBUG')
parList = [1, 2, 3]
job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
inputDataList = [
[
'/lhcb/data/data1',
'/lhcb/data/data2'
],
[
'/lhcb/data/data3',
'/lhcb/data/data4'
],
[
'/lhcb/data/data5',
'/lhcb/data/data6'
]
]
job.setParameterSequence('InputData', inputDataList, addToWorkflow=True)
jdl = job._toJDL()
try:
with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd:
expected = fd.read()
except IOError:
with open('./Interfaces/API/test/testWF.jdl') as fd:
expected = fd.read()
assert jdl == expected
clad = ClassAd('[' + jdl + ']')
arguments = clad.getAttributeString('Arguments')
job_id = clad.getAttributeString('JOB_ID')
inputData = clad.getAttributeString('InputData')
assert job_id == '%(JOB_ID)s'
assert inputData == '%(InputData)s'
assert 'jobDescription.xml' in arguments
assert '-o LogLevel=DEBUG' in arguments
assert'-p JOB_ID=%(JOB_ID)s' in arguments
assert'-p InputData=%(InputData)s' in arguments
def submitJob(jobPara):
dirac = Dirac()
j = Job()
j.setName(jobPara['jobName'])
j.setJobGroup(jobPara['jobGroup'])
j.setExecutable(jobPara['jobScript'], logFile = jobPara['jobScriptLog'])
j.setInputSandbox(jobPara['inputSandbox'])
j.setOutputSandbox(jobPara['outputSandbox'])
j.setOutputData(jobPara['outputData'], jobPara['SE'])
j.setDestination(jobPara['sites'])
j.setCPUTime(jobPara['CPUTime'])
result = dirac.submit(j)
if result['OK']:
print 'Job %s submitted successfully. ID = %d' %(jobPara['jobName'],result['Value'])
else:
print 'Job %s submitted failed' %jobPara['jobName']
return result
def launch_batch_pict( pitch_start, step, n_pict ):
j = Job()
j.setCPUTime(500)
j.setName('%s_%f' % (EXEC, pitch_start))
j.setJobGroup(JOBGROUP)
j.setInputSandbox([EXEC])
out_bmp_list=[]
pitch=pitch_start
for i in range(n_pict):
out_bmp='out_%f.bmp' % pitch
out_bmp_list.append(out_bmp)
j.setExecutable(EXEC,arguments="-W 600 -H 600 -X -0.77568377 -Y -0.13646737 -P %f -M 500 %s" % (pitch, out_bmp))
pitch+=step
j.setOutputSandbox(out_bmp_list + ["StdOut"] + ["StdErr"])
result = dirac.submit(j)
print 'Submission Result: ',result
return result
def test_SimpleParametricJob( self ):
job = Job()
job.setExecutable( 'myExec' )
job.setLogLevel( 'DEBUG' )
parList = [1,2,3]
job.setParameterSequence( 'JOB_ID', parList, addToWorkflow=True )
inputDataList = [
[
'/lhcb/data/data1',
'/lhcb/data/data2'
],
[
'/lhcb/data/data3',
'/lhcb/data/data4'
],
[
'/lhcb/data/data5',
'/lhcb/data/data6'
]
]
job.setParameterSequence( 'InputData', inputDataList, addToWorkflow=True )
jdl = job._toJDL()
print jdl
clad = ClassAd( '[' + jdl + ']' )
arguments = clad.getAttributeString( 'Arguments' )
job_id = clad.getAttributeString( 'JOB_ID' )
inputData = clad.getAttributeString( 'InputData' )
print "arguments", arguments
self.assertEqual( job_id, '%(JOB_ID)s' )
self.assertEqual( inputData, '%(InputData)s' )
self.assertIn( 'jobDescription.xml', arguments )
self.assertIn( '-o LogLevel=DEBUG', arguments )
self.assertIn( '-p JOB_ID=%(JOB_ID)s', arguments )
self.assertIn( '-p InputData=%(InputData)s', arguments )
def submit(self, param):
j = Job()
j.setName(param['jobName'])
j.setExecutable(param['jobScript'],logFile = param['jobScriptLog'])
if self.site:
j.setDestination(self.site)
if self.jobGroup:
j.setJobGroup(self.jobGroup)
j.setInputSandbox(param['inputSandbox'])
j.setOutputSandbox(param['outputSandbox'])
j.setOutputData(param['outputData'], outputSE = self.outputSE, outputPath = self.outputPath)
dirac = GridDirac()
result = dirac.submit(j)
status = {}
status['submit'] = result['OK']
if status['submit']:
status['job_id'] = result['Value']
return status
def basicTest(self):
j = Job()
j.setCPUTime(50000)
j.setExecutable('/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/myPythonScript.py')
# j.setExecutable('/bin/echo hello')
j.setOwner('paterson')
j.setType('test')
j.setName('MyJobName')
#j.setAncestorDepth(1)
j.setInputSandbox(['/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV.opts','/Users/stuart/dirac/workspace/DIRAC3/DIRAC/Interfaces/API/test/DV2.opts'])
j.setOutputSandbox(['firstfile.txt','anotherfile.root'])
j.setInputData(['/lhcb/production/DC04/v2/DST/00000742_00003493_11.dst',
'/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst'])
j.setOutputData(['my.dst','myfile.log'])
j.setDestination('LCG.CERN.ch')
j.setPlatform('LCG')
j.setSystemConfig('x86_64-slc5-gcc43-opt')
j.setSoftwareTags(['VO-lhcb-Brunel-v30r17','VO-lhcb-Boole-v12r10'])
#print j._toJDL()
#print j.printObj()
xml = j._toXML()
testFile = 'jobDescription.xml'
if os.path.exists(testFile):
os.remove(testFile)
xmlfile = open(testFile,'w')
xmlfile.write(xml)
xmlfile.close()
print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Creating code for the workflow'
print j.createCode()
print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Executing the workflow'
j.execute()
print '>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>Trying to run the same workflow from generated XML file'
workflow = fromXMLFile(testFile)
code = workflow.createCode()
print code
workflow.execute()
desc = 'just test'
prodJob = Job()
prodJob._addParameter(prodJob.workflow, 'PRODUCTION_ID', 'string', '00012345', 'ProductionID')
prodJob._addParameter(prodJob.workflow, 'JOB_ID', 'string', '00006789', 'ProductionJobID')
prodJob._addParameter(prodJob.workflow, 'eventType', 'string', 'TestEventType', 'Event Type of the prodJobuction')
prodJob._addParameter(prodJob.workflow, 'numberOfEvents', 'string', '-1', 'Number of events requested')
prodJob._addParameter(prodJob.workflow, 'ProcessingType', 'JDL', str('Test'), 'ProductionGroupOrType')
prodJob._addParameter(prodJob.workflow, 'Priority', 'JDL', str(9), 'UserPriority')
prodJob.setType(prodJobType)
prodJob.workflow.setName(transName)
prodJob.workflow.setDescrShort(desc)
prodJob.workflow.setDescription(desc)
prodJob.setCPUTime(86400)
prodJob.setInputDataPolicy('Download')
prodJob.setExecutable('/bin/ls', '-l')
# Let's submit the prodJobuction now
#result = prodJob.create()
name = prodJob.workflow.getName()
name = name.replace('/', '').replace('\\', '')
prodJob.workflow.toXMLFile(name)
print 'Workflow XML file name is: %s' % name
workflowBody = ''
if os.path.exists(name):
with open(name, 'r') as fopen:
workflowBody = fopen.read()
else:
from DIRAC.Interfaces.API.Job import Job from DIRAC.Interfaces.API.Dirac import Dirac #from tests.Workflow.Integration.Test_UserJobs import createJob gLogger.setLevel( 'DEBUG' ) cwd = os.path.realpath( '.' ) dirac = Dirac() # Simple Hello Word job to DIRAC.Jenkins.ch gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch" ) helloJ = Job() helloJ.setName( "helloWorld-TEST-TO-Jenkins" ) helloJ.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] ) helloJ.setExecutable( "exe-script.py", "", "helloWorld.log" ) helloJ.setCPUTime( 17800 ) helloJ.setDestination( 'DIRAC.Jenkins.ch' ) result = dirac.submit( helloJ ) gLogger.info( "Hello world job: ", result ) if not result['OK']: gLogger.error( "Problem submitting job", result['Message'] ) exit( 1 ) # Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN gLogger.info( "\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN" ) helloJMP = Job() helloJMP.setName( "helloWorld-TEST-TO-Jenkins-MP" ) helloJMP.setInputSandbox( [find_all( 'exe-script.py', '..', '/DIRAC/tests/Workflow/' )[0]] ) helloJMP.setExecutable( "exe-script.py", "", "helloWorld.log" ) helloJMP.setCPUTime( 17800 )
## prepare the output location in GRID storage; the input path will be the used also for GRID storage
outdir = grid_basedir_output + path + "/" + str(runnr)
## ALWAYS, INFO, VERBOSE, WARN, DEBUG
j.setLogLevel('debug')
j.setName('AUGER test simulation')
j.setDestinationCE(ce1)
## j.setDestination(site_dirac)
j.setCPUTime(345600) ## 4 days
## download the script for preparing corsika input file for usage with cvmfs
j.setExecutable( 'curl', arguments = ' -fsSLkO http://issaf.spacescience.ro/adrian/AUGER/make_run4cvmfs',logFile='cmd_logs.log')
j.setExecutable( 'chmod', arguments = ' +x make_run4cvmfs',logFile='cmd_logs.log')
## create the simulation script configured for use with cvmfs
## set the make_run4cvmfs arguments to include the corsika_version and corsika_bin
make_run4cvmfs_arg = input_file_base + " " + corsika_version + " " + corsika_bin
j.setExecutable( './make_run4cvmfs', arguments = make_run4cvmfs_arg, logFile='cmd_logs.log')
## run simulation
j.setExecutable( './execsim',logFile='cmd_logs.log')
j.setOutputSandbox(output_files)
j.setOutputData(output_files, outputSE=se, outputPath=outdir)
##j.runLocal() ## test local
""" simple hello world job """ from DIRAC.Interfaces.API.Job import Job from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers j = Job() j.setName( "helloWorld-test" ) j.setExecutable( "exe-script.py", "", "Executable.log" ) # <-- user settings j.setCPUTime( 172800 ) tier1s = DMSHelpers().getTiers( tier = ( 0, 1 ) ) j.setBannedSites( tier1s ) # user settings --> # print j.workflow # submit the job to dirac result = Dirac().submitJob(j) print result
from DIRAC.Core.Base import Script
Script.parseCommandLine()
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.Interfaces.API.Job import Job
dirac = Dirac()
j = Job()
j.setName("compile_mrfilter")
j.setCPUTime(80)
j.setInputSandbox(["dirac_compile_mrfilter_pilot.sh"])
j.setExecutable("dirac_compile_mrfilter_pilot.sh", "")
j.setOutputData(["mr_filter"], outputSE=None,
outputPath="cta/bin/mr_filter/v3_1/")
Dirac().submit(j)
break
else:
NJobs -= 1
j = Job()
# runtime in seconds times 8 (CPU normalisation factor)
j.setCPUTime(6 * 3600 * 8)
j.setName(job_name)
j.setInputSandbox(input_sandbox)
if banned_sites:
j.setBannedSites(banned_sites)
# j.setDestination( 'LCG.IN2P3-CC.fr' )
# mr_filter loses its executable property by uploading it to the GRID SE; reset
j.setExecutable('chmod', '+x mr_filter')
j.setExecutable('ls -lah')
for run_file in run_filelist:
file_token = re.split('_', run_file)[3]
# wait for a random number of seconds (up to five minutes) before starting
# to add a bit more entropy in the starting times of the dirac querries.
# if too many jobs try in parallel to access the SEs, the interface crashes
sleep = random.randint(0, 5 * 60)
j.setExecutable('sleep', str(sleep))
# consecutively downloads the data files, processes them, deletes the input
# and goes on to the next input file; afterwards, the output files are merged
j.setExecutable('dirac-dms-get-file', "LFN:" + run_file)
gLogger.error("file pipeline_wrapper not found in %s"%pipeline_wrapper)
dexit(1)
if os.path.isfile(os.path.join(pipeline_dict["PIPELINE_WORKDIR"],"script")):
script = os.path.join(pipeline_dict["PIPELINE_WORKDIR"],"script")
os.chmod(script,0755) # to make sure it's executable.
input_sandbox_files.append(script)
input_sandbox_files.append(pipeline_wrapper)
executable = "bash %s"%pipeline_wrapper
else:
executable = args[0].replace("bash ","").replace("./","")
if not os.path.isfile(executable):
gLogger.error("file %s not found."%executable)
dexit(1)
os.chmod(executable,0755) # make file executable
input_sandbox_files.append(executable)
j.setExecutable(str(executable))
else:
gLogger.error("No executable defined.")
dexit(1)
j.setName("MC job")
if not opts.name is None:
j.setName(opts.name)
j.setInputSandbox(input_sandbox_files) # all input files in the sandbox
j.setOutputSandbox(output_sandbox_files)
j.setCPUTime(opts.cpu)
if not opts.site is None:
j.setDestination(opts.site.split(","))#can also be a list
print("*")
print("* Uploading the following files:")
for fn in file_dict.keys():
print("*-> '%s'" % (fn))
print("*")
## The DIRAC job to submit.
j = Job(stdout='StdOut', stderr='StdErr')
# Set the name of the job (viewable in the web portal).
j.setName(jobname)
# As we're just copying the input sandbox to the storage element
# via OutputData, we'll just list the files as a check for the
# output written to StdOut.
j.setExecutable('/bin/ls -l')
# Here we add the names of the temporary copies of the frame data
# files in the dataset to the input sandbox. These will be uploaded
# to the grid with the job...
j.setInputSandbox(file_dict.keys())
#...and added to the desired storage element with the corresponding
# LFN via the job's OutputData. You may wish to change:
# * The Storage Element - by changing the outputSE parameter;
# * The LFN base name - by changing the outputPath parameter.
j.setOutputData(file_dict.keys(), \
outputSE='%s' % (se), \
outputPath='/%s/' % (gridoutdir)\
)
def helloWorldJob(): job = Job() job.setName( "helloWorld" ) job.setInputSandbox( '../../Integration/exe-script.py' ) job.setExecutable( "exe-script.py", "", "helloWorld.log" ) return job
print "Usage %s <scriptName> <jobName> <nbJobs>"%sys.argv[0]
sys.exit(1)
scriptName = sys.argv[1]
jobName = sys.argv[2]
nbJobs = int(sys.argv[3])
if not os.path.exists(jobName):
os.makedirs(jobName)
os.makedirs("%s/Done"%jobName)
os.makedirs("%s/Failed"%jobName)
else:
print "Folder %s exists"%jobName
sys.exit(1)
f = open("%s/jobIdList.txt"%jobName, 'w')
for i in xrange(nbJobs):
j = Job()
j.setCPUTime(10000)
j.setExecutable(scriptName)
j.addToOutputSandbox.append('myLog.txt')
j.addToOutputSandbox.append('clock.txt')
j.addToOutputSandbox.append('time.txt')
dirac = Dirac()
jobID = dirac.submit(j)
realId = jobID.get('JobID')
f.write("%s\n"%realId)
f.close()
