def test_SimpleParametricJob():
job = Job()
job.setExecutable('myExec')
job.setLogLevel('DEBUG')
parList = [1, 2, 3]
job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'],
['/lhcb/data/data3', '/lhcb/data/data4'],
['/lhcb/data/data5', '/lhcb/data/data6']]
job.setParameterSequence('InputData', inputDataList, addToWorkflow=True)
jdl = job._toJDL()
try:
with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd:
expected = fd.read()
except IOError:
with open('./Interfaces/API/test/testWF.jdl') as fd:
expected = fd.read()
assert jdl == expected
clad = ClassAd('[' + jdl + ']')
arguments = clad.getAttributeString('Arguments')
job_id = clad.getAttributeString('JOB_ID')
inputData = clad.getAttributeString('InputData')
assert job_id == '%(JOB_ID)s'
assert inputData == '%(InputData)s'
assert 'jobDescription.xml' in arguments
assert '-o LogLevel=DEBUG' in arguments
assert '-p JOB_ID=%(JOB_ID)s' in arguments
assert '-p InputData=%(InputData)s' in arguments
def test_fixed(self):
"""this tests executes a job that requires exactly 4 processors"""
j = Job()
j.setName("MP-test")
j.setExecutable(self.mpExe)
j.setInputSandbox(
find_all("mpTest.py", rootPath, "DIRAC/tests/Utilities")[0])
j.setNumberOfProcessors(
4) # This requires a fixed number of processors
j.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
j.setConfigArgs("pilot.cfg")
res = j.runLocal(self.d)
if multiprocessing.cpu_count() > 1:
self.assertTrue(res["OK"])
else:
self.assertFalse(res["OK"])
def test_min2(self):
"""this tests executes a job that requires at least 2 processors"""
j = Job()
j.setName("MP-test-min2")
# FIXME: the number of processors should be discovered at runtime using JobParameters.getNumberOfJobProcessors()
# here, and later
j.setExecutable(self.mpExeFlex, arguments="2")
j.setInputSandbox(
find_all("mpTest-flexible.py", rootPath,
"DIRAC/tests/Utilities")[0])
j.setNumberOfProcessors(
minNumberOfProcessors=2) # This requires at least 2 processors
j.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
j.setConfigArgs("pilot.cfg")
res = j.runLocal(self.d)
self.assertTrue(res["OK"])
def test_min2max4(self):
""" this tests executes a job that requires 2 to 4 processors
"""
j = Job()
j.setName("MP-test-min2max4")
j.setExecutable(self.mpExeFlex, arguments='2')
j.setInputSandbox(
find_all('mpTest-flexible.py', rootPath,
'DIRAC/tests/Utilities')[0])
j.setNumberOfProcessors(
minNumberOfProcessors=2,
maxNumberOfProcessors=4) # This requires 2 to 4 processors
j.setLogLevel('DEBUG')
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all('pilot.cfg',
os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
j.setConfigArgs('pilot.cfg')
res = j.runLocal(self.d)
self.assertTrue(res['OK'])
def test_SimpleParametricJob():
job = Job()
job.setExecutable("myExec")
job.setLogLevel("DEBUG")
parList = [1, 2, 3]
job.setParameterSequence("JOB_ID", parList, addToWorkflow=True)
inputDataList = [
["/lhcb/data/data1", "/lhcb/data/data2"],
["/lhcb/data/data3", "/lhcb/data/data4"],
["/lhcb/data/data5", "/lhcb/data/data6"],
]
job.setParameterSequence("InputData", inputDataList, addToWorkflow=True)
jdl = job._toJDL()
with open(join(dirname(__file__), "testWF.jdl")) as fd:
expected = fd.read()
assert jdl == expected
clad = ClassAd("[" + jdl + "]")
arguments = clad.getAttributeString("Arguments")
job_id = clad.getAttributeString("JOB_ID")
inputData = clad.getAttributeString("InputData")
assert job_id == "%(JOB_ID)s"
assert inputData == "%(InputData)s"
assert "jobDescription.xml" in arguments
assert "-o LogLevel=DEBUG" in arguments
assert "-p JOB_ID=%(JOB_ID)s" in arguments
assert "-p InputData=%(InputData)s" in arguments
def test_execute(self):
job = Job()
job.setName("helloWorld-test")
job.setExecutable(find_all("helloWorld.py", '.', 'Integration')[0],
arguments="This is an argument",
logFile="aLogFileForTest.txt",
parameters=[('executable', 'string', '',
"Executable Script"),
('arguments', 'string', '',
'Arguments for executable Script'),
('applicationLog', 'string', '',
"Log file name"),
('someCustomOne', 'string', '', "boh")],
paramValues=[('someCustomOne', 'aCustomValue')])
job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org'])
job.setOwner('ownerName')
job.setOwnerGroup('ownerGroup')
job.setName('jobName')
job.setJobGroup('jobGroup')
job.setType('jobType')
job.setDestination('DIRAC.someSite.ch')
job.setCPUTime(12345)
job.setLogLevel('DEBUG')
res = job.runLocal(self.d)
self.assertTrue(res['OK'])
def test_execute( self ):
job = Job()
job.setName( "helloWorld-test" )
job.setExecutable( find_all( "helloWorld.py", '.', 'Integration' )[0],
arguments = "This is an argument",
logFile = "aLogFileForTest.txt" ,
parameters=[('executable', 'string', '', "Executable Script"),
('arguments', 'string', '', 'Arguments for executable Script'),
( 'applicationLog', 'string', '', "Log file name" ),
( 'someCustomOne', 'string', '', "boh" )],
paramValues = [( 'someCustomOne', 'aCustomValue' )] )
job.setBannedSites( ['LCG.SiteA.com', 'DIRAC.SiteB.org'] )
job.setOwner( 'ownerName' )
job.setOwnerGroup( 'ownerGroup' )
job.setName( 'jobName' )
job.setJobGroup( 'jobGroup' )
job.setType( 'jobType' )
job.setDestination( 'DIRAC.someSite.ch' )
job.setCPUTime( 12345 )
job.setLogLevel( 'DEBUG' )
res = job.runLocal( self.d )
self.assertTrue( res['OK'] )
def test_SimpleParametricJob(self):
job = Job()
job.setExecutable('myExec')
job.setLogLevel('DEBUG')
parList = [1, 2, 3]
job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
inputDataList = [['/lhcb/data/data1', '/lhcb/data/data2'],
['/lhcb/data/data3', '/lhcb/data/data4'],
['/lhcb/data/data5', '/lhcb/data/data6']]
job.setParameterSequence('InputData',
inputDataList,
addToWorkflow=True)
jdl = job._toJDL()
print jdl
clad = ClassAd('[' + jdl + ']')
arguments = clad.getAttributeString('Arguments')
job_id = clad.getAttributeString('JOB_ID')
inputData = clad.getAttributeString('InputData')
print "arguments", arguments
self.assertEqual(job_id, '%(JOB_ID)s')
self.assertEqual(inputData, '%(InputData)s')
self.assertIn('jobDescription.xml', arguments)
self.assertIn('-o LogLevel=DEBUG', arguments)
self.assertIn('-p JOB_ID=%(JOB_ID)s', arguments)
self.assertIn('-p InputData=%(InputData)s', arguments)
def test_execute(self):
j = Job()
j.setName("helloWorld-test")
j.setExecutable(self.exeScriptLocation)
j.setLogLevel('DEBUG')
res = j.runLocal(self.d)
self.assertTrue(res['OK'])
def test_execute(self):
j = Job()
j.setName("helloWorld-test")
j.setExecutable(self.exeScriptLocation)
j.setLogLevel('DEBUG')
res = j.runLocal(self.d)
self.assertTrue(res['OK'])
def base():
job = Job()
job.setName("helloWorld-TEST-TO-Jenkins")
job.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]])
job.setExecutable("exe-script.py", "", "helloWorld.log")
job.setCPUTime(1780)
job.setDestination('DIRAC.Jenkins.ch')
job.setLogLevel('DEBUG')
return job
def test_execute(self):
""" just testing unix "ls"
"""
job = Job()
job.setName("ls-test")
job.setExecutable("/bin/ls", '-l')
job.setLogLevel('DEBUG')
res = job.runLocal(self.d)
self.assertTrue(res['OK'])
def test_execute(self):
""" just testing unix "ls"
"""
job = Job()
job.setName("ls-test")
job.setExecutable("/bin/ls", '-l')
job.setLogLevel('DEBUG')
res = job.runLocal(self.d)
self.assertTrue(res['OK'])
def base():
job = Job()
job.setName("helloWorld-TEST-TO-Jenkins")
executablePath = find_all("exe-script.py", "..",
"/DIRAC/tests/Workflow/")[0]
job.setInputSandbox([executablePath])
job.setExecutable(executablePath, "", "helloWorld.log")
job.setCPUTime(1780)
job.setDestination("DIRAC.Jenkins.ch")
job.setLogLevel("DEBUG")
return job
def test_MPJob(proc, minProc, maxProc, expectedProc, expectedMinProc, expectedMaxProc):
job = Job()
job.setExecutable("myExec")
job.setLogLevel("DEBUG")
job.setNumberOfProcessors(proc, minProc, maxProc)
jdl = job._toJDL()
clad = ClassAd("[" + jdl + "]")
processors = clad.getAttributeInt("NumberOfProcessors")
minProcessors = clad.getAttributeInt("MinNumberOfProcessors")
maxProcessors = clad.getAttributeInt("MaxNumberOfProcessors")
assert processors == expectedProc
assert minProcessors == expectedMinProc
assert maxProcessors == expectedMaxProc
def test_MPJob(proc, minProc, maxProc, expectedProc, expectedMinProc,
expectedMaxProc):
job = Job()
job.setExecutable('myExec')
job.setLogLevel('DEBUG')
job.setNumberOfProcessors(proc, minProc, maxProc)
jdl = job._toJDL()
clad = ClassAd('[' + jdl + ']')
processors = clad.getAttributeInt('NumberOfProcessors')
minProcessors = clad.getAttributeInt('MinNumberOfProcessors')
maxProcessors = clad.getAttributeInt('MaxNumberOfProcessors')
assert processors == expectedProc
assert minProcessors == expectedMinProc
assert maxProcessors == expectedMaxProc
def test_execute(self):
""" this one tests that I can execute a job that requires multi-processing
"""
j = Job()
j.setName("MP-test")
j.setExecutable(self.mpExe)
j.setInputSandbox(find_all('mpTest.py', rootPath, 'DIRAC/tests/Utilities')[0])
j.setTag('MultiProcessor')
j.setLogLevel('DEBUG')
res = j.runLocal(self.d)
if multiprocessing.cpu_count() > 1:
self.assertTrue(res['OK'])
else:
self.assertFalse(res['OK'])
def test_execute(self):
j = Job()
j.setName("helloWorld-test")
j.setExecutable(self.exeScriptLocation)
j.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
j.setConfigArgs("pilot.cfg")
res = j.runLocal(self.d)
self.assertTrue(res["OK"])
def test_execute(self):
j = Job()
j.setName("helloWorld-test")
j.setExecutable(self.exeScriptLocation)
j.setLogLevel('DEBUG')
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all('pilot.cfg',
os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
j.setConfigArgs('pilot.cfg')
res = j.runLocal(self.d)
self.assertTrue(res['OK'])
def test_SimpleParametricJob():
job = Job()
job.setExecutable('myExec')
job.setLogLevel('DEBUG')
parList = [1, 2, 3]
job.setParameterSequence('JOB_ID', parList, addToWorkflow=True)
inputDataList = [
[
'/lhcb/data/data1',
'/lhcb/data/data2'
],
[
'/lhcb/data/data3',
'/lhcb/data/data4'
],
[
'/lhcb/data/data5',
'/lhcb/data/data6'
]
]
job.setParameterSequence('InputData', inputDataList, addToWorkflow=True)
jdl = job._toJDL()
try:
with open('./DIRAC/Interfaces/API/test/testWF.jdl') as fd:
expected = fd.read()
except IOError:
with open('./Interfaces/API/test/testWF.jdl') as fd:
expected = fd.read()
assert jdl == expected
clad = ClassAd('[' + jdl + ']')
arguments = clad.getAttributeString('Arguments')
job_id = clad.getAttributeString('JOB_ID')
inputData = clad.getAttributeString('InputData')
assert job_id == '%(JOB_ID)s'
assert inputData == '%(InputData)s'
assert 'jobDescription.xml' in arguments
assert '-o LogLevel=DEBUG' in arguments
assert'-p JOB_ID=%(JOB_ID)s' in arguments
assert'-p InputData=%(InputData)s' in arguments
def test_execute(self):
"""just testing unix "ls" """
job = Job()
job.setName("ls-test")
job.setExecutable("/bin/ls", "-l")
job.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
job.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
job.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
job.setConfigArgs("pilot.cfg")
res = job.runLocal(self.d)
self.assertTrue(res["OK"])
def test_execute(self):
""" just testing unix "ls"
"""
job = Job()
job.setName("ls-test")
job.setExecutable("/bin/ls", '-l')
job.setLogLevel('DEBUG')
try:
# This is the standard location in Jenkins
job.setInputSandbox(
find_all('pilot.cfg',
os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
job.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
job.setConfigArgs('pilot.cfg')
res = job.runLocal(self.d)
self.assertTrue(res['OK'])
def test_SimpleParametricJob( self ):
job = Job()
job.setExecutable( 'myExec' )
job.setLogLevel( 'DEBUG' )
parList = [1,2,3]
job.setParameterSequence( 'JOB_ID', parList, addToWorkflow=True )
inputDataList = [
[
'/lhcb/data/data1',
'/lhcb/data/data2'
],
[
'/lhcb/data/data3',
'/lhcb/data/data4'
],
[
'/lhcb/data/data5',
'/lhcb/data/data6'
]
]
job.setParameterSequence( 'InputData', inputDataList, addToWorkflow=True )
jdl = job._toJDL()
print jdl
clad = ClassAd( '[' + jdl + ']' )
arguments = clad.getAttributeString( 'Arguments' )
job_id = clad.getAttributeString( 'JOB_ID' )
inputData = clad.getAttributeString( 'InputData' )
print "arguments", arguments
self.assertEqual( job_id, '%(JOB_ID)s' )
self.assertEqual( inputData, '%(InputData)s' )
self.assertIn( 'jobDescription.xml', arguments )
self.assertIn( '-o LogLevel=DEBUG', arguments )
self.assertIn( '-p JOB_ID=%(JOB_ID)s', arguments )
self.assertIn( '-p InputData=%(InputData)s', arguments )
def test_execute(self):
job = Job()
job._siteSet = {"DIRAC.someSite.ch"}
job.setName("helloWorld-test")
job.setExecutable(
self.helloWorld,
arguments="This is an argument",
logFile="aLogFileForTest.txt",
parameters=[
("executable", "string", "", "Executable Script"),
("arguments", "string", "", "Arguments for executable Script"),
("applicationLog", "string", "", "Log file name"),
("someCustomOne", "string", "", "boh"),
],
paramValues=[("someCustomOne", "aCustomValue")],
)
job.setBannedSites(["LCG.SiteA.com", "DIRAC.SiteB.org"])
job.setOwner("ownerName")
job.setOwnerGroup("ownerGroup")
job.setName("jobName")
job.setJobGroup("jobGroup")
job.setType("jobType")
job.setDestination("DIRAC.someSite.ch")
job.setCPUTime(12345)
job.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
job.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
job.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
job.setConfigArgs("pilot.cfg")
res = job.runLocal(self.d)
self.assertTrue(res["OK"])
def test_min1(self):
"""this tests executes a job that requires at least 1 processor"""
j = Job()
j.setName("MP-test-min1")
j.setExecutable(self.mpExeFlex, arguments="2")
j.setInputSandbox(
find_all("mpTest-flexible.py", rootPath,
"DIRAC/tests/Utilities")[0])
j.setNumberOfProcessors(
minNumberOfProcessors=1) # This requires 1 to infinite processors
j.setLogLevel("DEBUG")
try:
# This is the standard location in Jenkins
j.setInputSandbox(
find_all("pilot.cfg",
os.environ["WORKSPACE"] + "/PilotInstallDIR")[0])
except (IndexError, KeyError):
j.setInputSandbox(find_all("pilot.cfg", rootPath)[0])
j.setConfigArgs("pilot.cfg")
res = j.runLocal(self.d)
self.assertTrue(res["OK"])
def test_execute(self):
job = Job()
job._siteSet = {'DIRAC.someSite.ch'}
job.setName("helloWorld-test")
job.setExecutable(self.helloWorld,
arguments="This is an argument",
logFile="aLogFileForTest.txt",
parameters=[('executable', 'string', '',
"Executable Script"),
('arguments', 'string', '',
'Arguments for executable Script'),
('applicationLog', 'string', '',
"Log file name"),
('someCustomOne', 'string', '', "boh")],
paramValues=[('someCustomOne', 'aCustomValue')])
job.setBannedSites(['LCG.SiteA.com', 'DIRAC.SiteB.org'])
job.setOwner('ownerName')
job.setOwnerGroup('ownerGroup')
job.setName('jobName')
job.setJobGroup('jobGroup')
job.setType('jobType')
job.setDestination('DIRAC.someSite.ch')
job.setCPUTime(12345)
job.setLogLevel('DEBUG')
try:
# This is the standard location in Jenkins
job.setInputSandbox(
find_all('pilot.cfg',
os.environ['WORKSPACE'] + '/PilotInstallDIR')[0])
except (IndexError, KeyError):
job.setInputSandbox(find_all('pilot.cfg', rootPath)[0])
job.setConfigArgs('pilot.cfg')
res = job.runLocal(self.d)
self.assertTrue(res['OK'])
gLogger.setLevel('DEBUG')
cwd = os.path.realpath('.')
dirac = Dirac()
# Simple Hello Word job to DIRAC.Jenkins.ch
gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch")
helloJ = Job()
helloJ.setName("helloWorld-TEST-TO-Jenkins")
helloJ.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]])
helloJ.setExecutable("exe-script.py", "", "helloWorld.log")
helloJ.setCPUTime(1780)
helloJ.setDestination('DIRAC.Jenkins.ch')
helloJ.setLogLevel('DEBUG')
result = dirac.submitJob(helloJ)
gLogger.info("Hello world job: ", result)
if not result['OK']:
gLogger.error("Problem submitting job", result['Message'])
exit(1)
# Simple Hello Word job to DIRAC.Jenkins.ch, that needs to be matched by a MP WN
gLogger.info("\n Submitting hello world job targeting DIRAC.Jenkins.ch and a MP WN")
helloJMP = Job()
helloJMP.setName("helloWorld-TEST-TO-Jenkins-MP")
helloJMP.setInputSandbox([find_all('exe-script.py', '..', '/DIRAC/tests/Workflow/')[0]])
helloJMP.setExecutable("exe-script.py", "", "helloWorld.log")
helloJMP.setCPUTime(1780)
helloJMP.setDestination('DIRAC.Jenkins.ch')
helloJMP.setLogLevel('DEBUG')
j = Job(script=None, stdout='submission.out', stderr='submission.err')
j.setInputSandbox(input_file)
input_file_base = os.path.basename(input_file)
## prepare the list of output files
run_log = input_file_base + ".log"
dat = input_file_base.replace('aug', 'DAT')
datlong = dat + ".long"
output_files= [run_log, 'fluka11.out', 'fluka15.err', dat, datlong ]
## prepare the output location in GRID storage; the input path will be the used also for GRID storage
outdir = grid_basedir_output + path + "/" + str(runnr)
## ALWAYS, INFO, VERBOSE, WARN, DEBUG
j.setLogLevel('debug')
j.setName('AUGER test simulation')
j.setDestinationCE(ce1)
## j.setDestination(site_dirac)
j.setCPUTime(345600) ## 4 days
## download the script for preparing corsika input file for usage with cvmfs
j.setExecutable( 'curl', arguments = ' -fsSLkO http://issaf.spacescience.ro/adrian/AUGER/make_run4cvmfs',logFile='cmd_logs.log')
j.setExecutable( 'chmod', arguments = ' +x make_run4cvmfs',logFile='cmd_logs.log')
## create the simulation script configured for use with cvmfs
## set the make_run4cvmfs arguments to include the corsika_version and corsika_bin
make_run4cvmfs_arg = input_file_base + " " + corsika_version + " " + corsika_bin
