def test_a_CreateJob(self):
from Ganga.GPI import jobs, Job, Executable, Local
jobs.remove()
j = Job()
j.application = Executable()
j.backend = Local()
def submit_job(brunel_app, reco_type, input_files=None, local=RUN_LOCAL):
# Set EvtMax depending on if this is a local job
brunel_app.extraOpts += 'from Configurables import Brunel\n'
brunel_app.extraOpts += 'Brunel().EvtMax = {}'.format(2 * int(local) - 1)
# Configure the corresponding Job
job = Job(name='VP hybrid distortions',
comment='{reco_type} reconstruction {suffix}'.format(
reco_type=reco_type, suffix=['', '(local)'][local]),
application=brunel_app,
splitter=SplitByFiles(filesPerJob=1, ignoremissing=True),
parallel_submit=True)
if local:
job.backend = Local()
job.outputfiles = [LocalFile('*.xdst'), LocalFile('*.root')]
job.inputdata = dataset[:1]
else:
job.backend = Dirac()
job.outputfiles = [DiracFile('*.xdst'), DiracFile('*.root')]
job.inputdata = dataset
job.inputfiles = input_files or []
queues.add(job.submit)
return True
def test_Savannah18729(self):
from Ganga.GPI import Root, Job, Local
import os
from GangaTest.Framework.utils import sleep_until_completed
import tempfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
## Is this a test of files with a leading ' ' in the name? - rcurrie
#self.fname = os.path.join(tmpdir, ' test.C')
self.fname = os.path.join(tmpdir, 'test.C')
with open(self.fname, 'w') as f:
f.write('''
void test(const char* text, int i)
{
cout << gSystem->GetDynamicPath() << endl;
gSystem->Load("libTree");
cout << text << " " << i << endl;
}
''')
app = Root()
app.script = self.fname
app.args = ['abc', 1]
j = Job(backend=Local(), application=app)
j.submit()
self.assertTrue(sleep_until_completed(j, 120),
'Timeout on registering Interactive job as completed')
self.assertEqual(j.status, 'completed')
def setUp(self):
super(TestStructure, self).setUp()
from Ganga.GPI import Job, Executable, Local, File, LocalFile
from GangaTest.Framework.utils import write_file
self.jobslice = []
self.file_name = 'id_echo.sh'
for _ in range(5):
j = Job(application=Executable(), backend=Local())
scriptString = '''
#!/bin/sh
echo "Output from job $1." > out.txt
mkdir -p subdir
echo "Output from job $2." > subdir/out.txt
'''
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = 'sh'
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile('out.txt'), LocalFile('subdir/out.txt')]
self.jobslice.append(j)
def setUp(self):
super(TestSmartMerger, self).setUp()
from Ganga.GPI import Job, Executable, Local, File, LocalFile, config
config['Mergers']['associate'] = {'txt': 'TextMerger'}
self.jobslice = []
self.file_name = 'id_echo.sh'
for i in range(4):
j = Job(application=Executable(), backend=Local())
scriptString = '''
#!/bin/sh
echo "Output from job $1." > out.txt
echo "Output from job $2." > out2.txt
'''
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = 'sh'
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile('out.txt'), LocalFile('out2.txt')]
self.jobslice.append(j)
def test_e_testInMemory(self):
"""
Test the resubmit on a job in memory vs a job which has been loaded from disk
"""
from Ganga.GPI import Job, Local
j = Job()
j.splitter = self._getSplitter()
j.backend = Local()
j.submit()
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(j)
# Job has ben created, split, run and now exists in Memory (NOT SJXML)
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'resubmitOnlyFailedSubjobs', 'True')
j.resubmit()
sleep_until_completed(j)
j.subjobs(0).resubmit()
# We should get here if calling resubmit doesn't stall
j.subjobs(0).force_status('failed', force=True)
j.resubmit()
sleep_until_completed(j)
assert j.subjobs(0).status == 'completed'
# Test resubmit from the master job worked
j.subjobs(0).force_status('failed')
j.subjobs(0).resubmit()
sleep_until_completed(j)
assert j.subjobs(0).status == 'completed'
# Test that the resubmit from the subjob worked
setConfigOption('Configuration', 'resubmitOnlyFailedSubjobs', 'False')
j.resubmit()
sleep_until_completed(j)
j.subjobs(0).force_status('failed')
j.resubmit()
sleep_until_completed(j)
def test_e_UsingDifferentBackends(self):
from Ganga.GPI import Job, plugins, Local
# -- USINGDIFFERENTBACKENDS PLUGINS START
plugins("backends")
# -- USINGDIFFERENTBACKENDS PLUGINS STOP
# -- USINGDIFFERENTBACKENDS LOCAL START
j = Job()
j.backend = Local()
j.submit()
def test_a_JobSubmission(self):
"""
Create lots of subjobs and submit it
"""
from Ganga.GPI import Job, Local
j = Job()
j.application.exe = "sleep"
j.splitter = self._getSplitter()
j.backend = Local()
j.submit()
# Test we can submit a job and we're going to check the sj are created
assert len(j.subjobs) == TestSJSubmit.n_subjobs
def testLargeJobSubmission(self):
"""
Create lots of subjobs and submit it
"""
from Ganga.GPI import Job, GenericSplitter, Local
j = Job()
j.application.exe = "sleep"
j.splitter = GenericSplitter()
j.splitter.attribute = 'application.args'
j.splitter.values = [['400'] for _ in range(0, 20)]
j.backend = Local()
j.submit()
assert len(j.subjobs) == 20
def testMergeThatAlwaysFailsOverwrite(self):
from Ganga.GPI import Job, Executable, Local, LocalFile
j = Job()
j.application = Executable(exe='sh', args=['-c', 'echo foo > out.txt'])
j.backend = Local()
j.outputfiles = [LocalFile('out.txt')]
j.splitter = CopySplitter()
j.postprocessors = MergerTester(files=['out.txt'], overwrite=True)
j.submit()
assert run_until_state(j, 'failed', timeout=60)
assert os.path.exists(os.path.join(j.outputdir, 'out.txt.merge_summary')), 'Summary file should be created'
def test_i_MiscellaneousFunctionality(self):
from Ganga.GPI import JobTemplate, Local, Job, templates, jobtree
# -- MISCFUNCTIONALITY TEMPLATE1 START
j = JobTemplate()
j.name = 'LsExeLocal'
j.application.exe = 'ls'
j.backend = Local()
# -- MISCFUNCTIONALITY TEMPLATE1 STOP
# -- MISCFUNCTIONALITY TEMPLATE2 START
templates
# -- MISCFUNCTIONALITY TEMPLATE2 STOP
# -- MISCFUNCTIONALITY TEMPLATE3 START
j = Job(templates[0], name='JobFromTemplate')
j.submit()
# -- MISCFUNCTIONALITY TEMPLATE3 STOP
# -- MISCFUNCTIONALITY JOBTREE START
# show the current job tree (empty to start with)
jobtree
# make some dirs and subdirs
jobtree.mkdir('test_old')
jobtree.mkdir('test')
jobtree.mkdir('prod')
jobtree.mkdir('/test/jan')
jobtree.mkdir('/prod/full')
# have a look at the tree
jobtree.printtree()
# remove a dir
jobtree.rm('test_old')
# create some jobs and add them
jobtree.cd('/test/jan')
jobtree.add(Job())
jobtree.cd('/prod/full')
jobtree.add(Job())
jobtree.add(Job())
# look at the tree again
jobtree.printtree()
# submit the some jobs
jobtree.getjobs().submit()
def test_Savannah15630(self):
from Ganga.GPI import Job, Executable, Local, LocalFile
from GangaTest.Framework.utils import sleep_until_completed
j = Job()
j.application = Executable(exe='touch', args=['out.dat'])
j.backend = Local()
j.outputfiles = [LocalFile('out.dat')]
j.submit()
self.assertTrue(
sleep_until_completed(j, 60),
'Timeout on job submission: job is still not finished')
import os.path
p = os.path.join(j.outputdir, j.application.args[0])
self.assertTrue(os.path.exists(p))
def test_Savannah19123(self):
from Ganga.GPI import Job, Local
from GangaTest.Framework.utils import sleep_until_state
# check if stdout and stderr exists or not, flag indicates if files are required to exist or not
def check(exists_flag):
for fn in ['stdout', 'stderr']:
fn = os.path.join(j.outputdir, fn)
file_exists = os.path.exists(fn)
if exists_flag:
self.assertTrue(
file_exists,
'file %s should exist but it does not' % fn)
else:
self.assertFalse(
file_exists,
'file %s should not exist but it does' % fn)
j = Job()
j.application.exe = 'bash'
j.application.args = [
'-c', 'for i in `seq 1 30`; do echo $i; sleep 1; done'
]
j.backend = Local()
j.submit()
check(False)
if not sleep_until_state(j, 5, 'running'):
# problem with the test - print out stdout/stderr and assert
for fn in ['stdout', 'stderr']:
fn = os.path.join(j.outputdir, fn)
print " ---- Contents of " + fn
if os.path.exists(fn):
print open(fn).read()
else:
print "NO FILE AVAILABLE"
self.assertEqual(j.status, 'running')
j.kill()
check(True)
def setUp(self):
super(TestFileChecker, self).setUp()
from Ganga.GPI import Job, FileChecker, Executable, Local
self.c = FileChecker()
self.jobslice = []
args = ['1', '2', '12']
for arg in args:
j = Job(application=Executable(), backend=Local())
# write string to tmpfile
j.application.args = [arg]
self.jobslice.append(j)
for j in self.jobslice:
j.submit()
assert run_until_completed(
j), 'Timeout on job submission: job is still not finished'
self.assertEqual(j.status, 'completed')
def testKilling(self):
"""
Create some subjobs and kill them
"""
from Ganga.GPI import Job, GenericSplitter, Local
from GangaTest.Framework.utils import sleep_until_state
j = Job()
j.application.exe = "sleep"
j.splitter = GenericSplitter()
j.splitter.attribute = 'application.args'
j.splitter.values = [['400'] for _ in range(0, 5)]
j.backend = Local()
j.submit()
sleep_until_state(j, None, 'running')
assert j.status == 'running'
j.subjobs(0).kill()
assert j.subjobs(0).status == 'killed'
assert j.subjobs(1).status != 'killed'
j.kill()
assert j.status == 'killed'
assert all(sj.status == 'killed' for sj in j.subjobs)
def testMergeRemoval(self):
from Ganga.GPI import Job, Executable, Local, LocalFile, jobs
# see Savannah 33710
j = Job()
jobID = j.id
# job will run for at least 20 seconds
j.application = Executable(exe='sh',
args=['-c', 'sleep 20; echo foo > out.txt'])
j.backend = Local()
j.outputfiles = [LocalFile('out.txt')]
j.splitter = CopySplitter()
j.postprocessors = MergerTester(files=['out.txt'])
j.postprocessors[0].ignorefailed = True
j.postprocessors[0].alwaysfail = True
j.postprocessors[0].wait = 10
j.submit()
run_until_state(j, state='running')
j.remove()
with pytest.raises(KeyError):
jobs(jobID)
def Savannah19123(self):
from Ganga.GPI import Job, Local
from GangaTest.Framework.utils import sleep_until_state
# check if stdout and stderr exists or not, flag indicates if files are required to exist or not
def check(exists_flag):
for fn in ['stdout', 'stderr']:
fn = os.path.join(j.outputdir, fn)
file_exists = os.path.exists(fn)
if exists_flag:
self.assertTrue(
file_exists,
'file %s should exist but it does not' % fn)
else:
self.assertFalse(
file_exists,
'file %s should not exist but it does' % fn)
j = Job()
j.application.exe = 'bash'
j.application.args = [
'-c', 'for i in `seq 1 30`; do echo $i; sleep 1; done'
]
j.backend = Local()
j.submit()
check(False)
sleep_until_state(j, 5, 'running')
j.kill()
check(True)
def test_k_Tasks(self):
from Ganga.GPI import CoreTask, CoreTransform, Executable, Local, GenericSplitter, LocalFile, GangaDataset, \
GangaDatasetSplitter, TaskChainInput, File, tasks
# -- TASKS EXAMPLE START
# First create the overall Task
t = CoreTask()
# Now create the Transform ( -> Job template)
trf = CoreTransform()
trf.application = Executable()
trf.backend = Local()
# Set the unit splitter (unique to CoreTransform - you may have better ways of creating units in your own
# plugins). This will create a unit based on the splitting of any given splitter
# If you put in your own splitter here, use the trf.fields_to_copy string list to tell Tasks which fields of
# a Job to preserve from the split. Here, Tasks already knows about GenericSplitter and knows that we want to
# change the 'application' object for each Unit/Master Job
trf.unit_splitter = GenericSplitter()
trf.unit_splitter.attribute = "application.args"
trf.unit_splitter.values = ['arg 1', 'arg 2', 'arg 3']
# Append the transform
t.appendTransform(trf)
# set the maximum number of active jobs to have running (allows for throttling)
t.float = 100
# run the Task
t.run()
# -- TASKS EXAMPLE STOP
# -- TASKS OVERVIEW START
tasks
tasks(0).overview()
# -- TASKS OVERVIEW STOP
t = CoreTask()
trf = CoreTransform()
trf.application = Executable()
trf.backend = Local()
trf.unit_splitter = GenericSplitter()
trf.unit_splitter.attribute = "application.args"
trf.unit_splitter.values = ['arg 1', 'arg 2', 'arg 3']
t.appendTransform(trf)
t.float = 100
# -- TASKS OPTIONS START
# note - done at the transform level rather than task level as different backends may not need it
trf.max_active_threads = 10 # optional - specifies the max number of submissions to queue up
trf.submit_with_threads = True
# -- TASKS OPTIONS STOP
# -- TASKS JOBCHAIN START
# Create a test script
open('my_script3.sh', 'w').write("""#!/bin/bash
echo $PATH
ls -ltr
more __GangaInputData.txt__
echo "MY TEST FILE" > output_file.txt
sleep 120
""")
# Create the parent task
t = CoreTask()
# Create the first transform
trf1 = CoreTransform()
trf1.application = Executable()
trf1.application.exe = File('my_script3.sh')
trf1.outputfiles = [LocalFile("*.txt")]
d = GangaDataset()
d.files = [LocalFile("*.txt")]
d.treat_as_inputfiles = True
trf1.addInputData(d)
trf1.files_per_unit = 1
trf1.submit_with_threads = True
trf1.splitter = GangaDatasetSplitter()
trf1.splitter.files_per_subjob = 2
trf1.backend = Local()
t.appendTransform(trf1)
# Create the second transform
trf2 = CoreTransform()
trf2.application = Executable()
trf1.application.exe = File('my_script3.sh')
trf2.submit_with_threads = True
d = TaskChainInput()
d.input_trf_id = trf1.getID()
trf2.addInputData(d)
trf2.splitter = GangaDatasetSplitter()
trf2.splitter.files_per_subjob = 2
trf2.backend = Local()
t.appendTransform(trf2)
# Set the Task running
t.float = 1
t.run()
j = Job(name=JNAME.format(
polarity, year, mode
))
j.comment = (
'{1} {2} MC {0} ntuple creation for k3pi mixing measurement.'
.format(event_type, year, polarity)
)
j.application = DaVinci(version='v41r3')
j.application.optsfile = [s.format(path=base, year=year) for s in OPTIONS]
if args.test:
# If testing, run over a couple of files locally,
# saving the results to the sandbox
j.inputdata = dataset[0:1]
j.backend = Local()
# Prepend test string to job name
j.name = 'TEST_{0}'.format(j.name)
j.outputfiles = [LocalFile(tfn)]
else:
# If not testing, run over everything on the grid, splitting jobs
# into groups of 10 files, notifying me on job completion/subjob failure,
# and save the results on the grid storage
j.inputdata = dataset
j.backend = Dirac()
j.backend.settings['CPUTime'] = 60*60*24*7
j.do_auto_resubmit = True
j.splitter = SplitByFiles(filesPerJob=5, ignoremissing=True)
j.postprocessors = [Notifier(address=email)]
j.outputfiles = [DiracFile(tfn)]
def test_f_InputAndOutputData(self):
from Ganga.GPI import Job, File, LocalFile, GangaDataset, Local, plugins
# -- INPUTANDOUTPUTDATA BASIC START
# create a script to send
open('my_script2.sh', 'w').write("""#!/bin/bash
ls -ltr
more "my_input.txt"
echo "TESTING" > my_output.txt
""")
import os
os.system('chmod +x my_script2.sh')
# create a script to send
open('my_input.txt', 'w').write('Input Testing works!')
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [LocalFile('my_input.txt')]
j.outputfiles = [LocalFile('my_output.txt')]
j.submit()
# -- INPUTANDOUTPUTDATA BASIC STOP
# -- INPUTANDOUTPUTDATA PEEKOUTPUT START
j.peek() # list output dir contents
j.peek('my_output.txt')
# -- INPUTANDOUTPUTDATA PEEKOUTPUT STOP
# -- INPUTANDOUTPUTDATA FAILJOB START
# This job will fail
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [LocalFile('my_input.txt')]
j.outputfiles = [LocalFile('my_output_FAIL.txt')]
j.submit()
# -- INPUTANDOUTPUTDATA FAILJOB STOP
# -- INPUTANDOUTPUTDATA WILDCARD START
# This job will pick up both 'my_input.txt' and 'my_output.txt'
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [LocalFile('my_input.txt')]
j.outputfiles = [LocalFile('*.txt')]
j.submit()
# -- INPUTANDOUTPUTDATA WILDCARD STOP
# -- INPUTANDOUTPUTDATA OUTPUTFILES START
j.outputfiles
# -- INPUTANDOUTPUTDATA OUTPUTFILES STOP
# -- INPUTANDOUTPUTDATA INPUTDATA START
# Create a test script
open('my_script3.sh', 'w').write("""#!/bin/bash
echo $PATH
ls -ltr
more __GangaInputData.txt__
echo "MY TEST FILE" > output_file.txt
""")
import os
os.system('chmod +x my_script3.sh')
# Submit a job
j = Job()
j.application.exe = File('my_script3.sh')
j.inputdata = GangaDataset(files=[LocalFile('*.sh')])
j.backend = Local()
j.submit()
# -- INPUTANDOUTPUTDATA INPUTDATA STOP
# -- INPUTANDOUTPUTDATA GANGAFILES START
plugins('gangafiles')