def setUp(self):
super(TestStructure, self).setUp()
from Ganga.GPI import Job, Executable, Local, File, LocalFile
from GangaTest.Framework.utils import write_file
self.jobslice = []
self.file_name = "id_echo.sh"
for _ in range(5):
j = Job(application=Executable(), backend=Local())
scriptString = """
#!/bin/sh
echo "Output from job $1." > out.txt
mkdir -p subdir
echo "Output from job $2." > subdir/out.txt
"""
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = "sh"
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile("out.txt"), LocalFile("subdir/out.txt")]
self.jobslice.append(j)
def testStartUp():
""" Lets test the startup of Ganga mimicking first launch """
# Process options given at command line and in configuration file(s)
# Perform environment setup and bootstrap
from Ganga.Runtime import setupGanga
argv = ['ganga', '--no-mon', '-o[Configuration]gangadir=%s' % this_dir, '-o[Configuration]RUNTIME_PATH=GangaTest']
setupGanga(argv=argv, interactive=False)
for this_file in ['.gangarc', '.ganga.log']:
assert os.path.isfile(os.path.join(homeDir, this_file))
# No way known to mimic IPython starting up in a simple way
#assert os.path.isdir(os.path.join(homeDir, '.ipython-ganga'))
for this_folder in ['repository',]:
assert os.path.isdir(os.path.join(this_dir, this_folder))
from Ganga.GPI import Job
j=Job()
j.submit()
for this_folder in ['shared', 'workspace']:
assert os.path.isdir(os.path.join(this_dir, this_folder))
def test_Savannah8009(self):
from Ganga.GPI import Executable, Job, jobs, templates
from GangaTest.Framework.utils import sleep_until_completed
j = Job()
j.submit()
self.assertEqual(len(jobs), 1)
self.assertEqual(len(templates), 0)
if not sleep_until_completed(j, timeout=120):
assert not "Timeout on job submission: job is still not finished"
t = j.copy()
# make sure that copy creates a new job (and not the template)
self.assertEqual(len(jobs), 2)
self.assertEqual(len(templates), 0)
# make sure that output parameters are not carried forward
self.assertNotEqual(j.backend.id, t.backend.id)
self.assertNotEqual(j.backend.exitcode, t.backend.exitcode)
# make sure that input parameters are carried forward
self.assertEqual(j.application.exe, t.application.exe)
def test_job_complete(gpi):
from Ganga.GPI import Job, LCG
j = Job()
j.backend = LCG()
j.submit()
assert run_until_completed(j, timeout=1200, sleep_period=10), 'Timeout on job submission: job is still not finished'
def test_Savannah8111(self):
from Ganga.GPI import Job
j1 = Job()
j1.name = 'Gauss Job'
jj = j1.copy()
self.assertEqual(jj.name, 'Gauss Job')
def testInternal(self):
from Ganga.GPI import GaudiRun, Job, LocalFile, DiracFile
tmp_fol = gettempdir()
gaudi_testFol = path.join(tmp_fol, 'GaudiRunTest')
shutil.rmtree(gaudi_testFol, ignore_errors=True)
makedirs(gaudi_testFol)
gaudi_testOpts = path.join(gaudi_testFol, 'testOpts.py')
with open(gaudi_testOpts, 'w+') as temp_opt:
temp_opt.write("print('hello')")
gr = GaudiRun(directory=gaudi_testFol, myOpts=LocalFile(gaudi_testOpts))
assert isinstance(stripProxy(gr).getOptsFile(), stripProxy(LocalFile))
assert stripProxy(gr).getDir()
assert open(path.join(stripProxy(gr).getOptsFile().localDir, stripProxy(gr).getOptsFile().namePattern)).read() == "print('hello')"
assert stripProxy(gr).getDir() == gaudi_testFol
j=Job()
j.application = gr
assert isinstance(j.application, GaudiRun)
df = DiracFile(lfn='/not/some/file')
gr.myOpts = df
assert gr.myOpts.lfn == df.lfn
shutil.rmtree(gaudi_testFol, ignore_errors=True)
def testSavannah13404(self):
from Ganga.GPI import Job
j=Job()
assert( os.path.exists( j.inputdir + '/..' ) )
j.remove()
# The job directory should be deleted
assert( not os.path.exists( os.path.abspath( j.inputdir + '/..' ) ) )
def _check(self, template):
logger.info("------------------------------------------------")
logger.info("- Now checking template: '%s'" % template.name)
logger.info("------------------------------------------------")
j = Job(template)
j.submit()
self.assertTrue(sleep_until_completed(j))
def testDatasets(self):
from Ganga.GPI import DiracFile, PhysicalFile, LHCbDataset, Job, LocalFile
# test constructors/setters
ds = LHCbDataset(['lfn:a', 'pfn:b'])
assert len(ds) == 2
print(ds[0])
assert isinstance(ds[0], DiracFile)
assert isinstance(ds[1], PhysicalFile)
ds = LHCbDataset()
ds.files = ['lfn:a', 'pfn:b']
assert isinstance(ds[0], DiracFile)
assert isinstance(ds[1], PhysicalFile)
ds.files.append('lfn:c')
assert isinstance(ds[-1], DiracFile)
d = OutputData(['a', 'b'])
assert isinstance(d.files[0],str)
assert isinstance(d.files[1],str)
# check job assignments
j = Job()
j.inputdata = ['lfn:a', 'pfn:b']
assert isinstance(j.inputdata, LHCbDataset)
j.outputfiles = ['a', DiracFile('b')]
assert isinstance(j.outputfiles[0], LocalFile)
print(type(j.outputfiles[1]))
assert isinstance(j.outputfiles[1], DiracFile)
LFN_DATA = ['LFN:/lhcb/LHCb/Collision11/DIMUON.DST/00016768/0000/00016768_00000005_1.dimuon.dst',
'LFN:/lhcb/LHCb/Collision11/DIMUON.DST/00016768/0000/00016768_00000006_1.dimuon.dst']
ds = LHCbDataset(LFN_DATA)
assert len(ds.getReplicas().keys()) == 2
assert ds.getCatalog()
def test_Savannah18729(self):
from Ganga.GPI import Root, Job, Local
import os
from GangaTest.Framework.utils import sleep_until_completed
import tempfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
## Is this a test of files with a leading ' ' in the name? - rcurrie
#self.fname = os.path.join(tmpdir, ' test.C')
self.fname = os.path.join(tmpdir, 'test.C')
with open(self.fname, 'w') as f:
f.write('''
void test(const char* text, int i)
{
cout << gSystem->GetDynamicPath() << endl;
gSystem->Load("libTree");
cout << text << " " << i << endl;
}
''')
app = Root()
app.script = self.fname
app.args = ['abc', 1]
j = Job(backend=Local(), application=app)
j.submit()
self.assertTrue(sleep_until_completed(j,120), 'Timeout on registering Interactive job as completed')
self.assertEqual(j.status, 'completed')
def test_a_InstallAndBasicUsage(self):
from Ganga.GPI import Job, jobs
# -- INSTALLANDBASICUSAGE HELP START
help(Job)
# -- INSTALLANDBASICUSAGE HELP STOP
# -- INSTALLANDBASICUSAGE SUBMIT START
j = Job()
j.submit()
# -- INSTALLANDBASICUSAGE SUBMIT STOP
# -- INSTALLANDBASICUSAGE JOBS START
jobs(0)
# -- INSTALLANDBASICUSAGE JOBS STOP
# -- INSTALLANDBASICUSAGE JOBSAPP START
jobs(0).application
# -- INSTALLANDBASICUSAGE JOBSAPP STOP
# -- INSTALLANDBASICUSAGE EXECFILE START
open('submit.py', 'w').write("""
j = Job()
j.submit()
""")
execfile('submit.py')
def test_Savannah96158(self):
from Ganga.GPI import Job, jobs
#The first two tests check the new functionality, the remainder just check that we didn't break existing functionality with this bug-fix
a = Job()
a.name = 'TestName'
tmpList = jobs.select(name='*stN*')
self.assertEqual(len(tmpList), 1, 'Test 1: jobs.select using wildcard returned unexpected number of results')
a = Job()
a.name = 'ekdicjsheeksoawoq1'
a = Job()
a.name = 'ekdicjsheeksoawoq2'
a = Job()
a.name = 'ekdicjsheeksoawoq3'
a = Job()
a.name = 'ekdicjsheeksoawoq4'
tmpList = jobs.select(name='ekdicjsheeksoawoq?')
self.assertEqual(len(tmpList), 4, 'Test 2: jobs.select using wildcard returned unexpected number of results')
jobs.select(1)
jobs.select(1, 4)
jobs.select(status='new')
jobs.select(backend='Local')
jobs.select(application='Executable')
def setUp(self):
super(TestCustomMerger, self).setUp()
from Ganga.GPI import Job, Executable, Local, File, LocalFile
self.jobslice = []
self.file_name = 'id_echo.sh'
for i in range(2):
j = Job(application=Executable(), backend=Local())
scriptString = '''
#!/bin/sh
echo "Output from job $1." > out.txt
echo "Output from job $2." > out2.txt
'''
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = 'sh'
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile('out.txt'), LocalFile('out2.txt')]
self.jobslice.append(j)
def test_submit_kill_resubmit(gpi):
"""
Test that a simple submit-kill-resubmit-kill cycle works
"""
from Ganga.GPI import Job, LCG
j = Job()
j.backend = LCG()
with patch('Ganga.Lib.LCG.Grid.submit', return_value='https://example.com:9000/42') as submit:
j.submit()
submit.assert_called_once()
assert j.backend.id == 'https://example.com:9000/42'
with patch('Ganga.Lib.LCG.Grid.cancel', return_value=True) as cancel:
j.kill()
cancel.assert_called_once()
assert j.status == 'killed'
with patch('Ganga.Lib.LCG.Grid.submit', return_value='https://example.com:9000/43') as submit:
j.resubmit()
submit.assert_called_once()
assert j.backend.id == 'https://example.com:9000/43'
with patch('Ganga.Lib.LCG.Grid.cancel', return_value=True):
j.kill()
def test_AddRemove(self):
from Ganga.GPI import Job, jobs
j = Job()
assert(len(jobs) == 1)
j.remove()
assert(len(jobs) == 0)
def test_a_JobConstruction(self):
from Ganga.GPI import Job, jobs
j = Job()
assert len(jobs) == 1
j.submit()
assert j.status != 'new'
def test_Savannah13404(self):
from Ganga.GPI import Job
j = Job()
j.submit() # Needed in order to create the workspace
self.assertTrue(os.path.exists(j.inputdir + '/..'))
j.remove()
# The job directory should be deleted
self.assertFalse(os.path.exists(os.path.abspath(j.inputdir + '/..')))
def test_c_onlyCreate(self):
"""here for testing job create"""
from Ganga.GPI import Job, Executable, ArgSplitter, MassStorageFile
j=Job()
j.application=Executable(exe='touch')
j.splitter=ArgSplitter(args=[['abc.txt'], ['def.txt']])
j.outputfiles=[MassStorageFile(outputfilenameformat = '/test/{sjid}-{fname}', namePattern = '*.txt')]
def test_d_anotherNewJob(self):
from Ganga.GPI import Job, jobs
j=Job()
j.submit()
self.assertNotEqual(j.status, 'new')
def testSavannah10064(self):
from Ganga.GPI import Job, TestSubmitter, TestApplication
j = Job(backend=TestSubmitter(),application=TestApplication())
j.submit()
import os.path
assert(os.path.exists(j.inputdir))
templates.remove()
assert(os.path.exists(j.inputdir))
def test_Savannah10064(self):
from Ganga.GPI import Job, templates
j = Job()
j.submit()
import os.path
self.assertTrue(os.path.exists(j.inputdir))
templates.remove()
self.assertTrue(os.path.exists(j.inputdir))
def Savannah33303(self):
from Ganga.GPI import Job
j = Job()
id = j.id
for i in xrange(1, 20):
j = j.copy()
self.assertEqual(j.id, id + i)
def test_job_submit_and_monitor(gpi):
from Ganga.GPI import Job, LCG
j = Job()
j.backend = LCG()
j.submit()
assert j.status != 'new'
stripProxy(LCG).master_updateMonitoringInformation([stripProxy(j)])
def test_a_JobConstruction(self):
""" First construct the Job object (singular)"""
from Ganga.Utility.Config import getConfig
self.assertFalse(getConfig('TestingFramework')['AutoCleanup'])
from Ganga.GPI import Job, jobs
j=Job()
assert len(jobs) == 1
j.name = 'modified_name'
def test_c_JobManipulation(self):
from Ganga.GPI import runMonitoring, Job, jobs, export, load
runMonitoring()
# -- JOBMANIPULATION JOBCOPY START
j = Job(name = 'original')
j2 = j.copy()
j2.name = 'copy'
j.submit()
j3 = Job(j, name = 'copy2')
jobs
# -- JOBMANIPULATION JOBCOPY STOP
# -- JOBMANIPULATION REPOACCESS START
jobs(2)
# -- JOBMANIPULATION REPOACCESS STOP
# -- JOBMANIPULATION JOBSLICING START
jobs[2]
jobs[2:]
jobs['copy2']
# -- JOBMANIPULATION JOBSLICING STOP
jobs(0).kill()
# -- JOBMANIPULATION RESUBMIT START
jobs(0).resubmit()
# -- JOBMANIPULATION RESUBMIT STOP
# -- JOBMANIPULATION FORCESTATUS START
jobs(1).force_status('failed')
# -- JOBMANIPULATION FORCESTATUS STOP
# -- JOBMANIPULATION JOBREMOVE START
jobs(2).remove()
# -- JOBMANIPULATION JOBREMOVE STOP
# -- JOBMANIPULATION JOBSELECT START
# can select on ids, name, status, backend, application
jobs.select(status='new')
jobs.select(backend='Local')
jobs.select(ids=[1,3])
# can restrict on min/max id
jobs.select(1,3, application='Executable')
# -- JOBMANIPULATION JOBSELECT STOP
# -- JOBMANIPULATION JOBSELECTOP START
jobs.select(status='new').submit()
# -- JOBMANIPULATION JOBSELECTOP STOP
# -- JOBMANIPULATION EXPORTJOB START
export(jobs(0), 'my_job.txt')
jlist = load('my_job.txt')
jlist[0].submit()
def test_a_JobConstruction(self):
from Ganga.GPI import Job, jobs, disableMonitoring
j = Job()
self.assertEqual(len(jobs), 1)
j.submit()
self.assertNotEqual(j.status, 'new')
def test_Savannah9007(self):
from Ganga.GPI import Job
j = Job()
from Ganga.Utility.Config import getConfig
if not getConfig('Output')['ForbidLegacyInput']:
j.inputsandbox = ['x']
else:
j.inputfiles = ['x']
def test_job_equality(self):
"""Check that copies of Jobs are equal to each other"""
from Ganga.GPI import Job
j = Job()
j2 = j.copy()
j3 = Job(j)
assert j == j2
assert j2 == j3
assert stripProxy(j) == stripProxy(j2)
assert stripProxy(j2) == stripProxy(j3)
def test_Savannah9007(self):
from Ganga.GPI import Job
j = Job()
from Ganga.Utility.Config import getConfig
if not getConfig('Output')['ForbidLegacyInput']:
j.inputsandbox = ['x']
else:
j.inputfiles = ['x']
def testSubmitLocal(self):
from Ganga.GPI import DaVinci, Job, TestSubmitter, JobError
from GangaLHCb.testlib import addLocalTestSubmitter
ap = DaVinci()
j = Job(application=ap, backend=TestSubmitter())
# Test that submission fails before adding runtime handler
with pytest.raises(JobError):
j.submit()
def test_a_JobConstruction(self):
""" First construct the Job object (singular)"""
from Ganga.Utility.Config import getConfig
self.assertFalse(getConfig('TestingFramework')['AutoCleanup'])
from Ganga.GPI import Job, jobs
for i in range(job_num):
j = Job()
j.name = job_names[i]
self.assertEqual(len(jobs), job_num) # Don't really gain anything from assertEqual...
def test_a_JobConstruction(self):
from Ganga.GPI import Job, jobs, disableMonitoring
j=Job()
self.assertEqual(len(jobs), 1)
j.submit()
self.assertNotEqual(j.status, 'new')
def submit_job(brunel_app, reco_type, input_files=None, local=RUN_LOCAL):
# Set EvtMax depending on if this is a local job
brunel_app.extraOpts += 'from Configurables import Brunel\n'
brunel_app.extraOpts += 'Brunel().EvtMax = {}'.format(2 * int(local) - 1)
# Configure the corresponding Job
job = Job(name='VP hybrid distortions',
comment='{reco_type} reconstruction {suffix}'.format(
reco_type=reco_type, suffix=['', '(local)'][local]),
application=brunel_app,
splitter=SplitByFiles(filesPerJob=1, ignoremissing=True),
parallel_submit=True)
if local:
job.backend = Local()
job.outputfiles = [LocalFile('*.xdst'), LocalFile('*.root')]
job.inputdata = dataset[:1]
else:
job.backend = Dirac()
job.outputfiles = [DiracFile('*.xdst'), DiracFile('*.root')]
job.inputdata = dataset
job.inputfiles = input_files or []
queues.add(job.submit)
return True
def testArgSplitter(self):
from Ganga.GPI import Job, ArgSplitter
from GangaTest.Framework.utils import sleep_until_completed
j = Job()
j.splitter = ArgSplitter(args=[['1'], ['2'], ['3']])
j.submit()
self.assertTrue(sleep_until_completed(j, 60), 'Timeout on completing job')
self.assertEqual(len(j.subjobs), 3)
def test_clean(self): # asaroka
from Ganga.GPI import jobtree, Job
jobtree.cd()
jobtree.mkdir('testdir')
jobtree.add(Job())
print("%s" % jobtree)
jobtree.add(Job(), 'testdir')
jobtree.rm('/*')
assert (jobtree.listjobs('/') == [])
assert (jobtree.listdirs('/') == [])
def test_c_onlyCreate(self):
"""here for testing job create"""
from Ganga.GPI import Job, Executable, ArgSplitter, MassStorageFile
j = Job()
j.application = Executable(exe='touch')
j.splitter = ArgSplitter(args=[['abc.txt'], ['def.txt']])
j.outputfiles = [
MassStorageFile(outputfilenameformat='/test/{sjid}-{fname}',
namePattern='*.txt')
]
def test_b_EnableMonitoring(self):
from Ganga.GPI import enableMonitoring, Job, jobs
enableMonitoring()
j = Job()
j.submit()
dummySleep(j)
self.assertNotEqual(jobs(0).status, 'submitted')
def test_e_UsingDifferentBackends(self):
from Ganga.GPI import Job, plugins, Local
# -- USINGDIFFERENTBACKENDS PLUGINS START
plugins("backends")
# -- USINGDIFFERENTBACKENDS PLUGINS STOP
# -- USINGDIFFERENTBACKENDS LOCAL START
j = Job()
j.backend = Local()
j.submit()
def test_A_Construction(self):
from Ganga.GPI import Job
j = Job()
assert (j.application.is_prepared == None)
j.prepare()
assert (j.application.is_prepared != None)
TestShared.shared_area_location = j.application.is_prepared.path()
assert (path.isdir(TestShared.shared_area_location))
def testNoFilesSpecifiedAllSame(self):
from Ganga.GPI import LocalFile, Job
files = [LocalFile('foo.root'), LocalFile(
'bar.root'), LocalFile('out.log')]
j1 = Job(outputfiles=files)
j2 = Job(outputfiles=files)
assert j1.outputfiles == j2.outputfiles, 'File lists should be the same'
assert findFilesToMerge([j1, j2]) == ['foo.root', 'bar.root', 'out.log'], 'Should merge all files'
def test_a_JobConstruction(self):
""" First construct the Job object (singular)"""
from Ganga.Utility.Config import getConfig
self.assertFalse(getConfig('TestingFramework')['AutoCleanup'])
from Ganga.GPI import Job, jobs
for i in range(job_num):
j = Job()
j.name = job_names[i]
self.assertEqual(
len(jobs),
job_num) # Don't really gain anything from assertEqual...
def test_a_JobConstruction(self):
""" First construct the Job object (singular)"""
from Ganga.Utility.Config import getConfig
self.assertFalse(getConfig('TestingFramework')['AutoCleanup'])
from Ganga.GPI import Job, jobs
j = Job()
assert len(jobs) == 1
global global_AutoStartReg
global_AutoStartReg = False
j.name = 'modified_name'
def test_a_JobConstruction(self):
""" First construct the Job object (singular)"""
from Ganga.Utility.Config import getConfig
self.assertFalse(getConfig('TestingFramework')['AutoCleanup'])
from Ganga.GPI import Job, jobs, ArgSplitter
j=Job()
assert len(jobs) == 1
j.splitter = ArgSplitter()
j.splitter.args = getNestedList()
assert j.splitter.args == getNestedList()
def test_Savannah43249(self):
from Ganga.Core.exceptions import GangaException
from Ganga.GPI import Job, jobs
Job()
Job()
Job()
self.assertRaises(GangaException, jobs.remove, 10)
self.assertRaises(TypeError, jobs.remove, x=True)
jobs.remove(keep_going=True)
def test_Savannah13406(self):
#from Ganga.GPI import config
#config['Configuration']['autoGenerateJobWorkspace'] = True
import os
from Ganga.GPI import Job, jobs
jobs.remove()
j = Job()
j.submit() # Needed in order to create the workspace
self.assertTrue(os.path.exists(j.inputdir + '/../..'))
jobs.remove()
# Check is repository/Local or repository/Remote still exists
self.assertTrue(os.path.exists(os.path.abspath(j.inputdir + '/../..')))
def testSubjobsSubmit(self):
from Ganga.GPI import Job, Executable, TestSubmitter, ArgSplitter
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as gangaList
from Ganga.GPIDev.Base.Proxy import isType
j = Job(application=Executable(), backend=TestSubmitter(time=1))
j.splitter = ArgSplitter(args=[['A'], ['B'], ['C']])
j.submit()
assert run_until_completed(j), 'Job must complete'
assert len(j.subjobs) == 3, 'splitting must occur'
for jj in j.subjobs:
assert not isType(jj.master, gangaList)
def testSubmitLocal(self):
from Ganga.GPI import DaVinci, Job, TestSubmitter, JobError
from GangaLHCb.testlib import addLocalTestSubmitter
ap = DaVinci()
j = Job(application=ap, backend=TestSubmitter())
# Test that submission fails before adding runtime handler
with pytest.raises(JobError):
j.submit()
# Test that submission succeeds after adding it.
addLocalTestSubmitter()
assert j.submit()
def test_j_Queues(self):
from Ganga.GPI import queues, Job, GenericSplitter
queues
# -- QUEUES EXAMPLE START
for i in range(1, 10):
j = Job()
queues.add(j.submit)
# -- QUEUES EXAMPLE STOP
# -- QUEUES FUNCTION START
def f(x):
print x
queues.add(f, args=(123, ))
# -- QUEUES FUNCTION STOP
# -- QUEUES SPLIT START
j = Job()
j.splitter = GenericSplitter()
j.splitter.attribute = 'application.args'
j.splitter.values = [i for i in range(0, 10)]
j.parallel_submit = True
j.submit()
def test_e_testXMLContent(self):
# Check content of XML is as expected
from Ganga.Core.GangaRepository.VStreamer import to_file, from_file
from Ganga.GPI import jobs, Job
from Ganga.GPIDev.Base.Proxy import stripProxy
from tempfile import NamedTemporaryFile
j = jobs(0)
assert path.isfile(getXMLFile(j))
with open(getXMLFile(j)) as handler:
tmpobj, errs = from_file(handler)
assert hasattr(tmpobj, 'name')
assert tmpobj.name == testStr
ignore_subs = [
'time', 'subjobs', 'info', 'application', 'backend', 'id'
]
with NamedTemporaryFile(delete=False) as new_temp_file:
temp_name = new_temp_file.name
to_file(stripProxy(j), new_temp_file, ignore_subs)
new_temp_file.flush()
with NamedTemporaryFile(delete=False) as new_temp_file2:
temp_name2 = new_temp_file2.name
j2 = Job()
j2.name = testStr
j2.submit()
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(j2)
to_file(stripProxy(j2), new_temp_file2, ignore_subs)
new_temp_file2.flush()
#import filecmp
#assert filecmp.cmp(handler.name, new_temp_file.name)
#assert not filecmp.cmp(new_temp_file.name, new_temp_file2.name)
#assert open(getXMLFile(j)).read() == open(temp_name).read()
assert open(temp_name).read() == open(temp_name2).read()
unlink(temp_name)
unlink(temp_name2)
def test_f_reallyDisabled(self):
from Ganga.GPI import disableMonitoring, enableMonitoring, Job
disableMonitoring()
j = Job()
j.submit()
self.assertEqual(j.status, 'submitted')
enableMonitoring()
dummySleep(j)
self.assertEqual(j.status, 'completed')
def test_Savannah8534(self):
from Ganga.GPI import Job, TestApplication, TestSubmitter
app = TestApplication()
j = Job(backend=TestSubmitter(time=30), application=app)
j.submit()
self.assertNotEqual(j.status, 'new')
j2 = j.copy()
# make sure that the status is reset correctly as well as the output parameters
self.assertEqual(j2.status, 'new')
self.assertEqual(j2.backend.start_time, 0)
def test_e_reEnableMon(self):
from Ganga.GPI import disableMonitoring, enableMonitoring, Job, jobs
disableMonitoring()
enableMonitoring()
disableMonitoring()
enableMonitoring()
j = Job()
j.submit()
dummySleep(j)
self.assertEqual(j.status, 'completed')
def testNoFilesSpecifiedNoOverlap(self):
from Ganga.GPI import LocalFile, Job
j1 = Job(outputfiles=[
LocalFile('foo.root'),
LocalFile('bar.root'),
LocalFile('out.log')
])
j2 = Job(outputfiles=[
LocalFile('a.root'),
LocalFile('b.root'),
LocalFile('c.log')
])
assert findFilesToMerge([j1, j2]) == [], 'Should merge no files'
def test_lcg(gpi):
from Ganga.GPI import Job, LCG, VomsProxy, credential_store, jobs
logger.info('Submitting first job')
j1 = Job()
j1.backend = LCG()
j1.submit()
logger.info('Submitting second job')
j2 = Job()
j2.backend = LCG(credential_requirements=VomsProxy(vo='lhcb'))
j2.submit()
# Wipe out all the credentials to make sure they can be created on cue
for cred in credential_store:
cred.destroy()
logger.info('Monitoring jobs')
for j in jobs:
stripProxy(j).backend.master_updateMonitoringInformation(
[stripProxy(j)])
# Wipe out all the credentials to make sure they can be created on cue
credential_store.clear()
logger.info('Monitoring jobs')
for j in jobs:
stripProxy(j).backend.master_updateMonitoringInformation(
[stripProxy(j)])
# Wipe out all the credentials to make sure they can be created on cue
for cred in credential_store:
cred.destroy()
credential_store.clear()
logger.info('Monitoring jobs')
for j in jobs:
stripProxy(j).backend.master_updateMonitoringInformation(
[stripProxy(j)])
# Wipe out all the credentials to make sure they can be created on cue
for cred in credential_store:
cred.destroy()
credential_store.clear()
logger.info('Killing jobs')
for j in jobs:
j.kill()
def test_i_MiscellaneousFunctionality(self):
from Ganga.GPI import JobTemplate, Local, Job, templates, jobtree
# -- MISCFUNCTIONALITY TEMPLATE1 START
j = JobTemplate()
j.name = 'LsExeLocal'
j.application.exe = 'ls'
j.backend = Local()
# -- MISCFUNCTIONALITY TEMPLATE1 STOP
# -- MISCFUNCTIONALITY TEMPLATE2 START
templates
# -- MISCFUNCTIONALITY TEMPLATE2 STOP
# -- MISCFUNCTIONALITY TEMPLATE3 START
j = Job(templates[0], name='JobFromTemplate')
j.submit()
# -- MISCFUNCTIONALITY TEMPLATE3 STOP
# -- MISCFUNCTIONALITY JOBTREE START
# show the current job tree (empty to start with)
jobtree
# make some dirs and subdirs
jobtree.mkdir('test_old')
jobtree.mkdir('test')
jobtree.mkdir('prod')
jobtree.mkdir('/test/jan')
jobtree.mkdir('/prod/full')
# have a look at the tree
jobtree.printtree()
# remove a dir
jobtree.rm('test_old')
# create some jobs and add them
jobtree.cd('/test/jan')
jobtree.add(Job())
jobtree.cd('/prod/full')
jobtree.add(Job())
jobtree.add(Job())
# look at the tree again
jobtree.printtree()
# submit the some jobs
jobtree.getjobs().submit()
def test_Savannah31691(self):
from Ganga.GPI import config, Job, jobs
config['Configuration']['autoGenerateJobWorkspace'] = True
import Ganga.Runtime.Workspace_runtime
localDir = Ganga.Runtime.Workspace_runtime.getLocalRoot()
# Create 5 default jobs, then list content of local workspace
for i in range(5):
Job()
dir_list = os.listdir(localDir)
for i in range(5):
assert str(i) in dir_list
# Delete job 0, then try again to list content of local workspace
jobs(0).remove()
dir_list = os.listdir(localDir)
for i in range(1, 5):
assert str(i) in dir_list
assert '0' not in dir_list
def test_Savannah14799(self):
from Ganga.GPI import Job, jobtree, jobs
from Ganga.GPIDev.Base.Proxy import stripProxy
j = Job()
jobtree.add(j)
self.assertNotEqual(
stripProxy(j)._getRegistry(),
stripProxy(jobtree)._getRegistry())
self.assertTrue(str(j.id) in jobtree.listjobs())
jt2 = jobtree.copy()
self.assertTrue(str(j.id) in jt2.listjobs())
jobs(j.id).remove()
jt2.cleanlinks()
self.assertFalse(str(j.id) in jobtree.listjobs())
print jt2.listjobs()
print jt2
self.assertFalse(str(j.id) in jt2.listjobs())
jt3 = jobtree.copy()
l1 = jobtree.listjobs()
l3 = jt3.listjobs()
l1.sort()
l3.sort()
self.assertEqual(l1, l3)
def test_a_JobConstruction(self):
""" First construct the Job object (singular)"""
from Ganga.Utility.Config import getConfig
self.assertFalse(getConfig('TestingFramework')['AutoCleanup'])
from Ganga.GPI import Job, jobs, ArgSplitter
j = Job()
self.assertEqual(len(jobs),
1) # Don't really gain anything from assertEqual...
j.splitter = ArgSplitter(args=[[i] for i in range(global_subjob_num)])
j.submit()
self.assertEqual(len(j.subjobs), global_subjob_num)
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(j, 60)
def test_directory(self):
from Ganga.GPI import jobtree, Job
jobtree.cd()
assert (jobtree.pwd() == '/')
jobtree.mkdir('testdir')
# it is OK to mkdir an existing directory
jobtree.mkdir('testdir')
assert ('testdir' in jobtree.ls()['folders'])
jobtree.cd('testdir')
assert (jobtree.pwd() == '/testdir')
j = Job()
jobtree.add(j)
assert (str(j.id) in jobtree.ls()['jobs'])
assert (str(j.id) in jobtree.ls('/testdir')['jobs'])
jobtree.cd('..')
assert (jobtree.pwd() == '/')
assert (str(j.id) in jobtree.ls('/testdir')['jobs'])
assert (str(j.id) not in jobtree.ls()['jobs'])
jobtree.rm('/*')
def test_Savannah76973(self):
from Ganga.GPI import Job, JobTemplate, JobError
j = Job()
t = JobTemplate(j)
self.assertEqual(t.status, 'template')
self.assertRaises(JobError, t.submit)
