def test_Savannah18729(self):
from Ganga.GPI import Root, Job, Local
import os
from GangaTest.Framework.utils import sleep_until_completed
import tempfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
## Is this a test of files with a leading ' ' in the name? - rcurrie
#self.fname = os.path.join(tmpdir, ' test.C')
self.fname = os.path.join(tmpdir, 'test.C')
with open(self.fname, 'w') as f:
f.write('''
void test(const char* text, int i)
{
cout << gSystem->GetDynamicPath() << endl;
gSystem->Load("libTree");
cout << text << " " << i << endl;
}
''')
app = Root()
app.script = self.fname
app.args = ['abc', 1]
j = Job(backend=Local(), application=app)
j.submit()
self.assertTrue(sleep_until_completed(j,120), 'Timeout on registering Interactive job as completed')
self.assertEqual(j.status, 'completed')
def test_e_MultipleFiles(self):
"""Test that the wildcards work"""
from Ganga.GPI import LocalFile, MassStorageFile, Job, ArgSplitter
_ext = '.root'
_ext2 = '.txt'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
file_3 = generate_unique_temp_file(_ext2)
TestMassStorageWN._managed_files.append(file_1)
TestMassStorageWN._managed_files.append(file_2)
TestMassStorageWN._managed_files.append(file_3)
j = Job()
j.inputfiles = [
LocalFile(file_1),
LocalFile(file_2),
LocalFile(file_3)
]
j.splitter = ArgSplitter(
args=[[_] for _ in range(0, TestMassStorageWN.sj_len)])
j.outputfiles = [
MassStorageFile(namePattern='*' + _ext,
outputfilenameformat='{jid}/{sjid}/{fname}'),
MassStorageFile(namePattern='*' + _ext2)
]
j.submit()
def _check(self, template):
logger.info("------------------------------------------------")
logger.info("- Now checking template: '%s'" % template.name)
logger.info("------------------------------------------------")
j = Job(template)
j.submit()
self.assertTrue(sleep_until_completed(j))
def test_j_Queues(self):
from Ganga.GPI import queues, Job, GenericSplitter
queues
# -- QUEUES EXAMPLE START
for i in range(1, 10):
j = Job()
queues.add(j.submit)
# -- QUEUES EXAMPLE STOP
# -- QUEUES FUNCTION START
def f(x):
print x
queues.add(f, args=(123,))
# -- QUEUES FUNCTION STOP
# -- QUEUES SPLIT START
j = Job()
j.splitter = GenericSplitter()
j.splitter.attribute = 'application.args'
j.splitter.values = [i for i in range(0, 10)]
j.parallel_submit = True
j.submit()
def Savannah19123(self):
from Ganga.GPI import Job, Local
from GangaTest.Framework.utils import sleep_until_state
# check if stdout and stderr exists or not, flag indicates if files are required to exist or not
def check(exists_flag):
for fn in ['stdout','stderr']:
fn = os.path.join(j.outputdir,fn)
file_exists = os.path.exists(fn)
if exists_flag:
self.assertTrue(file_exists, 'file %s should exist but it does not' % fn)
else:
self.assertFalse(file_exists, 'file %s should not exist but it does' % fn)
j = Job()
j.application.exe = 'bash'
j.application.args = ['-c', 'for i in `seq 1 30`; do echo $i; sleep 1; done']
j.backend = Local()
j.submit()
check(False)
sleep_until_state(j, 5, 'running')
j.kill()
check(True)
def test_submit_kill_resubmit(gpi):
"""
Test that a simple submit-kill-resubmit-kill cycle works
"""
from Ganga.GPI import Job, LCG
j = Job()
j.backend = LCG()
with patch('Ganga.Lib.LCG.Grid.submit',
return_value='https://example.com:9000/42') as submit:
j.submit()
submit.assert_called_once()
assert j.backend.id == 'https://example.com:9000/42'
with patch('Ganga.Lib.LCG.Grid.cancel', return_value=True) as cancel:
j.kill()
cancel.assert_called_once()
assert j.status == 'killed'
with patch('Ganga.Lib.LCG.Grid.submit',
return_value='https://example.com:9000/43') as submit:
j.resubmit()
submit.assert_called_once()
assert j.backend.id == 'https://example.com:9000/43'
with patch('Ganga.Lib.LCG.Grid.cancel', return_value=True):
j.kill()
def _check(self, template):
logger.info("------------------------------------------------")
logger.info("- Now checking template: '%s'" % template.name)
logger.info("------------------------------------------------")
j = Job(template)
j.submit()
self.assertTrue(sleep_until_completed(j))
def test_d_RunningExecutables(self):
from Ganga.GPI import Job, File, Executable
# -- RUNNINGEXECUTABLES EXAMPLE START
# Already existing Exe
j = Job()
j.application = Executable()
j.application.exe = '/bin/ls'
j.application.args = ['-l', '-h']
j.submit()
# Wait for completion
j.peek("stdout")
# Send a script
open('my_script.sh', 'w').write("""#!/bin/bash
echo 'Current dir: ' `pwd`
echo 'Contents:'
ls -ltr
echo 'Args: ' $@
""")
import os
os.system('chmod +x my_script.sh')
j = Job()
j.application = Executable()
j.application.exe = File('my_script.sh')
j.submit()
# Wait for completion
j.peek("stdout")
def test_Savannah18729(self):
from Ganga.GPI import Root, Job, Local
import os
from GangaTest.Framework.utils import sleep_until_completed
import tempfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
## Is this a test of files with a leading ' ' in the name? - rcurrie
#self.fname = os.path.join(tmpdir, ' test.C')
self.fname = os.path.join(tmpdir, 'test.C')
with open(self.fname, 'w') as f:
f.write('''
void test(const char* text, int i)
{
cout << gSystem->GetDynamicPath() << endl;
gSystem->Load("libTree");
cout << text << " " << i << endl;
}
''')
app = Root()
app.script = self.fname
app.args = ['abc', 1]
j = Job(backend=Local(), application=app)
j.submit()
self.assertTrue(sleep_until_completed(j, 120),
'Timeout on registering Interactive job as completed')
self.assertEqual(j.status, 'completed')
def test_a_InstallAndBasicUsage(self):
from Ganga.GPI import Job, jobs
# -- INSTALLANDBASICUSAGE HELP START
help(Job)
# -- INSTALLANDBASICUSAGE HELP STOP
# -- INSTALLANDBASICUSAGE SUBMIT START
j = Job()
j.submit()
# -- INSTALLANDBASICUSAGE SUBMIT STOP
# -- INSTALLANDBASICUSAGE JOBS START
jobs(0)
# -- INSTALLANDBASICUSAGE JOBS STOP
# -- INSTALLANDBASICUSAGE JOBSAPP START
jobs(0).application
# -- INSTALLANDBASICUSAGE JOBSAPP STOP
# -- INSTALLANDBASICUSAGE EXECFILE START
open('submit.py', 'w').write("""
j = Job()
j.submit()
""")
execfile('submit.py')
def test_job_complete(gpi):
from Ganga.GPI import Job, LCG
j = Job()
j.backend = LCG()
j.submit()
assert run_until_completed(j, timeout=1200, sleep_period=10), 'Timeout on job submission: job is still not finished'
def test_a_InstallAndBasicUsage(self):
from Ganga.GPI import Job, jobs
# -- INSTALLANDBASICUSAGE HELP START
help(Job)
# -- INSTALLANDBASICUSAGE HELP STOP
# -- INSTALLANDBASICUSAGE SUBMIT START
j = Job()
j.submit()
# -- INSTALLANDBASICUSAGE SUBMIT STOP
# -- INSTALLANDBASICUSAGE JOBS START
jobs(0)
# -- INSTALLANDBASICUSAGE JOBS STOP
# -- INSTALLANDBASICUSAGE JOBSAPP START
jobs(0).application
# -- INSTALLANDBASICUSAGE JOBSAPP STOP
# -- INSTALLANDBASICUSAGE EXECFILE START
open('submit.py', 'w').write("""
j = Job()
j.submit()
""")
execfile('submit.py')
def testStartUp():
""" Lets test the startup of Ganga mimicking first launch """
# Process options given at command line and in configuration file(s)
# Perform environment setup and bootstrap
from Ganga.Runtime import setupGanga
argv = ['ganga', '--no-mon', '-o[Configuration]gangadir=%s' % this_dir, '-o[Configuration]RUNTIME_PATH=GangaTest']
setupGanga(argv=argv, interactive=False)
for this_file in ['.gangarc', '.ganga.log']:
assert os.path.isfile(os.path.join(homeDir, this_file))
# No way known to mimic IPython starting up in a simple way
#assert os.path.isdir(os.path.join(homeDir, '.ipython-ganga'))
for this_folder in ['repository',]:
assert os.path.isdir(os.path.join(this_dir, this_folder))
from Ganga.GPI import Job
j=Job()
j.submit()
for this_folder in ['shared', 'workspace']:
assert os.path.isdir(os.path.join(this_dir, this_folder))
def test_j_Queues(self):
from Ganga.GPI import queues, Job, GenericSplitter
queues
# -- QUEUES EXAMPLE START
for i in range(1, 10):
j = Job()
queues.add(j.submit)
# -- QUEUES EXAMPLE STOP
# -- QUEUES FUNCTION START
def f(x):
print x
queues.add(f, args=(123, ))
# -- QUEUES FUNCTION STOP
# -- QUEUES SPLIT START
j = Job()
j.splitter = GenericSplitter()
j.splitter.attribute = 'application.args'
j.splitter.values = [i for i in range(0, 10)]
j.parallel_submit = True
j.submit()
def test_job_kill(gpi):
from Ganga.GPI import Job, LCG
j = Job()
j.backend = LCG()
j.submit()
j.kill()
def test_submit_kill_resubmit(gpi):
"""
Test that a simple submit-kill-resubmit-kill cycle works
"""
from Ganga.GPI import Job, LCG
j = Job()
j.backend = LCG()
with patch('Ganga.Lib.LCG.Grid.submit', return_value='https://example.com:9000/42') as submit:
j.submit()
submit.assert_called_once()
assert j.backend.id == 'https://example.com:9000/42'
with patch('Ganga.Lib.LCG.Grid.cancel', return_value=True) as cancel:
j.kill()
cancel.assert_called_once()
assert j.status == 'killed'
with patch('Ganga.Lib.LCG.Grid.submit', return_value='https://example.com:9000/43') as submit:
j.resubmit()
submit.assert_called_once()
assert j.backend.id == 'https://example.com:9000/43'
with patch('Ganga.Lib.LCG.Grid.cancel', return_value=True):
j.kill()
def test_Savannah8009(self):
from Ganga.GPI import Executable, Job, jobs, templates
from GangaTest.Framework.utils import sleep_until_completed
j = Job()
j.submit()
self.assertEqual(len(jobs), 1)
self.assertEqual(len(templates), 0)
if not sleep_until_completed(j, timeout=120):
assert(not "Timeout on job submission: job is still not finished")
t = j.copy()
# make sure that copy creates a new job (and not the template)
self.assertEqual(len(jobs), 2)
self.assertEqual(len(templates), 0)
# make sure that output parameters are not carried forward
self.assertNotEqual(j.backend.id, t.backend.id)
self.assertNotEqual(j.backend.exitcode, t.backend.exitcode)
# make sure that input parameters are carried forward
self.assertEqual(j.application.exe, t.application.exe)
def test_d_RunningExecutables(self):
from Ganga.GPI import Job, File, Executable
# -- RUNNINGEXECUTABLES EXAMPLE START
# Already existing Exe
j = Job()
j.application = Executable()
j.application.exe = '/bin/ls'
j.application.args = ['-l', '-h']
j.submit()
# Wait for completion
j.peek("stdout")
# Send a script
open('my_script.sh', 'w').write("""#!/bin/bash
echo 'Current dir: ' `pwd`
echo 'Contents:'
ls -ltr
echo 'Args: ' $@
""")
import os
os.system('chmod +x my_script.sh')
j = Job()
j.application = Executable()
j.application.exe = File('my_script.sh')
j.submit()
# Wait for completion
j.peek("stdout")
def test_job_kill(gpi):
from Ganga.GPI import Job, CREAM
vo = getConfig('LCG')['VirtualOrganisation']
call = subprocess.Popen(['lcg-infosites', 'ce', 'cream', '--vo', vo],
stdout=subprocess.PIPE)
stdout, stderr = call.communicate()
# Based on output of:
#
# # CPU Free Total Jobs Running Waiting ComputingElement
# ----------------------------------------------------------------
# 19440 2089 17760 17351 409 arc-ce01.gridpp.rl.ac.uk:2811/nordugrid-Condor-grid3000M
# 3240 0 1594 1250 344 carceri.hec.lancs.ac.uk:8443/cream-sge-grid
# 1176 30 1007 587 420 ce01.tier2.hep.manchester.ac.uk:8443/cream-pbs-long
#
# Select the CREAM CEs (URL path starts with '/cream') and how many free slots they have
ces = re.findall(
r'^\s*\d+\s*(?P<free>\d+)\s*\d+\s*\d+\s*\d+\s*(?P<ce>[^:/\s]+uk:\d+/cream.*)$',
stdout, re.MULTILINE)
# Grab the one with the most empty slots
ce = sorted(ces)[-1][1]
j = Job()
j.backend = CREAM(CE=ce)
j.submit()
j.kill()
def test_job_kill(gpi):
from Ganga.GPI import Job, LCG
j = Job()
j.backend = LCG()
j.submit()
j.kill()
def testStartUp():
""" Lets test the startup of Ganga mimicking first launch """
# Process options given at command line and in configuration file(s)
# Perform environment setup and bootstrap
from Ganga.Runtime import setupGanga
argv = [
'ganga', '--no-mon',
'-o[Configuration]gangadir=%s' % this_dir,
'-o[Configuration]RUNTIME_PATH=GangaTest'
]
setupGanga(argv=argv, interactive=False)
for this_file in ['.gangarc', '.ganga.log']:
assert os.path.isfile(os.path.join(homeDir, this_file))
# No way known to mimic IPython starting up in a simple way
#assert os.path.isdir(os.path.join(homeDir, '.ipython-ganga'))
for this_folder in [
'repository',
]:
assert os.path.isdir(os.path.join(this_dir, this_folder))
from Ganga.GPI import Job
j = Job()
j.submit()
for this_folder in ['shared', 'workspace']:
assert os.path.isdir(os.path.join(this_dir, this_folder))
def test_Savannah8009(self):
from Ganga.GPI import Executable, Job, jobs, templates
from GangaTest.Framework.utils import sleep_until_completed
j = Job()
j.submit()
self.assertEqual(len(jobs), 1)
self.assertEqual(len(templates), 0)
if not sleep_until_completed(j, timeout=120):
assert not "Timeout on job submission: job is still not finished"
t = j.copy()
# make sure that copy creates a new job (and not the template)
self.assertEqual(len(jobs), 2)
self.assertEqual(len(templates), 0)
# make sure that output parameters are not carried forward
self.assertNotEqual(j.backend.id, t.backend.id)
self.assertNotEqual(j.backend.exitcode, t.backend.exitcode)
# make sure that input parameters are carried forward
self.assertEqual(j.application.exe, t.application.exe)
def test_a_JobConstruction(self):
from Ganga.GPI import Job, jobs
j = Job()
assert len(jobs) == 1
j.submit()
assert j.status != 'new'
def test_d_anotherNewJob(self):
from Ganga.GPI import Job, jobs
j = Job()
j.submit()
self.assertNotEqual(j.status, 'new')
def test_d_anotherNewJob(self):
from Ganga.GPI import Job, jobs
j=Job()
j.submit()
self.assertNotEqual(j.status, 'new')
def test_Savannah13404(self):
from Ganga.GPI import Job
j = Job()
j.submit() # Needed in order to create the workspace
self.assertTrue(os.path.exists(j.inputdir + '/..'))
j.remove()
# The job directory should be deleted
self.assertFalse(os.path.exists(os.path.abspath(j.inputdir + '/..')))
def test_Savannah13404(self):
from Ganga.GPI import Job
j = Job()
j.submit() # Needed in order to create the workspace
self.assertTrue(os.path.exists(j.inputdir + '/..'))
j.remove()
# The job directory should be deleted
self.assertFalse(os.path.exists(os.path.abspath(j.inputdir + '/..')))
def testSavannah10064(self):
from Ganga.GPI import Job, TestSubmitter, TestApplication
j = Job(backend=TestSubmitter(),application=TestApplication())
j.submit()
import os.path
assert(os.path.exists(j.inputdir))
templates.remove()
assert(os.path.exists(j.inputdir))
def test_a_JobConstruction(self):
from Ganga.GPI import Job, jobs
j = Job()
assert len(jobs) == 1
j.submit()
assert j.status != 'new'
def test_Savannah10064(self):
from Ganga.GPI import Job, templates
j = Job()
j.submit()
import os.path
self.assertTrue(os.path.exists(j.inputdir))
templates.remove()
self.assertTrue(os.path.exists(j.inputdir))
def test_Savannah10064(self):
from Ganga.GPI import Job, templates
j = Job()
j.submit()
import os.path
self.assertTrue(os.path.exists(j.inputdir))
templates.remove()
self.assertTrue(os.path.exists(j.inputdir))
def test_e_testInMemory(self):
"""
Test the resubmit on a job in memory vs a job which has been loaded from disk
"""
from Ganga.GPI import Job, Local
j=Job()
j.splitter = self._getSplitter()
j.backend = Local()
j.submit()
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(j)
# Job has ben created, split, run and now exists in Memory (NOT SJXML)
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'resubmitOnlyFailedSubjobs', 'True')
j.resubmit()
sleep_until_completed(j)
j.subjobs(0).resubmit()
# We should get here if calling resubmit doesn't stall
j.subjobs(0).force_status('failed')
j.resubmit()
sleep_until_completed(j)
assert j.subjobs(0).status == 'completed'
# Test resubmit from the master job worked
j.subjobs(0).force_status('failed')
j.subjobs(0).resubmit()
sleep_until_completed(j)
assert j.subjobs(0).status == 'completed'
# Test that the resubmit from the subjob worked
setConfigOption('Configuration', 'resubmitOnlyFailedSubjobs', 'False')
j.resubmit()
sleep_until_completed(j)
j.subjobs(0).force_status('failed')
j.resubmit()
sleep_until_completed(j)
def test_e_testInMemory(self):
"""
Test the resubmit on a job in memory vs a job which has been loaded from disk
"""
from Ganga.GPI import Job, Local
j = Job()
j.splitter = self._getSplitter()
j.backend = Local()
j.submit()
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(j)
# Job has ben created, split, run and now exists in Memory (NOT SJXML)
from Ganga.Utility.Config import setConfigOption
setConfigOption('Configuration', 'resubmitOnlyFailedSubjobs', 'True')
j.resubmit()
sleep_until_completed(j)
j.subjobs(0).resubmit()
# We should get here if calling resubmit doesn't stall
j.subjobs(0).force_status('failed', force=True)
j.resubmit()
sleep_until_completed(j)
assert j.subjobs(0).status == 'completed'
# Test resubmit from the master job worked
j.subjobs(0).force_status('failed')
j.subjobs(0).resubmit()
sleep_until_completed(j)
assert j.subjobs(0).status == 'completed'
# Test that the resubmit from the subjob worked
setConfigOption('Configuration', 'resubmitOnlyFailedSubjobs', 'False')
j.resubmit()
sleep_until_completed(j)
j.subjobs(0).force_status('failed')
j.resubmit()
sleep_until_completed(j)
def test_job_submit_and_monitor(gpi):
from Ganga.GPI import Job, LCG
j = Job()
j.backend = LCG()
j.submit()
assert j.status != 'new'
stripProxy(LCG).master_updateMonitoringInformation([stripProxy(j)])
def test_job_submit_and_monitor(gpi):
from Ganga.GPI import Job, LCG
j = Job()
j.backend = LCG()
j.submit()
assert j.status != 'new'
stripProxy(LCG).master_updateMonitoringInformation([stripProxy(j)])
def test_c_JobManipulation(self):
from Ganga.GPI import runMonitoring, Job, jobs, export, load
runMonitoring()
# -- JOBMANIPULATION JOBCOPY START
j = Job(name = 'original')
j2 = j.copy()
j2.name = 'copy'
j.submit()
j3 = Job(j, name = 'copy2')
jobs
# -- JOBMANIPULATION JOBCOPY STOP
# -- JOBMANIPULATION REPOACCESS START
jobs(2)
# -- JOBMANIPULATION REPOACCESS STOP
# -- JOBMANIPULATION JOBSLICING START
jobs[2]
jobs[2:]
jobs['copy2']
# -- JOBMANIPULATION JOBSLICING STOP
jobs(0).kill()
# -- JOBMANIPULATION RESUBMIT START
jobs(0).resubmit()
# -- JOBMANIPULATION RESUBMIT STOP
# -- JOBMANIPULATION FORCESTATUS START
jobs(1).force_status('failed')
# -- JOBMANIPULATION FORCESTATUS STOP
# -- JOBMANIPULATION JOBREMOVE START
jobs(2).remove()
# -- JOBMANIPULATION JOBREMOVE STOP
# -- JOBMANIPULATION JOBSELECT START
# can select on ids, name, status, backend, application
jobs.select(status='new')
jobs.select(backend='Local')
jobs.select(ids=[1,3])
# can restrict on min/max id
jobs.select(1,3, application='Executable')
# -- JOBMANIPULATION JOBSELECT STOP
# -- JOBMANIPULATION JOBSELECTOP START
jobs.select(status='new').submit()
# -- JOBMANIPULATION JOBSELECTOP STOP
# -- JOBMANIPULATION EXPORTJOB START
export(jobs(0), 'my_job.txt')
jlist = load('my_job.txt')
jlist[0].submit()
def test_c_JobManipulation(self):
from Ganga.GPI import runMonitoring, Job, jobs, export, load
runMonitoring()
# -- JOBMANIPULATION JOBCOPY START
j = Job(name='original')
j2 = j.copy()
j2.name = 'copy'
j.submit()
j3 = Job(j, name='copy2')
jobs
# -- JOBMANIPULATION JOBCOPY STOP
# -- JOBMANIPULATION REPOACCESS START
jobs(2)
# -- JOBMANIPULATION REPOACCESS STOP
# -- JOBMANIPULATION JOBSLICING START
jobs[2]
jobs[2:]
jobs['copy2']
# -- JOBMANIPULATION JOBSLICING STOP
jobs(0).kill()
# -- JOBMANIPULATION RESUBMIT START
jobs(0).resubmit()
# -- JOBMANIPULATION RESUBMIT STOP
# -- JOBMANIPULATION FORCESTATUS START
jobs(1).force_status('failed')
# -- JOBMANIPULATION FORCESTATUS STOP
# -- JOBMANIPULATION JOBREMOVE START
jobs(2).remove()
# -- JOBMANIPULATION JOBREMOVE STOP
# -- JOBMANIPULATION JOBSELECT START
# can select on ids, name, status, backend, application
jobs.select(status='new')
jobs.select(backend='Local')
jobs.select(ids=[1, 3])
# can restrict on min/max id
jobs.select(1, 3, application='Executable')
# -- JOBMANIPULATION JOBSELECT STOP
# -- JOBMANIPULATION JOBSELECTOP START
jobs.select(status='new').submit()
# -- JOBMANIPULATION JOBSELECTOP STOP
# -- JOBMANIPULATION EXPORTJOB START
export(jobs(0), 'my_job.txt')
jlist = load('my_job.txt')
jlist[0].submit()
def testSubmitLocal(self):
from Ganga.GPI import DaVinci, Job, TestSubmitter, JobError
from GangaLHCb.testlib import addLocalTestSubmitter
ap = DaVinci()
j = Job(application=ap, backend=TestSubmitter())
# Test that submission fails before adding runtime handler
with pytest.raises(JobError):
j.submit()
def test_a_JobConstruction(self):
from Ganga.GPI import Job, jobs, disableMonitoring
j = Job()
self.assertEqual(len(jobs), 1)
j.submit()
self.assertNotEqual(j.status, 'new')
def test_a_JobConstruction(self):
from Ganga.GPI import Job, jobs, disableMonitoring
j=Job()
self.assertEqual(len(jobs), 1)
j.submit()
self.assertNotEqual(j.status, 'new')
def test_e_UsingDifferentBackends(self):
from Ganga.GPI import Job, plugins, Local
# -- USINGDIFFERENTBACKENDS PLUGINS START
plugins("backends")
# -- USINGDIFFERENTBACKENDS PLUGINS STOP
# -- USINGDIFFERENTBACKENDS LOCAL START
j = Job()
j.backend = Local()
j.submit()
def test_b_EnableMonitoring(self):
from Ganga.GPI import enableMonitoring, Job, jobs
enableMonitoring()
j=Job()
j.submit()
dummySleep(j)
self.assertNotEqual(jobs(0).status, 'submitted')
def testArgSplitter(self):
from Ganga.GPI import Job, ArgSplitter
from GangaTest.Framework.utils import sleep_until_completed
j = Job()
j.splitter = ArgSplitter(args=[['1'], ['2'], ['3']])
j.submit()
self.assertTrue(sleep_until_completed(j, 60), 'Timeout on completing job')
self.assertEqual(len(j.subjobs), 3)
def test_b_EnableMonitoring(self):
from Ganga.GPI import enableMonitoring, Job, jobs
enableMonitoring()
j = Job()
j.submit()
dummySleep(j)
self.assertNotEqual(jobs(0).status, 'submitted')
def test_e_UsingDifferentBackends(self):
from Ganga.GPI import Job, plugins, Local
# -- USINGDIFFERENTBACKENDS PLUGINS START
plugins("backends")
# -- USINGDIFFERENTBACKENDS PLUGINS STOP
# -- USINGDIFFERENTBACKENDS LOCAL START
j = Job()
j.backend = Local()
j.submit()
def test_a_jobSubmit(self):
"""here for testing a submit"""
from Ganga.GPI import Job, Executable, ArgSplitter, MassStorageFile
j=Job()
j.application=Executable(exe='touch')
j.splitter=ArgSplitter(args=[['abc.txt'], ['def.txt']])
j.outputfiles=[MassStorageFile(outputfilenameformat = '/test/{sjid}-{fname}', namePattern = '*.txt')]
j.submit()
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(j)
def test_Savannah13406(self):
#from Ganga.GPI import config
#config['Configuration']['autoGenerateJobWorkspace'] = True
import os
from Ganga.GPI import Job, jobs
jobs.remove()
j = Job()
j.submit() # Needed in order to create the workspace
self.assertTrue(os.path.exists(j.inputdir + '/../..'))
jobs.remove()
# Check is repository/Local or repository/Remote still exists
self.assertTrue(os.path.exists(os.path.abspath(j.inputdir + '/../..')))
def test_Savannah13406(self):
#from Ganga.GPI import config
#config['Configuration']['autoGenerateJobWorkspace'] = True
import os
from Ganga.GPI import Job, jobs
jobs.remove()
j = Job()
j.submit() # Needed in order to create the workspace
self.assertTrue(os.path.exists(j.inputdir + '/../..'))
jobs.remove()
# Check is repository/Local or repository/Remote still exists
self.assertTrue(os.path.exists(os.path.abspath(j.inputdir + '/../..')))
def test_unprepareTrue(gpi):
from Ganga.GPI import Job, Executable
j = Job(application=Executable(exe='/bin/echo', args=['hello']))
j.submit()
assert j.application.is_prepared is not None
j2 = j.copy()
assert j2.application.is_prepared is None
j3 = Job(j)
assert j3.application.is_prepared is None
def testSubjobsSubmit(self):
from Ganga.GPI import Job, Executable, TestSubmitter, ArgSplitter
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as gangaList
from Ganga.GPIDev.Base.Proxy import isType
j = Job(application=Executable(), backend=TestSubmitter(time=1))
j.splitter = ArgSplitter(args=[['A'], ['B'], ['C']])
j.submit()
assert run_until_completed(j), 'Job must complete'
assert len(j.subjobs) == 3, 'splitting must occur'
for jj in j.subjobs:
assert not isType(jj.master, gangaList)
def testMergeThatAlwaysFailsOverwrite(self):
from Ganga.GPI import Job, Executable, Local, LocalFile
j = Job()
j.application = Executable(exe='sh', args=['-c', 'echo foo > out.txt'])
j.backend = Local()
j.outputfiles = [LocalFile('out.txt')]
j.splitter = CopySplitter()
j.postprocessors = MergerTester(files=['out.txt'], overwrite=True)
j.submit()
assert run_until_state(j, 'failed', timeout=60)
assert os.path.exists(os.path.join(j.outputdir, 'out.txt.merge_summary')), 'Summary file should be created'
def testLargeJobSubmission(self):
"""
Create lots of subjobs and submit it
"""
from Ganga.GPI import Job, GenericSplitter, Local
j = Job()
j.application.exe = "sleep"
j.splitter = GenericSplitter()
j.splitter.attribute = 'application.args'
j.splitter.values = [['400'] for _ in range(0, 20)]
j.backend = Local()
j.submit()
assert len(j.subjobs) == 20
def testMergeThatAlwaysFailsOverwrite(self):
from Ganga.GPI import Job, Executable, Local, LocalFile
j = Job()
j.application = Executable(exe='sh', args=['-c', 'echo foo > out.txt'])
j.backend = Local()
j.outputfiles = [LocalFile('out.txt')]
j.splitter = CopySplitter()
j.postprocessors = MergerTester(files=['out.txt'], overwrite=True)
j.submit()
assert run_until_state(j, 'failed', timeout=60)
assert os.path.exists(os.path.join(j.outputdir, 'out.txt.merge_summary')), 'Summary file should be created'
def test_a_JobSubmission(self):
"""
Create lots of subjobs and submit it
"""
from Ganga.GPI import Job, Local
j = Job()
j.application.exe = "sleep"
j.splitter = self._getSplitter()
j.backend = Local()
j.submit()
# Test we can submit a job and we're going to check the sj are created
assert len(j.subjobs) == TestSJSubmit.n_subjobs
def test_Savannah44116(self):
from Ganga.GPI import Job, TestApplication, TestSubmitter
from GangaTest.Framework.utils import sleep_until_state
j = Job()
j.application = TestApplication()
j.application.postprocess_mark_as_failed = True
j.backend = TestSubmitter()
j.backend.time = 1
j.submit()
self.assertTrue(sleep_until_state(j, 10, 'failed'), 'Job is not marked as failed despite app.postprocess() hook')
def testLargeJobSubmission(self):
"""
Create lots of subjobs and submit it
"""
from Ganga.GPI import Job, GenericSplitter, Local
j = Job()
j.application.exe = "sleep"
j.splitter = GenericSplitter()
j.splitter.attribute = 'application.args'
j.splitter.values = [['400'] for _ in range(0, 20)]
j.backend = Local()
j.submit()
assert len(j.subjobs) == 20
def test_e_testXMLContent(self):
# Check content of XML is as expected
from Ganga.Core.GangaRepository.VStreamer import to_file, from_file
from Ganga.GPI import jobs, Job
from Ganga.GPIDev.Base.Proxy import stripProxy
from tempfile import NamedTemporaryFile
j = jobs(0)
assert path.isfile(getXMLFile(j))
with open(getXMLFile(j)) as handler:
tmpobj, errs = from_file(handler)
assert hasattr(tmpobj, 'name')
assert tmpobj.name == testStr
ignore_subs = [
'time', 'subjobs', 'info', 'application', 'backend', 'id'
]
with NamedTemporaryFile(delete=False) as new_temp_file:
temp_name = new_temp_file.name
to_file(stripProxy(j), new_temp_file, ignore_subs)
new_temp_file.flush()
with NamedTemporaryFile(delete=False) as new_temp_file2:
temp_name2 = new_temp_file2.name
j2 = Job()
j2.name = testStr
j2.submit()
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(j2)
to_file(stripProxy(j2), new_temp_file2, ignore_subs)
new_temp_file2.flush()
#import filecmp
#assert filecmp.cmp(handler.name, new_temp_file.name)
#assert not filecmp.cmp(new_temp_file.name, new_temp_file2.name)
#assert open(getXMLFile(j)).read() == open(temp_name).read()
assert open(temp_name).read() == open(temp_name2).read()
unlink(temp_name)
unlink(temp_name2)
