def test_update(self): from Ganga import GPI t = GPI.LHCbTask() tr = GPI.LHCbTransform(application=DaVinci(), backend=Dirac()) t.appendTransform(tr) try: bkQueryList = [GPI.BKTestQuery(stripping20up)] tr.updateQuery() assert false, 'Should have thrown exception if updated with no query' except: tr.addQuery(GPI.BKTestQuery(stripping20down)) # Check some new data added assert len(tr.inputdata), 'No data added after call to update' try: # Shouldn't allow a second update before processed the data in # toProcess_dataset tr.updateQuery() assert false, 'Should have thrown an error if updated with files already to process' except: # run so can update again with a removed dataset recall that jobs with the # old dataset only created when run called. t.run() assert len(tr.getJobs()), "No Jobs created upon run()" job = GPI.jobs(int(tr.getJobs()[0].fqid.split('.')[0])) sleep_until_state(job, 300, 'submitted') del tr._impl.query.dataset.files[0] tr.update(True) # Check the dead dataset is picked up assert len(tr._impl.removed_data.files ), "Didn\'t Pick up loss of a dataset" job.remove()
def Savannah19123(self): from Ganga.GPI import Job, Local from GangaTest.Framework.utils import sleep_until_state # check if stdout and stderr exists or not, flag indicates if files are required to exist or not def check(exists_flag): for fn in ['stdout','stderr']: fn = os.path.join(j.outputdir,fn) file_exists = os.path.exists(fn) if exists_flag: self.assertTrue(file_exists, 'file %s should exist but it does not' % fn) else: self.assertFalse(file_exists, 'file %s should not exist but it does' % fn) j = Job() j.application.exe = 'bash' j.application.args = ['-c', 'for i in `seq 1 30`; do echo $i; sleep 1; done'] j.backend = Local() j.submit() check(False) sleep_until_state(j, 5, 'running') j.kill() check(True)
def testMergeRemoval(self): # see Savannah 33710 j = Job() jobID = j.id # job will run for at least 20 seconds j.application = Executable( exe='sh', args=['-c', 'sleep 20; echo foo > out.txt']) j.backend = Local() j.outputfiles = [LocalFile('out.txt')] j.splitter = CopySplitter() j.postprocessors = MergerTester(files=['out.txt']) j.postprocessors[0].ignorefailed = True j.postprocessors[0].alwaysfail = True j.postprocessors[0].wait = 10 j.submit() sleep_until_state(j, state='running') j.remove() try: jobs(jobID) assert False, 'Job should not be found' except KeyError: pass
def test_update(self): from Ganga import GPI t = GPI.LHCbTask() tr = GPI.LHCbTransform(application=DaVinci(), backend=Dirac()) t.appendTransform(tr) try: bkQueryList = [GPI.BKTestQuery(stripping20up)] tr.updateQuery() assert false, 'Should have thrown exception if updated with no query' except: tr.addQuery(GPI.BKTestQuery(stripping20down)) # Check some new data added assert len(tr.inputdata), 'No data added after call to update' try: # Shouldn't allow a second update before processed the data in # toProcess_dataset tr.updateQuery() assert false, 'Should have thrown an error if updated with files already to process' except: # run so can update again with a removed dataset recall that jobs with the # old dataset only created when run called. t.run() assert len(tr.getJobs()), "No Jobs created upon run()" job = GPI.jobs(int(tr.getJobs()[0].fqid.split('.')[0])) sleep_until_state(job, 300, 'submitted') del tr._impl.query.dataset.files[0] tr.update(True) # Check the dead dataset is picked up assert len( tr._impl.removed_data.files), "Didn\'t Pick up loss of a dataset" job.remove()
def testProperSubmit(self): config.ROOT.arch = "x86_64-slc6-gcc48-opt" j = Job(application=Root(script=script_file), backend=Dirac()) j.submit() sleep_until_state(j, state="submitted") assert j.status == "submitted", "Job should submit" j.kill()
def testProperSubmit(self): config.ROOT.arch = 'x86_64-slc6-gcc48-opt' j = Job(application=Root(script=script_file), backend=Dirac()) j.submit() sleep_until_state(j, state='submitted') assert j.status == 'submitted', 'Job should submit' j.kill()
def testMergeWhenSubjobsHaveFailed(self): j = self.jobslice[0] j.splitter = CopySplitter() j.splitter.number = 5 j.splitter.function_hook = 'makeFirstSubJobFailExecutable' tm = TextMerger() tm.files = ['out.txt'] tm.ignorefailed = True j.postprocessors = tm j.submit() if not sleep_until_state(j, state='failed'): assert False, 'Test timed out' assert j.status == 'failed', 'Job should be failed' output = os.path.join(j.outputdir, 'out.txt') assert os.path.exists(output) for sj in j.subjobs[1:]: out_txt = os.path.join(sj.outputdir, 'out.txt') assert file_contains( output, out_txt), 'File must contain the output of each individual job' out_txt = os.path.join(j.subjobs[0].outputdir, 'out.txt') assert not file_contains( output, out_txt), 'The failed subjob must have been skipped'
def test_Dirac_peek(self): j = Job(backend=Dirac()) j.submit() sleep_until_state(j, state='running') print j.status stdout = sys.stdout tmpdir = tempfile.mktemp() f = tempfile.NamedTemporaryFile() sys.stdout = f j.backend._impl.peek('stdout') sys.stdout = stdout j.kill() f.flush() fstdout = open(f.name, 'r') s = fstdout.read() assert s, 'Something should have been written to std output' f.close()
def testKilling(self): """ Create some subjobs and kill them """ from Ganga.GPI import Job, GenericSplitter, Local from GangaTest.Framework.utils import sleep_until_state j = Job() j.application.exe = "sleep" j.splitter = GenericSplitter() j.splitter.attribute = 'application.args' j.splitter.values = [['400'] for _ in range(0, 5)] j.backend = Local() j.submit() sleep_until_state(j, None, 'running') assert j.status == 'running' j.subjobs(0).kill() assert j.subjobs(0).status == 'killed' assert j.subjobs(1).status != 'killed' j.kill() assert j.status == 'killed' assert all(sj.status == 'killed' for sj in j.subjobs)
def test_Savannah44116(self): from Ganga.GPI import Job, TestApplication, TestSubmitter from GangaTest.Framework.utils import sleep_until_state j = Job() j.application = TestApplication() j.application.postprocess_mark_as_failed = True j.backend = TestSubmitter() j.backend.time = 1 j.submit() self.assertTrue(sleep_until_state(j, 10, 'failed'), 'Job is not marked as failed despite app.postprocess() hook')
def test_Savannah47814(self): from Ganga.GPI import Job, Executable from GangaTest.Framework.utils import sleep_until_state, file_contains j = Job() j.application = Executable(exe='ThisScriptDoesNotExist') j.submit() failed = sleep_until_state(j, 60, state='failed', break_states=['new', 'killed', 'completed', 'unknown', 'removed']) self.assertTrue(failed, 'Job with illegal script should fail. Instead it went into the state %s' % j.status) import os.path f = os.path.join(j.outputdir, '__jobstatus__') self.assertTrue(file_contains(f, 'No such file or directory'), '__jobstatus__ file should contain error')
def Savannah19123(self): from Ganga.GPI import Job, Local from GangaTest.Framework.utils import sleep_until_state # check if stdout and stderr exists or not, flag indicates if files are required to exist or not def check(exists_flag): for fn in ['stdout', 'stderr']: fn = os.path.join(j.outputdir, fn) file_exists = os.path.exists(fn) if exists_flag: self.assertTrue( file_exists, 'file %s should exist but it does not' % fn) else: self.assertFalse( file_exists, 'file %s should not exist but it does' % fn) j = Job() j.application.exe = 'bash' j.application.args = [ '-c', 'for i in `seq 1 30`; do echo $i; sleep 1; done' ] j.backend = Local() j.submit() check(False) sleep_until_state(j, 5, 'running') j.kill() check(True)
def testMergeRemoval(self): from Ganga.GPI import Job, Executable, Local, LocalFile, jobs # see Savannah 33710 j = Job() jobID = j.id # job will run for at least 20 seconds j.application = Executable( exe='sh', args=['-c', 'sleep 20; echo foo > out.txt']) j.backend = Local() j.outputfiles = [LocalFile('out.txt')] j.splitter = CopySplitter() j.postprocessors = MergerTester(files=['out.txt']) j.postprocessors[0].ignorefailed = True j.postprocessors[0].alwaysfail = True j.postprocessors[0].wait = 10 j.submit() sleep_until_state(j, state='running') j.remove() with pytest.raises(KeyError): jobs(jobID)
def test_Savannah44116(self): from Ganga.GPI import Job, TestApplication, TestSubmitter from GangaTest.Framework.utils import sleep_until_state j = Job() j.application = TestApplication() j.application.postprocess_mark_as_failed = True j.backend = TestSubmitter() j.backend.time = 1 j.submit() self.assertTrue( sleep_until_state(j, 10, 'failed'), 'Job is not marked as failed despite app.postprocess() hook')
def test_Savannah19123(self): from Ganga.GPI import Job, Local from GangaTest.Framework.utils import sleep_until_state # check if stdout and stderr exists or not, flag indicates if files are required to exist or not def check(exists_flag): for fn in ['stdout', 'stderr']: fn = os.path.join(j.outputdir, fn) file_exists = os.path.exists(fn) if exists_flag: self.assertTrue( file_exists, 'file %s should exist but it does not' % fn) else: self.assertFalse( file_exists, 'file %s should not exist but it does' % fn) j = Job() j.application.exe = 'bash' j.application.args = [ '-c', 'for i in `seq 1 30`; do echo $i; sleep 1; done' ] j.backend = Local() j.submit() check(False) if not sleep_until_state(j, 5, 'running'): # problem with the test - print out stdout/stderr and assert for fn in ['stdout', 'stderr']: fn = os.path.join(j.outputdir, fn) print " ---- Contents of " + fn if os.path.exists(fn): print open(fn).read() else: print "NO FILE AVAILABLE" self.assertEqual(j.status, 'running') j.kill() check(True)
def Savannah28511(self): from Ganga.GPI import Job, TestSubmitter from GangaTest.Framework.utils import sleep_until_completed, sleep_until_state j = Job() j.submit() self.assertTrue(sleep_until_completed(j, 20), 'Job is not completed') j.resubmit() self.assertTrue(sleep_until_completed(j, 30), 'Job is not completed after fail during resubmit') j._impl.updateStatus('failed') self.assertTrue(sleep_until_state(j, 20, 'failed'), 'Job is not failed') j.resubmit() self.assertTrue(sleep_until_completed(j, 20), 'Job is not completed after resubmit')
def test_Savannah19123(self): from Ganga.GPI import Job, Local from GangaTest.Framework.utils import sleep_until_state # check if stdout and stderr exists or not, flag indicates if files are required to exist or not def check(exists_flag): for fn in ['stdout','stderr']: fn = os.path.join(j.outputdir,fn) file_exists = os.path.exists(fn) if exists_flag: self.assertTrue(file_exists, 'file %s should exist but it does not' % fn) else: self.assertFalse(file_exists, 'file %s should not exist but it does' % fn) j = Job() j.application.exe = 'bash' j.application.args = ['-c', 'for i in `seq 1 30`; do echo $i; sleep 1; done'] j.backend = Local() j.submit() check(False) if not sleep_until_state(j, 5, 'running'): # problem with the test - print out stdout/stderr and assert for fn in ['stdout','stderr']: fn = os.path.join(j.outputdir,fn) print " ---- Contents of " + fn if os.path.exists(fn): print open(fn).read() else: print "NO FILE AVAILABLE" self.assertEqual(j.status, 'running') j.kill() check(True)
def Savannah47814(self): from Ganga.GPI import Job, Executable from GangaTest.Framework.utils import sleep_until_state, file_contains j = Job() j.application = Executable(exe='ThisScriptDoesNotExist') j.submit() failed = sleep_until_state( j, 60, state='failed', break_states=['new', 'killed', 'completed', 'unknown', 'removed']) self.assertTrue( failed, 'Job with illegal script should fail. Instead it went into the state %s' % j.status) import os.path f = os.path.join(j.outputdir, '__jobstatus__') self.assertTrue(file_contains(f, 'No such file or directory'), '__jobstatus__ file should contain error')