def test_d_RunningExecutables(self): from Ganga.GPI import Job, File, Executable # -- RUNNINGEXECUTABLES EXAMPLE START # Already existing Exe j = Job() j.application = Executable() j.application.exe = '/bin/ls' j.application.args = ['-l', '-h'] j.submit() # Wait for completion j.peek("stdout") # Send a script open('my_script.sh', 'w').write("""#!/bin/bash echo 'Current dir: ' `pwd` echo 'Contents:' ls -ltr echo 'Args: ' $@ """) import os os.system('chmod +x my_script.sh') j = Job() j.application = Executable() j.application.exe = File('my_script.sh') j.submit() # Wait for completion j.peek("stdout")
def test_ConfigDefaults(self): from Ganga.GPI import Executable from Ganga.Utility.Config import setConfigOption setConfigOption('defaults_Executable', 'exe', 'echo2') setConfigOption('defaults_Executable', 'args', ['Hello World2']) a = Executable() assert a.exe == 'echo2' assert a.args == ['Hello World2'] setConfigOption('defaults_Executable', 'exe', '/bin/echo') k = Executable() assert k.exe == '/bin/echo' assert k.args == ['Hello World2']
def setUp(self): super(TestStructure, self).setUp() from Ganga.GPI import Job, Executable, Local, File, LocalFile from GangaTest.Framework.utils import write_file self.jobslice = [] self.file_name = 'id_echo.sh' for _ in range(5): j = Job(application=Executable(), backend=Local()) scriptString = ''' #!/bin/sh echo "Output from job $1." > out.txt mkdir -p subdir echo "Output from job $2." > subdir/out.txt ''' # write string to tmpfile tmpdir = tempfile.mktemp() os.mkdir(tmpdir) fileName = os.path.join(tmpdir, self.file_name) write_file(fileName, scriptString) j.application.exe = 'sh' j.application.args = [File(fileName), str(j.id), str(j.id * 10)] j.outputfiles = [LocalFile('out.txt'), LocalFile('subdir/out.txt')] self.jobslice.append(j)
def test_a_CreateJob(self): from Ganga.GPI import jobs, Job, Executable, Local jobs.remove() j = Job() j.application = Executable() j.backend = Local()
def setUp(self): super(TestSmartMerger, self).setUp() from Ganga.GPI import Job, Executable, Local, File, LocalFile, config config['Mergers']['associate'] = {'txt': 'TextMerger'} self.jobslice = [] self.file_name = 'id_echo.sh' for i in range(4): j = Job(application=Executable(), backend=Local()) scriptString = ''' #!/bin/sh echo "Output from job $1." > out.txt echo "Output from job $2." > out2.txt ''' # write string to tmpfile tmpdir = tempfile.mktemp() os.mkdir(tmpdir) fileName = os.path.join(tmpdir, self.file_name) write_file(fileName, scriptString) j.application.exe = 'sh' j.application.args = [File(fileName), str(j.id), str(j.id * 10)] j.outputfiles = [LocalFile('out.txt'), LocalFile('out2.txt')] self.jobslice.append(j)
def testAppendWrongComponentItem(self): from Ganga.GPI import Executable from Ganga.GPIDev.Base.Proxy import TypeMismatchError assert len( self.test_job.application.gListComp) == 10, 'List is as we expect' # start appending TFiles [ self.test_job.application.gListComp.append(self._makeRandomTFile()) for _ in range(10) ] assert len( self.test_job.application.gListComp) == 20, 'List is as we expect' # test that the shortcut is called correctly [ self.test_job.application.gListComp.append( self._makeRandomString()) for _ in range(10) ] assert len( self.test_job.application.gListComp) == 30, 'List is as we expect' for g in self.test_job.application.gListComp: assert g._impl._category == 'files' try: self.test_job.application.gListComp.append(Executable()) assert False, 'Error must be thrown here' except TypeMismatchError: pass assert len( self.test_job.application.gListComp) == 30, 'List is as we expect'
def test_Savannah13459(self): from Ganga.GPI import Job, config from Ganga.GPI import Job, Executable j = Job() j.application = Executable() j.application.args = ['1', '2', '3'] self.assertEqual(j.application.args, ['1', '2', '3']) j.application.args[0] = '0' self.assertEqual(j.application.args, ['0', '2', '3'])
def test_c_onlyCreate(self): """here for testing job create""" from Ganga.GPI import Job, Executable, ArgSplitter, MassStorageFile j = Job() j.application = Executable(exe='touch') j.splitter = ArgSplitter(args=[['abc.txt'], ['def.txt']]) j.outputfiles = [ MassStorageFile(outputfilenameformat='/test/{sjid}-{fname}', namePattern='*.txt') ]
def test_Savannah87262(self): from Ganga.GPI import Executable, Job, shareref e = Executable() e.prepare() j = Job(application=e) s = shareref._impl.name print("%s" % s) print("%s" % e.is_prepared.name) print("%s" % j.application) self.assertEqual(s[e.is_prepared.name], 1)
def testSubjobsSubmit(self): from Ganga.GPI import Job, Executable, TestSubmitter, ArgSplitter from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as gangaList from Ganga.GPIDev.Base.Proxy import isType j = Job(application=Executable(), backend=TestSubmitter(time=1)) j.splitter = ArgSplitter(args=[['A'], ['B'], ['C']]) j.submit() assert run_until_completed(j), 'Job must complete' assert len(j.subjobs) == 3, 'splitting must occur' for jj in j.subjobs: assert not isType(jj.master, gangaList)
def testMergeThatAlwaysFailsOverwrite(self): from Ganga.GPI import Job, Executable, Local, LocalFile j = Job() j.application = Executable(exe='sh', args=['-c', 'echo foo > out.txt']) j.backend = Local() j.outputfiles = [LocalFile('out.txt')] j.splitter = CopySplitter() j.postprocessors = MergerTester(files=['out.txt'], overwrite=True) j.submit() assert run_until_state(j, 'failed', timeout=60) assert os.path.exists(os.path.join(j.outputdir, 'out.txt.merge_summary')), 'Summary file should be created'
def test_unprepareTrue(gpi): from Ganga.GPI import Job, Executable j = Job(application=Executable(exe='/bin/echo', args=['hello'])) j.submit() assert j.application.is_prepared is not None j2 = j.copy() assert j2.application.is_prepared is None j3 = Job(j) assert j3.application.is_prepared is None
def createTask(self): """create a task with some defaults""" from Ganga.GPI import tasks, CoreTask, CoreTransform, Executable, GenericSplitter t = CoreTask() trf = CoreTransform() trf.application = Executable() trf.unit_splitter = GenericSplitter() trf.unit_splitter.attribute = "application.args" trf.unit_splitter.values = ['arg 1', 'arg 2', 'arg 3'] t.appendTransform(trf) t.float = 20 return t
def test_unprepareFalse(gpi): from Ganga.GPI import Job, Executable k = Job(application=Executable(exe='/bin/echo', args=['hello'])) k.submit() assert k.application.is_prepared is not None k2 = k.copy() assert k2.application.is_prepared is not None k3 = Job(k) assert k.application.is_prepared is not None
def test_a_jobSubmit(self): """here for testing a submit""" from Ganga.GPI import Job, Executable, ArgSplitter, MassStorageFile j = Job() j.application = Executable(exe='touch') j.splitter = ArgSplitter(args=[['abc.txt'], ['def.txt']]) j.outputfiles = [ MassStorageFile(outputfilenameformat='/test/{sjid}-{fname}', namePattern='*.txt') ] j.submit() from GangaTest.Framework.utils import sleep_until_completed sleep_until_completed(j)
def test_Savannah15630(self): from Ganga.GPI import Job, Executable, Local, LocalFile from GangaTest.Framework.utils import sleep_until_completed j = Job() j.application = Executable(exe='touch', args=['out.dat']) j.backend = Local() j.outputfiles = [LocalFile('out.dat')] j.submit() self.assertTrue( sleep_until_completed(j, 60), 'Timeout on job submission: job is still not finished') import os.path p = os.path.join(j.outputdir, j.application.args[0]) self.assertTrue(os.path.exists(p))
def test_unprepareFalse(self): from Ganga.Utility.Config import setConfigOption from Ganga.GPI import Job, Executable setConfigOption('Preparable', 'unprepare_on_copy', 'False') k = Job(application=Executable(exe='/bin/echo', args=['hello'])) k.submit() assert (k.application.is_prepared != None) k2 = k.copy() assert (k2.application.is_prepared != None) k3 = Job(k) assert (k.application.is_prepared != None)
def test_unprepareTrue(self): from Ganga.Utility.Config import setConfigOption from Ganga.GPI import Job, Executable setConfigOption('Preparable', 'unprepare_on_copy', 'True') j = Job(application=Executable(exe='/bin/echo', args=['hello'])) j.submit() assert (j.application.is_prepared != None) j2 = j.copy() assert (j2.application.is_prepared == None) j3 = Job(j) assert (j3.application.is_prepared == None)
def test_Savannah19059(self): from Ganga.GPI import Executable, Job, Interactive, LocalFile import os.path from GangaTest.Framework.utils import sleep_until_completed # Test if Interactive backend copies back sandbox app = Executable() app.exe = 'touch' self.fname = 'abc' app.args = [self.fname] self.j = Job(backend=Interactive(), application=app, outputfiles=[LocalFile(self.fname)]) self.j.submit() self.assertTrue(sleep_until_completed(self.j, 60), 'Timeout on registering Interactive job as completed') self.assertTrue(os.path.exists(os.path.join(self.j.outputdir, self.fname)))
def setUp(self): super(TestFileChecker, self).setUp() from Ganga.GPI import Job, FileChecker, Executable, Local self.c = FileChecker() self.jobslice = [] args = ['1', '2', '12'] for arg in args: j = Job(application=Executable(), backend=Local()) # write string to tmpfile j.application.args = [arg] self.jobslice.append(j) for j in self.jobslice: j.submit() assert run_until_completed( j), 'Timeout on job submission: job is still not finished' self.assertEqual(j.status, 'completed')
def test_Savannah13979(self): from Ganga.GPI import Job, Executable, export, load self.fname = 'test_savannah_13979.ganga' j = Job(application=Executable()) args_set = [['a'], ['''a b'''], [ '''a b''', 'simple', 'normal\nnewline', """another multiline""" ]] for args in args_set: j.application.args = args export(j, self.fname) j2 = load(self.fname)[0] self.assertEqual(j2.application.args, args)
def testInterfaceLookFeel(self): """ This test tests the Executable app and that the DaVinci are assignable """ from Ganga.GPI import Job, LSF, Executable, DaVinci j1 = Job(name='my',application='DaVinci') j2 = Job(application = DaVinci()) j1.backend = LSF() j1.backend.queue = '8nm' j2.backend = j1.backend # deepcopy j2.backend.queue = '16nh' # shortcut bk2 = j2.backend # reference assert j2.backend.queue == '16nh' bk2.queue = '100nh' assert j2.backend.queue == '100nh' ap = Executable() j1.application = ap # deepcopy
def testFailure(self): """ Check a simple job fails and raises the correct exception """ from Ganga.GPI import Job, Dirac, Executable import time j = Job(backend=Dirac()) j.application = Executable(exe='ech') j.application.args = ['Hello World'] j.submit() assert run_until_state(j, 'failed', 220) filepath = os.path.join(j.outputdir, 'Ganga_Executable.log') i = 0 while not os.path.exists(filepath) and i < 10: i = i + 1 time.sleep(5) found = False with open(filepath, 'r') as f: for line in f: if "Exception occured in running process: ['ech', 'Hello World']" in line: found = True assert found
def Savannah47814(self): from Ganga.GPI import Job, Executable from GangaTest.Framework.utils import sleep_until_state, file_contains j = Job() j.application = Executable(exe='ThisScriptDoesNotExist') j.submit() failed = sleep_until_state( j, 60, state='failed', break_states=['new', 'killed', 'completed', 'unknown', 'removed']) self.assertTrue( failed, 'Job with illegal script should fail. Instead it went into the state %s' % j.status) import os.path f = os.path.join(j.outputdir, '__jobstatus__') self.assertTrue(file_contains(f, 'No such file or directory'), '__jobstatus__ file should contain error')
def testMergeRemoval(self): from Ganga.GPI import Job, Executable, Local, LocalFile, jobs # see Savannah 33710 j = Job() jobID = j.id # job will run for at least 20 seconds j.application = Executable(exe='sh', args=['-c', 'sleep 20; echo foo > out.txt']) j.backend = Local() j.outputfiles = [LocalFile('out.txt')] j.splitter = CopySplitter() j.postprocessors = MergerTester(files=['out.txt']) j.postprocessors[0].ignorefailed = True j.postprocessors[0].alwaysfail = True j.postprocessors[0].wait = 10 j.submit() run_until_state(j, state='running') j.remove() with pytest.raises(KeyError): jobs(jobID)
def test_k_Tasks(self): from Ganga.GPI import CoreTask, CoreTransform, Executable, Local, GenericSplitter, LocalFile, GangaDataset, \ GangaDatasetSplitter, TaskChainInput, File, tasks # -- TASKS EXAMPLE START # First create the overall Task t = CoreTask() # Now create the Transform ( -> Job template) trf = CoreTransform() trf.application = Executable() trf.backend = Local() # Set the unit splitter (unique to CoreTransform - you may have better ways of creating units in your own # plugins). This will create a unit based on the splitting of any given splitter # If you put in your own splitter here, use the trf.fields_to_copy string list to tell Tasks which fields of # a Job to preserve from the split. Here, Tasks already knows about GenericSplitter and knows that we want to # change the 'application' object for each Unit/Master Job trf.unit_splitter = GenericSplitter() trf.unit_splitter.attribute = "application.args" trf.unit_splitter.values = ['arg 1', 'arg 2', 'arg 3'] # Append the transform t.appendTransform(trf) # set the maximum number of active jobs to have running (allows for throttling) t.float = 100 # run the Task t.run() # -- TASKS EXAMPLE STOP # -- TASKS OVERVIEW START tasks tasks(0).overview() # -- TASKS OVERVIEW STOP t = CoreTask() trf = CoreTransform() trf.application = Executable() trf.backend = Local() trf.unit_splitter = GenericSplitter() trf.unit_splitter.attribute = "application.args" trf.unit_splitter.values = ['arg 1', 'arg 2', 'arg 3'] t.appendTransform(trf) t.float = 100 # -- TASKS OPTIONS START # note - done at the transform level rather than task level as different backends may not need it trf.max_active_threads = 10 # optional - specifies the max number of submissions to queue up trf.submit_with_threads = True # -- TASKS OPTIONS STOP # -- TASKS JOBCHAIN START # Create a test script open('my_script3.sh', 'w').write("""#!/bin/bash echo $PATH ls -ltr more __GangaInputData.txt__ echo "MY TEST FILE" > output_file.txt sleep 120 """) # Create the parent task t = CoreTask() # Create the first transform trf1 = CoreTransform() trf1.application = Executable() trf1.application.exe = File('my_script3.sh') trf1.outputfiles = [LocalFile("*.txt")] d = GangaDataset() d.files = [LocalFile("*.txt")] d.treat_as_inputfiles = True trf1.addInputData(d) trf1.files_per_unit = 1 trf1.submit_with_threads = True trf1.splitter = GangaDatasetSplitter() trf1.splitter.files_per_subjob = 2 trf1.backend = Local() t.appendTransform(trf1) # Create the second transform trf2 = CoreTransform() trf2.application = Executable() trf1.application.exe = File('my_script3.sh') trf2.submit_with_threads = True d = TaskChainInput() d.input_trf_id = trf1.getID() trf2.addInputData(d) trf2.splitter = GangaDatasetSplitter() trf2.splitter.files_per_subjob = 2 trf2.backend = Local() t.appendTransform(trf2) # Set the Task running t.float = 1 t.run()
def test_OptionsOptions(self): from Ganga.GPI import Job, Executable j = Job(application=Executable(args=['hello'])) assert (j.application.args[0] == 'hello')