def test_d_RunningExecutables(self):
from Ganga.GPI import Job, File, Executable
# -- RUNNINGEXECUTABLES EXAMPLE START
# Already existing Exe
j = Job()
j.application = Executable()
j.application.exe = '/bin/ls'
j.application.args = ['-l', '-h']
j.submit()
# Wait for completion
j.peek("stdout")
# Send a script
open('my_script.sh', 'w').write("""#!/bin/bash
echo 'Current dir: ' `pwd`
echo 'Contents:'
ls -ltr
echo 'Args: ' $@
""")
import os
os.system('chmod +x my_script.sh')
j = Job()
j.application = Executable()
j.application.exe = File('my_script.sh')
j.submit()
# Wait for completion
j.peek("stdout")
def test_Savannah87262(self):
from Ganga.GPI import Executable, Job, shareref
e = Executable()
e.prepare()
j = Job(application=e)
s = shareref._impl.name
print("%s" % s)
print("%s" % e.is_prepared.name)
print("%s" % j.application)
self.assertEqual(s[e.is_prepared.name], 1)
def test_Savannah87262(self):
from Ganga.GPI import Executable, Job, shareref
e = Executable()
e.prepare()
j = Job(application=e)
s = shareref._impl.name
print("%s" % s)
print("%s" % e.is_prepared.name)
print("%s" % j.application)
self.assertEqual(s[e.is_prepared.name], 1)
def test_ConfigDefaults(self):
from Ganga.GPI import Executable
from Ganga.Utility.Config import setConfigOption
setConfigOption('defaults_Executable', 'exe', 'echo2')
setConfigOption('defaults_Executable', 'args', ['Hello World2'])
a = Executable()
assert a.exe == 'echo2'
assert a.args == ['Hello World2']
setConfigOption('defaults_Executable', 'exe', '/bin/echo')
k = Executable()
assert k.exe == '/bin/echo'
assert k.args == ['Hello World2']
def testAppendWrongComponentItem(self):
from Ganga.GPI import Executable
from Ganga.GPIDev.Base.Proxy import TypeMismatchError
assert len(
self.test_job.application.gListComp) == 10, 'List is as we expect'
# start appending TFiles
[
self.test_job.application.gListComp.append(self._makeRandomTFile())
for _ in range(10)
]
assert len(
self.test_job.application.gListComp) == 20, 'List is as we expect'
# test that the shortcut is called correctly
[
self.test_job.application.gListComp.append(
self._makeRandomString()) for _ in range(10)
]
assert len(
self.test_job.application.gListComp) == 30, 'List is as we expect'
for g in self.test_job.application.gListComp:
assert g._impl._category == 'files'
try:
self.test_job.application.gListComp.append(Executable())
assert False, 'Error must be thrown here'
except TypeMismatchError:
pass
assert len(
self.test_job.application.gListComp) == 30, 'List is as we expect'
def test_a_CreateJob(self):
from Ganga.GPI import jobs, Job, Executable, Local
jobs.remove()
j = Job()
j.application = Executable()
j.backend = Local()
def setUp(self):
super(TestSmartMerger, self).setUp()
from Ganga.GPI import Job, Executable, Local, File, LocalFile, config
config['Mergers']['associate'] = {'txt': 'TextMerger'}
self.jobslice = []
self.file_name = 'id_echo.sh'
for i in range(4):
j = Job(application=Executable(), backend=Local())
scriptString = '''
#!/bin/sh
echo "Output from job $1." > out.txt
echo "Output from job $2." > out2.txt
'''
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = 'sh'
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile('out.txt'), LocalFile('out2.txt')]
self.jobslice.append(j)
def setUp(self):
super(TestStructure, self).setUp()
from Ganga.GPI import Job, Executable, Local, File, LocalFile
from GangaTest.Framework.utils import write_file
self.jobslice = []
self.file_name = 'id_echo.sh'
for _ in range(5):
j = Job(application=Executable(), backend=Local())
scriptString = '''
#!/bin/sh
echo "Output from job $1." > out.txt
mkdir -p subdir
echo "Output from job $2." > subdir/out.txt
'''
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = 'sh'
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile('out.txt'), LocalFile('subdir/out.txt')]
self.jobslice.append(j)
def test_Savannah19059(self):
from Ganga.GPI import Executable, Job, Interactive, LocalFile
import os.path
from GangaTest.Framework.utils import sleep_until_completed
# Test if Interactive backend copies back sandbox
app = Executable()
app.exe = 'touch'
self.fname = 'abc'
app.args = [self.fname]
self.j = Job(backend=Interactive(), application=app, outputfiles=[LocalFile(self.fname)])
self.j.submit()
self.assertTrue(sleep_until_completed(self.j, 60), 'Timeout on registering Interactive job as completed')
self.assertTrue(os.path.exists(os.path.join(self.j.outputdir, self.fname)))
def test_Savannah19059(self):
from Ganga.GPI import Executable, Job, Interactive, LocalFile
import os.path
from GangaTest.Framework.utils import sleep_until_completed
# Test if Interactive backend copies back sandbox
app = Executable()
app.exe = 'touch'
self.fname = 'abc'
app.args = [self.fname]
self.j = Job(backend=Interactive(), application=app, outputfiles=[LocalFile(self.fname)])
self.j.submit()
self.assertTrue(sleep_until_completed(self.j, 60), 'Timeout on registering Interactive job as completed')
self.assertTrue(os.path.exists(os.path.join(self.j.outputdir, self.fname)))
def test_Savannah13459(self):
from Ganga.GPI import Job, config
from Ganga.GPI import Job, Executable
j = Job()
j.application = Executable()
j.application.args = ['1', '2', '3']
self.assertEqual(j.application.args, ['1', '2', '3'])
j.application.args[0] = '0'
self.assertEqual(j.application.args, ['0', '2', '3'])
def test_c_onlyCreate(self):
"""here for testing job create"""
from Ganga.GPI import Job, Executable, ArgSplitter, MassStorageFile
j = Job()
j.application = Executable(exe='touch')
j.splitter = ArgSplitter(args=[['abc.txt'], ['def.txt']])
j.outputfiles = [
MassStorageFile(outputfilenameformat='/test/{sjid}-{fname}',
namePattern='*.txt')
]
def testSubjobsSubmit(self):
from Ganga.GPI import Job, Executable, TestSubmitter, ArgSplitter
from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList as gangaList
from Ganga.GPIDev.Base.Proxy import isType
j = Job(application=Executable(), backend=TestSubmitter(time=1))
j.splitter = ArgSplitter(args=[['A'], ['B'], ['C']])
j.submit()
assert run_until_completed(j), 'Job must complete'
assert len(j.subjobs) == 3, 'splitting must occur'
for jj in j.subjobs:
assert not isType(jj.master, gangaList)
def testMergeThatAlwaysFailsOverwrite(self):
from Ganga.GPI import Job, Executable, Local, LocalFile
j = Job()
j.application = Executable(exe='sh', args=['-c', 'echo foo > out.txt'])
j.backend = Local()
j.outputfiles = [LocalFile('out.txt')]
j.splitter = CopySplitter()
j.postprocessors = MergerTester(files=['out.txt'], overwrite=True)
j.submit()
assert run_until_state(j, 'failed', timeout=60)
assert os.path.exists(os.path.join(j.outputdir, 'out.txt.merge_summary')), 'Summary file should be created'
def test_unprepareTrue(gpi):
from Ganga.GPI import Job, Executable
j = Job(application=Executable(exe='/bin/echo', args=['hello']))
j.submit()
assert j.application.is_prepared is not None
j2 = j.copy()
assert j2.application.is_prepared is None
j3 = Job(j)
assert j3.application.is_prepared is None
def createTask(self):
"""create a task with some defaults"""
from Ganga.GPI import tasks, CoreTask, CoreTransform, Executable, GenericSplitter
t = CoreTask()
trf = CoreTransform()
trf.application = Executable()
trf.unit_splitter = GenericSplitter()
trf.unit_splitter.attribute = "application.args"
trf.unit_splitter.values = ['arg 1', 'arg 2', 'arg 3']
t.appendTransform(trf)
t.float = 20
return t
def test_unprepareFalse(gpi):
from Ganga.GPI import Job, Executable
k = Job(application=Executable(exe='/bin/echo', args=['hello']))
k.submit()
assert k.application.is_prepared is not None
k2 = k.copy()
assert k2.application.is_prepared is not None
k3 = Job(k)
assert k.application.is_prepared is not None
def test_a_jobSubmit(self):
"""here for testing a submit"""
from Ganga.GPI import Job, Executable, ArgSplitter, MassStorageFile
j = Job()
j.application = Executable(exe='touch')
j.splitter = ArgSplitter(args=[['abc.txt'], ['def.txt']])
j.outputfiles = [
MassStorageFile(outputfilenameformat='/test/{sjid}-{fname}',
namePattern='*.txt')
]
j.submit()
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(j)
def test_Savannah15630(self):
from Ganga.GPI import Job, Executable, Local, LocalFile
from GangaTest.Framework.utils import sleep_until_completed
j = Job()
j.application = Executable(exe='touch', args=['out.dat'])
j.backend = Local()
j.outputfiles = [LocalFile('out.dat')]
j.submit()
self.assertTrue(
sleep_until_completed(j, 60),
'Timeout on job submission: job is still not finished')
import os.path
p = os.path.join(j.outputdir, j.application.args[0])
self.assertTrue(os.path.exists(p))
def test_unprepareFalse(self):
from Ganga.Utility.Config import setConfigOption
from Ganga.GPI import Job, Executable
setConfigOption('Preparable', 'unprepare_on_copy', 'False')
k = Job(application=Executable(exe='/bin/echo', args=['hello']))
k.submit()
assert (k.application.is_prepared != None)
k2 = k.copy()
assert (k2.application.is_prepared != None)
k3 = Job(k)
assert (k.application.is_prepared != None)
def test_unprepareTrue(self):
from Ganga.Utility.Config import setConfigOption
from Ganga.GPI import Job, Executable
setConfigOption('Preparable', 'unprepare_on_copy', 'True')
j = Job(application=Executable(exe='/bin/echo', args=['hello']))
j.submit()
assert (j.application.is_prepared != None)
j2 = j.copy()
assert (j2.application.is_prepared == None)
j3 = Job(j)
assert (j3.application.is_prepared == None)
def setUp(self):
super(TestFileChecker, self).setUp()
from Ganga.GPI import Job, FileChecker, Executable, Local
self.c = FileChecker()
self.jobslice = []
args = ['1', '2', '12']
for arg in args:
j = Job(application=Executable(), backend=Local())
# write string to tmpfile
j.application.args = [arg]
self.jobslice.append(j)
for j in self.jobslice:
j.submit()
assert run_until_completed(
j), 'Timeout on job submission: job is still not finished'
self.assertEqual(j.status, 'completed')
def test_Savannah13979(self):
from Ganga.GPI import Job, Executable, export, load
self.fname = 'test_savannah_13979.ganga'
j = Job(application=Executable())
args_set = [['a'], ['''a
b'''],
[
'''a
b''', 'simple', 'normal\nnewline', """another
multiline"""
]]
for args in args_set:
j.application.args = args
export(j, self.fname)
j2 = load(self.fname)[0]
self.assertEqual(j2.application.args, args)
def testInterfaceLookFeel(self):
"""
This test tests the Executable app and that the DaVinci are assignable
"""
from Ganga.GPI import Job, LSF, Executable, DaVinci
j1 = Job(name='my',application='DaVinci')
j2 = Job(application = DaVinci())
j1.backend = LSF()
j1.backend.queue = '8nm'
j2.backend = j1.backend # deepcopy
j2.backend.queue = '16nh' # shortcut
bk2 = j2.backend # reference
assert j2.backend.queue == '16nh'
bk2.queue = '100nh'
assert j2.backend.queue == '100nh'
ap = Executable()
j1.application = ap # deepcopy
def testFailure(self):
"""
Check a simple job fails and raises the correct exception
"""
from Ganga.GPI import Job, Dirac, Executable
import time
j = Job(backend=Dirac())
j.application = Executable(exe='ech')
j.application.args = ['Hello World']
j.submit()
assert run_until_state(j, 'failed', 220)
filepath = os.path.join(j.outputdir, 'Ganga_Executable.log')
i = 0
while not os.path.exists(filepath) and i < 10:
i = i + 1
time.sleep(5)
found = False
with open(filepath, 'r') as f:
for line in f:
if "Exception occured in running process: ['ech', 'Hello World']" in line:
found = True
assert found
def Savannah47814(self):
from Ganga.GPI import Job, Executable
from GangaTest.Framework.utils import sleep_until_state, file_contains
j = Job()
j.application = Executable(exe='ThisScriptDoesNotExist')
j.submit()
failed = sleep_until_state(
j,
60,
state='failed',
break_states=['new', 'killed', 'completed', 'unknown', 'removed'])
self.assertTrue(
failed,
'Job with illegal script should fail. Instead it went into the state %s'
% j.status)
import os.path
f = os.path.join(j.outputdir, '__jobstatus__')
self.assertTrue(file_contains(f, 'No such file or directory'),
'__jobstatus__ file should contain error')
def testMergeRemoval(self):
from Ganga.GPI import Job, Executable, Local, LocalFile, jobs
# see Savannah 33710
j = Job()
jobID = j.id
# job will run for at least 20 seconds
j.application = Executable(exe='sh',
args=['-c', 'sleep 20; echo foo > out.txt'])
j.backend = Local()
j.outputfiles = [LocalFile('out.txt')]
j.splitter = CopySplitter()
j.postprocessors = MergerTester(files=['out.txt'])
j.postprocessors[0].ignorefailed = True
j.postprocessors[0].alwaysfail = True
j.postprocessors[0].wait = 10
j.submit()
run_until_state(j, state='running')
j.remove()
with pytest.raises(KeyError):
jobs(jobID)
def test_k_Tasks(self):
from Ganga.GPI import CoreTask, CoreTransform, Executable, Local, GenericSplitter, LocalFile, GangaDataset, \
GangaDatasetSplitter, TaskChainInput, File, tasks
# -- TASKS EXAMPLE START
# First create the overall Task
t = CoreTask()
# Now create the Transform ( -> Job template)
trf = CoreTransform()
trf.application = Executable()
trf.backend = Local()
# Set the unit splitter (unique to CoreTransform - you may have better ways of creating units in your own
# plugins). This will create a unit based on the splitting of any given splitter
# If you put in your own splitter here, use the trf.fields_to_copy string list to tell Tasks which fields of
# a Job to preserve from the split. Here, Tasks already knows about GenericSplitter and knows that we want to
# change the 'application' object for each Unit/Master Job
trf.unit_splitter = GenericSplitter()
trf.unit_splitter.attribute = "application.args"
trf.unit_splitter.values = ['arg 1', 'arg 2', 'arg 3']
# Append the transform
t.appendTransform(trf)
# set the maximum number of active jobs to have running (allows for throttling)
t.float = 100
# run the Task
t.run()
# -- TASKS EXAMPLE STOP
# -- TASKS OVERVIEW START
tasks
tasks(0).overview()
# -- TASKS OVERVIEW STOP
t = CoreTask()
trf = CoreTransform()
trf.application = Executable()
trf.backend = Local()
trf.unit_splitter = GenericSplitter()
trf.unit_splitter.attribute = "application.args"
trf.unit_splitter.values = ['arg 1', 'arg 2', 'arg 3']
t.appendTransform(trf)
t.float = 100
# -- TASKS OPTIONS START
# note - done at the transform level rather than task level as different backends may not need it
trf.max_active_threads = 10 # optional - specifies the max number of submissions to queue up
trf.submit_with_threads = True
# -- TASKS OPTIONS STOP
# -- TASKS JOBCHAIN START
# Create a test script
open('my_script3.sh', 'w').write("""#!/bin/bash
echo $PATH
ls -ltr
more __GangaInputData.txt__
echo "MY TEST FILE" > output_file.txt
sleep 120
""")
# Create the parent task
t = CoreTask()
# Create the first transform
trf1 = CoreTransform()
trf1.application = Executable()
trf1.application.exe = File('my_script3.sh')
trf1.outputfiles = [LocalFile("*.txt")]
d = GangaDataset()
d.files = [LocalFile("*.txt")]
d.treat_as_inputfiles = True
trf1.addInputData(d)
trf1.files_per_unit = 1
trf1.submit_with_threads = True
trf1.splitter = GangaDatasetSplitter()
trf1.splitter.files_per_subjob = 2
trf1.backend = Local()
t.appendTransform(trf1)
# Create the second transform
trf2 = CoreTransform()
trf2.application = Executable()
trf1.application.exe = File('my_script3.sh')
trf2.submit_with_threads = True
d = TaskChainInput()
d.input_trf_id = trf1.getID()
trf2.addInputData(d)
trf2.splitter = GangaDatasetSplitter()
trf2.splitter.files_per_subjob = 2
trf2.backend = Local()
t.appendTransform(trf2)
# Set the Task running
t.float = 1
t.run()
def test_OptionsOptions(self):
from Ganga.GPI import Job, Executable
j = Job(application=Executable(args=['hello']))
assert (j.application.args[0] == 'hello')
