def test_e_MultipleFiles(self):
"""Test that the wildcards work"""
from Ganga.GPI import LocalFile, MassStorageFile, Job, ArgSplitter
_ext = '.root'
_ext2 = '.txt'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
file_3 = generate_unique_temp_file(_ext2)
TestMassStorageWN._managed_files.append(file_1)
TestMassStorageWN._managed_files.append(file_2)
TestMassStorageWN._managed_files.append(file_3)
j = Job()
j.inputfiles = [
LocalFile(file_1),
LocalFile(file_2),
LocalFile(file_3)
]
j.splitter = ArgSplitter(
args=[[_] for _ in range(0, TestMassStorageWN.sj_len)])
j.outputfiles = [
MassStorageFile(namePattern='*' + _ext,
outputfilenameformat='{jid}/{sjid}/{fname}'),
MassStorageFile(namePattern='*' + _ext2)
]
j.submit()
def submit_job(brunel_app, reco_type, input_files=None, local=RUN_LOCAL):
# Set EvtMax depending on if this is a local job
brunel_app.extraOpts += 'from Configurables import Brunel\n'
brunel_app.extraOpts += 'Brunel().EvtMax = {}'.format(2 * int(local) - 1)
# Configure the corresponding Job
job = Job(name='VP hybrid distortions',
comment='{reco_type} reconstruction {suffix}'.format(
reco_type=reco_type, suffix=['', '(local)'][local]),
application=brunel_app,
splitter=SplitByFiles(filesPerJob=1, ignoremissing=True),
parallel_submit=True)
if local:
job.backend = Local()
job.outputfiles = [LocalFile('*.xdst'), LocalFile('*.root')]
job.inputdata = dataset[:1]
else:
job.backend = Dirac()
job.outputfiles = [DiracFile('*.xdst'), DiracFile('*.root')]
job.inputdata = dataset
job.inputfiles = input_files or []
queues.add(job.submit)
return True
def test_Savannah9007(self):
from Ganga.GPI import Job
j = Job()
from Ganga.Utility.Config import getConfig
if not getConfig('Output')['ForbidLegacyInput']:
j.inputsandbox = ['x']
else:
j.inputfiles = ['x']
def test_c_SplitJob(self):
"""Test submitting subjobs"""
from Ganga.GPI import Job, LocalFile, MassStorageFile, ArgSplitter
_ext = '.txt2'
file_1 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
j = Job()
j.inputfiles = [LocalFile(file_1)]
j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ])
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat='{jid}/{sjid}/{fname}')]
j.submit()
def test_a_testClientSideSubmit(self):
"""Test the client side code whilst stil using the Local backend"""
from Ganga.GPI import LocalFile, Job, ArgSplitter
file_1 = generate_unique_temp_file(TestLocalFileClient._ext)
file_2 = generate_unique_temp_file(TestLocalFileClient._ext)
TestLocalFileClient._managed_files.append(file_1)
TestLocalFileClient._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(args = [[_] for _ in range(TestLocalFileClient.sj_len)])
j.outputfiles = [LocalFile(namePattern='*'+TestLocalFileClient._ext)]
j.submit()
def test_a_Submit(self):
"""Test the ability to submit a job with some LocalFiles"""
from Ganga.GPI import jobs, Job, LocalFile, MassStorageFile
TestMassStorageWN.cleanUp()
_ext = '.txt'
file_1 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
j = Job()
j.inputfiles = [LocalFile(file_1)]
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat='{jid}/{fname}')]
j.submit()
def test_g_MultipleFiles(self):
"""Test that the wildcards work"""
from Ganga.GPI import LocalFile, MassStorageFile, Job, ArgSplitter
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
TestMassStorageWN._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ])
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat='{jid}_{sjid}_{fname}')]
j.submit()
def test_a_testClientSideSubmit(self):
"""Test the client side code whilst stil using the Local backend"""
from Ganga.GPI import LocalFile, Job, ArgSplitter
file_1 = generate_unique_temp_file(TestLocalFileClient._ext)
file_2 = generate_unique_temp_file(TestLocalFileClient._ext)
TestLocalFileClient._managed_files.append(file_1)
TestLocalFileClient._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(
args=[[_] for _ in range(TestLocalFileClient.sj_len)])
j.outputfiles = [LocalFile(namePattern='*' + TestLocalFileClient._ext)]
j.submit()
def testRun(self):
"""
Run a notebook application and check that it is executed
"""
from Ganga.GPI import Job, Notebook, LocalFile, jobs
j = Job()
a = Notebook()
testfilename = 'Test.ipynb'
dir = dirname(abspath(inspect.getfile(inspect.currentframe())))
j.inputfiles = [LocalFile(join(dir, testfilename))]
j.outputfiles = [LocalFile(testfilename)]
j.submit()
sleep_until_completed(jobs(0))
assert j.status in ['completed']
assert exists(join(j.outputdir, 'Test.ipynb'))
def testJobCopy(self):
"""Test that a job copy copies everything properly"""
from Ganga.GPI import Job, ARC, GenericSplitter, GangaDataset, LocalFile, FileChecker
from Ganga.GPIDev.Base.Proxy import isType
j = Job()
j.application.exe = "sleep"
j.application.args = ['myarg']
j.backend = ARC()
j.backend.CE = "my.ce"
j.inputdata = GangaDataset()
j.inputdata.files = [ LocalFile("*.txt") ]
j.inputfiles = [ LocalFile("*.txt") ]
j.name = "testname"
j.outputfiles = [ LocalFile("*.txt") ]
j.postprocessors = FileChecker(files=['stdout'], searchStrings = ['my search'])
j.splitter = GenericSplitter()
j.splitter.attribute = "application.args"
j.splitter.values = ['arg 1', 'arg 2', 'arg 3']
j2 = j.copy()
# test the copy has worked
self.assertTrue( isType(j2, Job) )
self.assertEqual( j2.application.exe, "sleep" )
self.assertEqual( j2.application.args, ["myarg"] )
self.assertTrue( isType(j2.backend, ARC) )
self.assertEqual( j2.backend.CE, "my.ce" )
self.assertTrue( isType(j2.inputdata, GangaDataset) )
self.assertEqual( len(j2.inputdata.files), 1 )
self.assertTrue( isType(j2.inputdata.files[0], LocalFile) )
self.assertEqual( j2.inputdata.files[0].namePattern, "*.txt" )
self.assertEqual( len(j2.inputfiles), 1 )
self.assertTrue( isType(j2.inputfiles[0], LocalFile) )
self.assertEqual( j2.inputfiles[0].namePattern, "*.txt" )
self.assertEqual( j2.name, "testname" )
self.assertEqual( len(j2.outputfiles), 1 )
self.assertTrue( isType(j2.outputfiles[0], LocalFile) )
self.assertEqual( j2.outputfiles[0].namePattern, "*.txt" )
self.assertEqual( len(j2.postprocessors), 1 )
self.assertTrue( isType(j2.postprocessors[0], FileChecker) )
self.assertEqual( j2.postprocessors[0].files, ["stdout"] )
self.assertEqual( j2.postprocessors[0].searchStrings, ["my search"] )
self.assertTrue( isType(j2.splitter, GenericSplitter) )
self.assertEqual( j2.splitter.attribute, "application.args" )
self.assertEqual( j2.splitter.values, ['arg 1', 'arg 2', 'arg 3'])
def test_c_SplitJob(self):
"""Test submitting subjobs"""
MassStorageFile = self.fileClass
from Ganga.GPI import Job, LocalFile, ArgSplitter
_ext = '.txt2'
file_1 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
j = Job()
j.inputfiles = [LocalFile(file_1)]
j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ])
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat=self.extendedFormat)]
j.submit()
for f in j.outputfiles:
assert f.outputfilenameformat == self.extendedFormat
def test_a_Submit(self):
"""Test the ability to submit a job with some LocalFiles"""
from Ganga.GPI import jobs, Job, LocalFile, MassStorageFile
TestMassStorageWN.cleanUp()
_ext = '.txt'
file_1 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
j = Job()
j.inputfiles = [LocalFile(file_1)]
j.outputfiles = [
MassStorageFile(namePattern='*' + _ext,
outputfilenameformat='{jid}/{fname}')
]
j.submit()
def test_a_Submit(self):
"""Test the ability to submit a job with some LocalFiles"""
MassStorageFile = self.fileClass
from Ganga.GPI import jobs, Job, LocalFile
_ext = '.txt'
file_1 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
j = Job()
j.inputfiles = [LocalFile(file_1)]
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat=self.standardFormat)]
j.submit()
for f in j.outputfiles:
assert f.outputfilenameformat == self.standardFormat
def test_a_testClientSideSubmit(self):
"""Test the client side code whilst stil using the Local backend"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
TestMassStorageClient.cleanUp()
assert getConfig('Output')[_getName(self.fileClass)]['backendPostprocess']['Local'] == 'client'
file_1 = generate_unique_temp_file(TestMassStorageClient._ext)
file_2 = generate_unique_temp_file(TestMassStorageClient._ext)
TestMassStorageClient._managed_files.append(file_1)
TestMassStorageClient._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(args = [[_] for _ in range(TestMassStorageClient.sj_len)])
j.outputfiles = [MassStorageFile(namePattern='*'+TestMassStorageClient._ext)]
j.submit()
def test_a_testClientSideSubmit(self):
"""Test the client side code whilst stil using the Local backend"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
TestMassStorageClient.cleanUp()
assert getConfig('Output')[_getName(self.fileClass)]['backendPostprocess']['Local'] == 'client'
file_1 = generate_unique_temp_file(TestMassStorageClient._ext)
file_2 = generate_unique_temp_file(TestMassStorageClient._ext)
TestMassStorageClient._managed_files.append(file_1)
TestMassStorageClient._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(args = [[_] for _ in range(TestMassStorageClient.sj_len)])
j.outputfiles = [MassStorageFile(namePattern='*'+TestMassStorageClient._ext)]
j.submit()
def test_a_testClientInputSubmit(self):
"""Test that a job can be submitted with inputfiles in the input"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
self._managed_files.append(file_1)
self._managed_files.append(file_2)
msf_1 = MassStorageFile(file_1)
msf_2 = MassStorageFile(file_2)
msf_1.put()
msf_2.put()
j = Job()
j.inputfiles = [msf_1, msf_2]
j.splitter = ArgSplitter(args = [[_] for _ in range(self.sj_len)])
j.outputfiles = [LocalFile(namePattern='*'+_ext)]
j.submit()
def test_a_testClientInputSubmit(self):
"""Test that a job can be submitted with inputfiles in the input"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
self._managed_files.append(file_1)
self._managed_files.append(file_2)
msf_1 = MassStorageFile(file_1)
msf_2 = MassStorageFile(file_2)
msf_1.put()
msf_2.put()
j = Job()
j.inputfiles = [msf_1, msf_2]
j.splitter = ArgSplitter(args=[[_] for _ in range(self.sj_len)])
j.outputfiles = [LocalFile(namePattern='*' + _ext)]
j.submit()
def test_g_MultipleFiles(self):
"""Test that the wildcards work"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
TestMassStorageWN._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ])
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat=self.customOutputFormat)]
for f in j.outputfiles:
assert f.outputfilenameformat == self.customOutputFormat
j.submit()
for f in j.outputfiles:
assert f.outputfilenameformat == self.customOutputFormat
def testShortCuts(self):
"""Make sure that shortcuts are called"""
from Ganga.GPIDev.Base.Proxy import isType, TypeMismatchError
from Ganga.GPI import Job, LocalFile
j = Job()
from Ganga.GPIDev.Lib.File import File as gFile
from Ganga.Utility.Config import getConfig
if not getConfig('Output')['ForbidLegacyInput']:
def testList(_list):
for l in _list:
assert isType(l, gFile), 'All entries must be Files'
j.inputsandbox = [
File(self._makeRandomString()) for _ in range(10)
]
assert len(j.inputsandbox) == 10, 'Must be added correctly'
testList(j.inputsandbox)
# now create with shortcuts - must still work
list_one = [self._makeRandomString() for _ in range(10)]
j.inputsandbox = list_one
assert len(j.inputsandbox) == 10, 'Must be added correctly'
testList(j.inputsandbox)
# now use mutable methods instead
list_two = [self._makeRandomString() for _ in range(10)]
j.inputsandbox.extend(list_two)
assert len(j.inputsandbox) == 20, 'Must be added correctly'
testList(j.inputsandbox)
try:
# no shortcut for this
j.inputsandbox.append(666)
assert False, 'Must get an Error here'
except TypeMismatchError:
pass
else:
def testList(_list):
for l in _list:
print("Testing: '%s' type: '%s'" % (l, type(l)))
assert isType(
l,
IGangaFile), "All entries must be of type IGangaFile"
list_one = [LocalFile(self._makeRandomString()) for _ in range(10)]
j.inputfiles = list_one
assert len(j.inputfiles) == 10, "Must add correctly"
testList(j.inputfiles)
list_two = [LocalFile(self._makeRandomString()) for _ in range(10)]
j.inputfiles = list_two
assert len(j.inputfiles) == 10, "Must still be added correctly"
assert j.inputfiles == list_two, "Must have been overloaded"
testList(j.inputfiles)
list_third = [
LocalFile(self._makeRandomString()) for _ in range(10)
]
j.inputfiles.extend(list_third)
assert len(j.inputfiles) == 20, "Must be added correctly finally"
testList(j.inputfiles)
try:
j.inputfiles.append(666)
assert False, "Must get an Error Here!"
except:
pass
def testShortCuts(self):
"""Make sure that shortcuts are called"""
from Ganga.GPIDev.Base.Proxy import isType, TypeMismatchError
from Ganga.GPI import Job, LocalFile
j = Job()
from Ganga.GPIDev.Lib.File import File as gFile
from Ganga.Utility.Config import getConfig
if not getConfig('Output')['ForbidLegacyInput']:
def testList(_list):
for l in _list:
assert isType(l, gFile), 'All entries must be Files'
j.inputsandbox = [File(self._makeRandomString())
for _ in range(10)]
assert len(j.inputsandbox) == 10, 'Must be added correctly'
testList(j.inputsandbox)
# now create with shortcuts - must still work
list_one = [self._makeRandomString() for _ in range(10)]
j.inputsandbox = list_one
assert len(j.inputsandbox) == 10, 'Must be added correctly'
testList(j.inputsandbox)
# now use mutable methods instead
list_two = [self._makeRandomString() for _ in range(10)]
j.inputsandbox.extend(list_two)
assert len(j.inputsandbox) == 20, 'Must be added correctly'
testList(j.inputsandbox)
try:
# no shortcut for this
j.inputsandbox.append(666)
assert False, 'Must get an Error here'
except TypeMismatchError:
pass
else:
def testList(_list):
for l in _list:
print("Testing: '%s' type: '%s'" % (l, type(l)))
assert isType(l, IGangaFile), "All entries must be of type IGangaFile"
list_one = [LocalFile(self._makeRandomString()) for _ in range(10)]
j.inputfiles = list_one
assert len(j.inputfiles) == 10, "Must add correctly"
testList(j.inputfiles)
list_two = [LocalFile(self._makeRandomString()) for _ in range(10)]
j.inputfiles = list_two
assert len(j.inputfiles) == 10, "Must still be added correctly"
assert j.inputfiles == list_two, "Must have been overloaded"
testList(j.inputfiles)
list_third = [LocalFile(self._makeRandomString()) for _ in range(10)]
j.inputfiles.extend(list_third)
assert len(j.inputfiles) == 20, "Must be added correctly finally"
testList(j.inputfiles)
try:
j.inputfiles.append(666)
assert False, "Must get an Error Here!"
except:
pass
def test_f_InputAndOutputData(self):
from Ganga.GPI import Job, File, LocalFile, GangaDataset, Local, plugins
# -- INPUTANDOUTPUTDATA BASIC START
# create a script to send
open('my_script2.sh', 'w').write("""#!/bin/bash
ls -ltr
more "my_input.txt"
echo "TESTING" > my_output.txt
""")
import os
os.system('chmod +x my_script2.sh')
# create a script to send
open('my_input.txt', 'w').write('Input Testing works!')
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [ LocalFile('my_input.txt') ]
j.outputfiles = [ LocalFile('my_output.txt') ]
j.submit()
# -- INPUTANDOUTPUTDATA BASIC STOP
# -- INPUTANDOUTPUTDATA PEEKOUTPUT START
j.peek() # list output dir contents
j.peek('my_output.txt')
# -- INPUTANDOUTPUTDATA PEEKOUTPUT STOP
# -- INPUTANDOUTPUTDATA FAILJOB START
# This job will fail
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [ LocalFile('my_input.txt') ]
j.outputfiles = [ LocalFile('my_output_FAIL.txt') ]
j.submit()
# -- INPUTANDOUTPUTDATA FAILJOB STOP
# -- INPUTANDOUTPUTDATA WILDCARD START
# This job will pick up both 'my_input.txt' and 'my_output.txt'
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [LocalFile('my_input.txt')]
j.outputfiles = [LocalFile('*.txt')]
j.submit()
# -- INPUTANDOUTPUTDATA WILDCARD STOP
# -- INPUTANDOUTPUTDATA OUTPUTFILES START
j.outputfiles
# -- INPUTANDOUTPUTDATA OUTPUTFILES STOP
# -- INPUTANDOUTPUTDATA INPUTDATA START
# Create a test script
open('my_script3.sh', 'w').write("""#!/bin/bash
echo $PATH
ls -ltr
more __GangaInputData.txt__
echo "MY TEST FILE" > output_file.txt
""")
import os
os.system('chmod +x my_script3.sh')
# Submit a job
j = Job()
j.application.exe = File('my_script3.sh')
j.inputdata = GangaDataset(files=[LocalFile('*.sh')])
j.backend = Local()
j.submit()
# -- INPUTANDOUTPUTDATA INPUTDATA STOP
# -- INPUTANDOUTPUTDATA GANGAFILES START
plugins('gangafiles')
def test_f_InputAndOutputData(self):
from Ganga.GPI import Job, File, LocalFile, GangaDataset, Local, plugins
# -- INPUTANDOUTPUTDATA BASIC START
# create a script to send
open('my_script2.sh', 'w').write("""#!/bin/bash
ls -ltr
more "my_input.txt"
echo "TESTING" > my_output.txt
""")
import os
os.system('chmod +x my_script2.sh')
# create a script to send
open('my_input.txt', 'w').write('Input Testing works!')
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [LocalFile('my_input.txt')]
j.outputfiles = [LocalFile('my_output.txt')]
j.submit()
# -- INPUTANDOUTPUTDATA BASIC STOP
# -- INPUTANDOUTPUTDATA PEEKOUTPUT START
j.peek() # list output dir contents
j.peek('my_output.txt')
# -- INPUTANDOUTPUTDATA PEEKOUTPUT STOP
# -- INPUTANDOUTPUTDATA FAILJOB START
# This job will fail
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [LocalFile('my_input.txt')]
j.outputfiles = [LocalFile('my_output_FAIL.txt')]
j.submit()
# -- INPUTANDOUTPUTDATA FAILJOB STOP
# -- INPUTANDOUTPUTDATA WILDCARD START
# This job will pick up both 'my_input.txt' and 'my_output.txt'
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [LocalFile('my_input.txt')]
j.outputfiles = [LocalFile('*.txt')]
j.submit()
# -- INPUTANDOUTPUTDATA WILDCARD STOP
# -- INPUTANDOUTPUTDATA OUTPUTFILES START
j.outputfiles
# -- INPUTANDOUTPUTDATA OUTPUTFILES STOP
# -- INPUTANDOUTPUTDATA INPUTDATA START
# Create a test script
open('my_script3.sh', 'w').write("""#!/bin/bash
echo $PATH
ls -ltr
more __GangaInputData.txt__
echo "MY TEST FILE" > output_file.txt
""")
import os
os.system('chmod +x my_script3.sh')
# Submit a job
j = Job()
j.application.exe = File('my_script3.sh')
j.inputdata = GangaDataset(files=[LocalFile('*.sh')])
j.backend = Local()
j.submit()
# -- INPUTANDOUTPUTDATA INPUTDATA STOP
# -- INPUTANDOUTPUTDATA GANGAFILES START
plugins('gangafiles')
