def submit_job(brunel_app, reco_type, input_files=None, local=RUN_LOCAL):
# Set EvtMax depending on if this is a local job
brunel_app.extraOpts += 'from Configurables import Brunel\n'
brunel_app.extraOpts += 'Brunel().EvtMax = {}'.format(2 * int(local) - 1)
# Configure the corresponding Job
job = Job(name='VP hybrid distortions',
comment='{reco_type} reconstruction {suffix}'.format(
reco_type=reco_type, suffix=['', '(local)'][local]),
application=brunel_app,
splitter=SplitByFiles(filesPerJob=1, ignoremissing=True),
parallel_submit=True)
if local:
job.backend = Local()
job.outputfiles = [LocalFile('*.xdst'), LocalFile('*.root')]
job.inputdata = dataset[:1]
else:
job.backend = Dirac()
job.outputfiles = [DiracFile('*.xdst'), DiracFile('*.root')]
job.inputdata = dataset
job.inputfiles = input_files or []
queues.add(job.submit)
return True
def setUp(self):
super(TestCustomMerger, self).setUp()
from Ganga.GPI import Job, Executable, Local, File, LocalFile
self.jobslice = []
self.file_name = 'id_echo.sh'
for i in range(2):
j = Job(application=Executable(), backend=Local())
scriptString = '''
#!/bin/sh
echo "Output from job $1." > out.txt
echo "Output from job $2." > out2.txt
'''
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = 'sh'
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile('out.txt'), LocalFile('out2.txt')]
self.jobslice.append(j)
def test_e_MultipleFiles(self):
"""Test that the wildcards work"""
from Ganga.GPI import LocalFile, MassStorageFile, Job, ArgSplitter
_ext = '.root'
_ext2 = '.txt'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
file_3 = generate_unique_temp_file(_ext2)
TestMassStorageWN._managed_files.append(file_1)
TestMassStorageWN._managed_files.append(file_2)
TestMassStorageWN._managed_files.append(file_3)
j = Job()
j.inputfiles = [
LocalFile(file_1),
LocalFile(file_2),
LocalFile(file_3)
]
j.splitter = ArgSplitter(
args=[[_] for _ in range(0, TestMassStorageWN.sj_len)])
j.outputfiles = [
MassStorageFile(namePattern='*' + _ext,
outputfilenameformat='{jid}/{sjid}/{fname}'),
MassStorageFile(namePattern='*' + _ext2)
]
j.submit()
def testDatasets(self):
from Ganga.GPI import DiracFile, PhysicalFile, LHCbDataset, Job, LocalFile
# test constructors/setters
ds = LHCbDataset(['lfn:a', 'pfn:b'])
assert len(ds) == 2
print(ds[0])
assert isinstance(ds[0], DiracFile)
assert isinstance(ds[1], PhysicalFile)
ds = LHCbDataset()
ds.files = ['lfn:a', 'pfn:b']
assert isinstance(ds[0], DiracFile)
assert isinstance(ds[1], PhysicalFile)
ds.files.append('lfn:c')
assert isinstance(ds[-1], DiracFile)
d = OutputData(['a', 'b'])
assert isinstance(d.files[0],str)
assert isinstance(d.files[1],str)
# check job assignments
j = Job()
j.inputdata = ['lfn:a', 'pfn:b']
assert isinstance(j.inputdata, LHCbDataset)
j.outputfiles = ['a', DiracFile('b')]
assert isinstance(j.outputfiles[0], LocalFile)
print(type(j.outputfiles[1]))
assert isinstance(j.outputfiles[1], DiracFile)
LFN_DATA = ['LFN:/lhcb/LHCb/Collision11/DIMUON.DST/00016768/0000/00016768_00000005_1.dimuon.dst',
'LFN:/lhcb/LHCb/Collision11/DIMUON.DST/00016768/0000/00016768_00000006_1.dimuon.dst']
ds = LHCbDataset(LFN_DATA)
assert len(ds.getReplicas().keys()) == 2
assert ds.getCatalog()
def setUp(self):
super(TestStructure, self).setUp()
from Ganga.GPI import Job, Executable, Local, File, LocalFile
from GangaTest.Framework.utils import write_file
self.jobslice = []
self.file_name = 'id_echo.sh'
for _ in range(5):
j = Job(application=Executable(), backend=Local())
scriptString = '''
#!/bin/sh
echo "Output from job $1." > out.txt
mkdir -p subdir
echo "Output from job $2." > subdir/out.txt
'''
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = 'sh'
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile('out.txt'), LocalFile('subdir/out.txt')]
self.jobslice.append(j)
def setUp(self):
super(TestSmartMerger, self).setUp()
from Ganga.GPI import Job, Executable, Local, File, LocalFile, config
config['Mergers']['associate'] = {'txt': 'TextMerger'}
self.jobslice = []
self.file_name = 'id_echo.sh'
for i in range(4):
j = Job(application=Executable(), backend=Local())
scriptString = '''
#!/bin/sh
echo "Output from job $1." > out.txt
echo "Output from job $2." > out2.txt
'''
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = 'sh'
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile('out.txt'), LocalFile('out2.txt')]
self.jobslice.append(j)
def setUp(self):
super(TestStructure, self).setUp()
from Ganga.GPI import Job, Executable, Local, File, LocalFile
from GangaTest.Framework.utils import write_file
self.jobslice = []
self.file_name = "id_echo.sh"
for _ in range(5):
j = Job(application=Executable(), backend=Local())
scriptString = """
#!/bin/sh
echo "Output from job $1." > out.txt
mkdir -p subdir
echo "Output from job $2." > subdir/out.txt
"""
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = "sh"
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile("out.txt"), LocalFile("subdir/out.txt")]
self.jobslice.append(j)
def test_c_onlyCreate(self):
"""here for testing job create"""
from Ganga.GPI import Job, Executable, ArgSplitter, MassStorageFile
j=Job()
j.application=Executable(exe='touch')
j.splitter=ArgSplitter(args=[['abc.txt'], ['def.txt']])
j.outputfiles=[MassStorageFile(outputfilenameformat = '/test/{sjid}-{fname}', namePattern = '*.txt')]
def test_c_onlyCreate(self):
"""here for testing job create"""
from Ganga.GPI import Job, Executable, ArgSplitter, MassStorageFile
j = Job()
j.application = Executable(exe='touch')
j.splitter = ArgSplitter(args=[['abc.txt'], ['def.txt']])
j.outputfiles = [
MassStorageFile(outputfilenameformat='/test/{sjid}-{fname}',
namePattern='*.txt')
]
def test_a_jobSubmit(self):
"""here for testing a submit"""
from Ganga.GPI import Job, Executable, ArgSplitter, MassStorageFile
j=Job()
j.application=Executable(exe='touch')
j.splitter=ArgSplitter(args=[['abc.txt'], ['def.txt']])
j.outputfiles=[MassStorageFile(outputfilenameformat = '/test/{sjid}-{fname}', namePattern = '*.txt')]
j.submit()
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(j)
def testMergeThatAlwaysFailsOverwrite(self):
from Ganga.GPI import Job, Executable, Local, LocalFile
j = Job()
j.application = Executable(exe='sh', args=['-c', 'echo foo > out.txt'])
j.backend = Local()
j.outputfiles = [LocalFile('out.txt')]
j.splitter = CopySplitter()
j.postprocessors = MergerTester(files=['out.txt'], overwrite=True)
j.submit()
assert run_until_state(j, 'failed', timeout=60)
assert os.path.exists(os.path.join(j.outputdir, 'out.txt.merge_summary')), 'Summary file should be created'
def testMergeThatAlwaysFailsOverwrite(self):
from Ganga.GPI import Job, Executable, Local, LocalFile
j = Job()
j.application = Executable(exe='sh', args=['-c', 'echo foo > out.txt'])
j.backend = Local()
j.outputfiles = [LocalFile('out.txt')]
j.splitter = CopySplitter()
j.postprocessors = MergerTester(files=['out.txt'], overwrite=True)
j.submit()
assert run_until_state(j, 'failed', timeout=60)
assert os.path.exists(os.path.join(j.outputdir, 'out.txt.merge_summary')), 'Summary file should be created'
def test_c_SplitJob(self):
"""Test submitting subjobs"""
from Ganga.GPI import Job, LocalFile, MassStorageFile, ArgSplitter
_ext = '.txt2'
file_1 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
j = Job()
j.inputfiles = [LocalFile(file_1)]
j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ])
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat='{jid}/{sjid}/{fname}')]
j.submit()
def test_a_jobSubmit(self):
"""here for testing a submit"""
from Ganga.GPI import Job, Executable, ArgSplitter, MassStorageFile
j = Job()
j.application = Executable(exe='touch')
j.splitter = ArgSplitter(args=[['abc.txt'], ['def.txt']])
j.outputfiles = [
MassStorageFile(outputfilenameformat='/test/{sjid}-{fname}',
namePattern='*.txt')
]
j.submit()
from GangaTest.Framework.utils import sleep_until_completed
sleep_until_completed(j)
def testMergeThatAlwaysFailsIgnoreFailed(self):
from Ganga.GPI import Job, Executable, Local, LocalFile
j = Job()
j.application = Executable(exe='sh', args=['-c', 'echo foo > out.txt'])
j.backend = Local()
j.outputfiles = [LocalFile('out.txt')]
j.splitter = CopySplitter()
j.postprocessors = MergerTester(files=['out.txt'], ignorefailed=True)
j.submit()
sleep_until_completed(j, 60)
assert j.status == 'failed'
assert os.path.exists(os.path.join(j.outputdir, 'out.txt.merge_summary')), 'Summary file should be created'
def test_a_testClientSideSubmit(self):
"""Test the client side code whilst stil using the Local backend"""
from Ganga.GPI import LocalFile, Job, ArgSplitter
file_1 = generate_unique_temp_file(TestLocalFileClient._ext)
file_2 = generate_unique_temp_file(TestLocalFileClient._ext)
TestLocalFileClient._managed_files.append(file_1)
TestLocalFileClient._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(args = [[_] for _ in range(TestLocalFileClient.sj_len)])
j.outputfiles = [LocalFile(namePattern='*'+TestLocalFileClient._ext)]
j.submit()
def test_a_Submit(self):
"""Test the ability to submit a job with some LocalFiles"""
from Ganga.GPI import jobs, Job, LocalFile, MassStorageFile
TestMassStorageWN.cleanUp()
_ext = '.txt'
file_1 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
j = Job()
j.inputfiles = [LocalFile(file_1)]
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat='{jid}/{fname}')]
j.submit()
def testMergeThatAlwaysFailsFlagsSet(self):
from Ganga.GPI import Job, Executable, Local, LocalFile
j = Job()
j.application = Executable(exe="sh", args=["-c", "echo foo > out.txt"])
j.backend = Local()
j.outputfiles = [LocalFile("out.txt")]
j.splitter = CopySplitter()
j.postprocessors = MergerTester(files=["out.txt"], ignorefailed=True, overwrite=True)
j.submit()
run_until_completed(j, timeout=60)
assert j.status == "failed"
assert os.path.exists(os.path.join(j.outputdir, "out.txt.merge_summary")), "Summary file should be created"
def test_Savannah15630(self):
from Ganga.GPI import Job, Executable, Local, LocalFile
from GangaTest.Framework.utils import sleep_until_completed
j = Job()
j.application = Executable(exe='touch', args=['out.dat'])
j.backend = Local()
j.outputfiles = [LocalFile('out.dat')]
j.submit()
self.assertTrue(sleep_until_completed(j, 60), 'Timeout on job submission: job is still not finished')
import os.path
p = os.path.join(j.outputdir, j.application.args[0])
self.assertTrue(os.path.exists(p))
def test_g_MultipleFiles(self):
"""Test that the wildcards work"""
from Ganga.GPI import LocalFile, MassStorageFile, Job, ArgSplitter
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
TestMassStorageWN._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ])
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat='{jid}_{sjid}_{fname}')]
j.submit()
def test_a_testClientSideSubmit(self):
"""Test the client side code whilst stil using the Local backend"""
from Ganga.GPI import LocalFile, Job, ArgSplitter
file_1 = generate_unique_temp_file(TestLocalFileClient._ext)
file_2 = generate_unique_temp_file(TestLocalFileClient._ext)
TestLocalFileClient._managed_files.append(file_1)
TestLocalFileClient._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(
args=[[_] for _ in range(TestLocalFileClient.sj_len)])
j.outputfiles = [LocalFile(namePattern='*' + TestLocalFileClient._ext)]
j.submit()
def testRun(self):
"""
Run a notebook application and check that it is executed
"""
from Ganga.GPI import Job, Notebook, LocalFile, jobs
j = Job()
a = Notebook()
testfilename = 'Test.ipynb'
dir = dirname(abspath(inspect.getfile(inspect.currentframe())))
j.inputfiles = [LocalFile(join(dir, testfilename))]
j.outputfiles = [LocalFile(testfilename)]
j.submit()
sleep_until_completed(jobs(0))
assert j.status in ['completed']
assert exists(join(j.outputdir, 'Test.ipynb'))
def test_Savannah15630(self):
from Ganga.GPI import Job, Executable, Local, LocalFile
from GangaTest.Framework.utils import sleep_until_completed
j = Job()
j.application = Executable(exe='touch', args=['out.dat'])
j.backend = Local()
j.outputfiles = [LocalFile('out.dat')]
j.submit()
self.assertTrue(
sleep_until_completed(j, 60),
'Timeout on job submission: job is still not finished')
import os.path
p = os.path.join(j.outputdir, j.application.args[0])
self.assertTrue(os.path.exists(p))
def testJobCopy(self):
"""Test that a job copy copies everything properly"""
from Ganga.GPI import Job, ARC, GenericSplitter, GangaDataset, LocalFile, FileChecker
from Ganga.GPIDev.Base.Proxy import isType
j = Job()
j.application.exe = "sleep"
j.application.args = ['myarg']
j.backend = ARC()
j.backend.CE = "my.ce"
j.inputdata = GangaDataset()
j.inputdata.files = [ LocalFile("*.txt") ]
j.inputfiles = [ LocalFile("*.txt") ]
j.name = "testname"
j.outputfiles = [ LocalFile("*.txt") ]
j.postprocessors = FileChecker(files=['stdout'], searchStrings = ['my search'])
j.splitter = GenericSplitter()
j.splitter.attribute = "application.args"
j.splitter.values = ['arg 1', 'arg 2', 'arg 3']
j2 = j.copy()
# test the copy has worked
self.assertTrue( isType(j2, Job) )
self.assertEqual( j2.application.exe, "sleep" )
self.assertEqual( j2.application.args, ["myarg"] )
self.assertTrue( isType(j2.backend, ARC) )
self.assertEqual( j2.backend.CE, "my.ce" )
self.assertTrue( isType(j2.inputdata, GangaDataset) )
self.assertEqual( len(j2.inputdata.files), 1 )
self.assertTrue( isType(j2.inputdata.files[0], LocalFile) )
self.assertEqual( j2.inputdata.files[0].namePattern, "*.txt" )
self.assertEqual( len(j2.inputfiles), 1 )
self.assertTrue( isType(j2.inputfiles[0], LocalFile) )
self.assertEqual( j2.inputfiles[0].namePattern, "*.txt" )
self.assertEqual( j2.name, "testname" )
self.assertEqual( len(j2.outputfiles), 1 )
self.assertTrue( isType(j2.outputfiles[0], LocalFile) )
self.assertEqual( j2.outputfiles[0].namePattern, "*.txt" )
self.assertEqual( len(j2.postprocessors), 1 )
self.assertTrue( isType(j2.postprocessors[0], FileChecker) )
self.assertEqual( j2.postprocessors[0].files, ["stdout"] )
self.assertEqual( j2.postprocessors[0].searchStrings, ["my search"] )
self.assertTrue( isType(j2.splitter, GenericSplitter) )
self.assertEqual( j2.splitter.attribute, "application.args" )
self.assertEqual( j2.splitter.values, ['arg 1', 'arg 2', 'arg 3'])
def test_c_SplitJob(self):
"""Test submitting subjobs"""
MassStorageFile = self.fileClass
from Ganga.GPI import Job, LocalFile, ArgSplitter
_ext = '.txt2'
file_1 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
j = Job()
j.inputfiles = [LocalFile(file_1)]
j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ])
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat=self.extendedFormat)]
j.submit()
for f in j.outputfiles:
assert f.outputfilenameformat == self.extendedFormat
def test_a_Submit(self):
"""Test the ability to submit a job with some LocalFiles"""
MassStorageFile = self.fileClass
from Ganga.GPI import jobs, Job, LocalFile
_ext = '.txt'
file_1 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
j = Job()
j.inputfiles = [LocalFile(file_1)]
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat=self.standardFormat)]
j.submit()
for f in j.outputfiles:
assert f.outputfilenameformat == self.standardFormat
def test_a_Submit(self):
"""Test the ability to submit a job with some LocalFiles"""
from Ganga.GPI import jobs, Job, LocalFile, MassStorageFile
TestMassStorageWN.cleanUp()
_ext = '.txt'
file_1 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
j = Job()
j.inputfiles = [LocalFile(file_1)]
j.outputfiles = [
MassStorageFile(namePattern='*' + _ext,
outputfilenameformat='{jid}/{fname}')
]
j.submit()
def testInterfaceLookFeel(self):
# Just testing that the job construction works
from Ganga.GPI import Job, Im3ShapeApp, Im3ShapeSplitter, DiracFile, LocalFile, GangaDataset, Dirac
j=Job()
app = Im3ShapeApp(
im3_location=DiracFile(lfn='/lsst/y1a1-v2-z/software/2016-02-24/im3shape-grid.tar.gz'),
ini_location=LocalFile('/afs/cern.ch/user/r/rcurrie/cmtuser/GANGA/GANGA_LSST/install/ganga/python/params_disc.ini'),
blacklist=LocalFile('/afs/cern.ch/user/r/rcurrie/cmtuser/GANGA/GANGA_LSST/install/ganga/python/blacklist-y1.txt')
)
j.application = app
j.backend=Dirac()
mydata=GangaDataset()
mydata.append(DiracFile(lfn='/lsst/DES0005+0043-z-meds-y1a1-gamma.fits.fz'))
j.inputdata = mydata
j.splitter=Im3ShapeSplitter(size=20)
j.outputfiles = [DiracFile('*.main.txt'), DiracFile('*.epoch.txt')]
def test_b_test_get(self):
"""Test that the files were made accessible to the WN area and collected as LocalFile objects in outputfiles"""
from Ganga.GPI import Job
tmpdir = tempfile.mkdtemp()
# Test in the case that the files don't have a parent or a localDir
for file_ in self._managed_files:
file_.localDir = ''
try:
assert file_.localDir == ''
file_.get()
print("Unexpected localDir: %s" % file_.localDir)
failed = False
except GangaException:
failed = True
assert failed
# Test in the case that the localDir has been set
for file_ in self._managed_files:
file_.localDir = tmpdir
print("localDir: %s" % file_.localDir)
file_.get()
assert os.path.isfile(os.path.join(tmpdir, file_.namePattern))
file_.localDir = ''
assert file_.localDir == ''
# Test in the case that the object is 'owned' by a Job
j = Job()
outputdir = stripProxy(j).getOutputWorkspace(create=True).getPath()
j.outputfiles = self._managed_files
for file_ in j.outputfiles:
assert stripProxy(file_).getJobObject() is stripProxy(j)
assert file_.localDir == ''
file_.get()
assert os.path.isfile(os.path.join(outputdir, file_.namePattern))
shutil.rmtree(tmpdir, ignore_errors=True)
self.cleanUp()
def test_b_test_get(self):
"""Test that the files were made accessible to the WN area and collected as LocalFile objects in outputfiles"""
from Ganga.GPI import Job
tmpdir = tempfile.mkdtemp()
# Test in the case that the files don't have a parent or a localDir
for file_ in self._managed_files:
file_.localDir = ''
try:
assert file_.localDir == ''
file_.get()
print("Unexpected localDir: %s" % file_.localDir)
failed = False
except GangaException:
failed = True
assert failed
# Test in the case that the localDir has been set
for file_ in self._managed_files:
file_.localDir = tmpdir
print("localDir: %s" % file_.localDir)
file_.get()
assert os.path.isfile(os.path.join(tmpdir, file_.namePattern))
file_.localDir = ''
assert file_.localDir == ''
# Test in the case that the object is 'owned' by a Job
j=Job()
outputdir = stripProxy(j).getOutputWorkspace(create=True).getPath()
j.outputfiles = self._managed_files
for file_ in j.outputfiles:
assert stripProxy(file_).getJobObject() is stripProxy(j)
assert file_.localDir == ''
file_.get()
assert os.path.isfile(os.path.join(outputdir, file_.namePattern))
shutil.rmtree(tmpdir, ignore_errors=True)
self.cleanUp()
def testDatasetsFunctions(self):
from Ganga.GPI import DiracFile, PhysicalFile, LHCbDataset, Job, LocalFile
# test constructors/setters
ds = LHCbDataset(['lfn:a', 'pfn:b'])
assert len(ds) == 2
print(ds[0])
assert isinstance(ds[0], DiracFile)
assert isinstance(ds[1], LocalFile)
ds = LHCbDataset()
ds.files = ['lfn:a', 'pfn:b']
assert isinstance(ds[0], DiracFile)
assert isinstance(ds[1], LocalFile)
assert ds.getFullFileNames() == ['LFN:a', 'PFN:b']
ds.files.append('lfn:c')
assert isinstance(ds[-1], DiracFile)
# check job assignments
j = Job()
j.inputdata = ['lfn:a', 'pfn:b']
assert isinstance(j.inputdata, LHCbDataset)
j.outputfiles = ['a', DiracFile('b')]
assert isinstance(j.outputfiles[0], LocalFile)
print(type(j.outputfiles[1]))
assert isinstance(j.outputfiles[1], DiracFile)
# check the LHCbDataset functions:
assert ds.getLFNs() == ['a', 'c']
assert ds.getPFNs() == ['b']
ds2 = LHCbDataset(['lfn:a', 'lfn:d'])
ds.extend(ds2, True)
assert len(ds) == 4
# check the useful difference functions etc
assert sorted(ds.difference(ds2).getFileNames()) == ['b', 'c']
assert sorted(ds.symmetricDifference(ds2).getFileNames()) == ['b', 'c']
assert sorted(ds.intersection(ds2).getFileNames()) == ['a', 'd']
assert sorted(ds.union(ds2).getFileNames()) == ['a', 'b', 'c', 'd']
def test_a_testClientInputSubmit(self):
"""Test that a job can be submitted with inputfiles in the input"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
self._managed_files.append(file_1)
self._managed_files.append(file_2)
msf_1 = MassStorageFile(file_1)
msf_2 = MassStorageFile(file_2)
msf_1.put()
msf_2.put()
j = Job()
j.inputfiles = [msf_1, msf_2]
j.splitter = ArgSplitter(args=[[_] for _ in range(self.sj_len)])
j.outputfiles = [LocalFile(namePattern='*' + _ext)]
j.submit()
def test_a_testClientSideSubmit(self):
"""Test the client side code whilst stil using the Local backend"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
TestMassStorageClient.cleanUp()
assert getConfig('Output')[_getName(self.fileClass)]['backendPostprocess']['Local'] == 'client'
file_1 = generate_unique_temp_file(TestMassStorageClient._ext)
file_2 = generate_unique_temp_file(TestMassStorageClient._ext)
TestMassStorageClient._managed_files.append(file_1)
TestMassStorageClient._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(args = [[_] for _ in range(TestMassStorageClient.sj_len)])
j.outputfiles = [MassStorageFile(namePattern='*'+TestMassStorageClient._ext)]
j.submit()
def test_a_testClientInputSubmit(self):
"""Test that a job can be submitted with inputfiles in the input"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
self._managed_files.append(file_1)
self._managed_files.append(file_2)
msf_1 = MassStorageFile(file_1)
msf_2 = MassStorageFile(file_2)
msf_1.put()
msf_2.put()
j = Job()
j.inputfiles = [msf_1, msf_2]
j.splitter = ArgSplitter(args = [[_] for _ in range(self.sj_len)])
j.outputfiles = [LocalFile(namePattern='*'+_ext)]
j.submit()
def test_a_testClientSideSubmit(self):
"""Test the client side code whilst stil using the Local backend"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
TestMassStorageClient.cleanUp()
assert getConfig('Output')[_getName(self.fileClass)]['backendPostprocess']['Local'] == 'client'
file_1 = generate_unique_temp_file(TestMassStorageClient._ext)
file_2 = generate_unique_temp_file(TestMassStorageClient._ext)
TestMassStorageClient._managed_files.append(file_1)
TestMassStorageClient._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(args = [[_] for _ in range(TestMassStorageClient.sj_len)])
j.outputfiles = [MassStorageFile(namePattern='*'+TestMassStorageClient._ext)]
j.submit()
def testMergeRemoval(self):
from Ganga.GPI import Job, Executable, Local, LocalFile, jobs
# see Savannah 33710
j = Job()
jobID = j.id
# job will run for at least 20 seconds
j.application = Executable(exe="sh", args=["-c", "sleep 20; echo foo > out.txt"])
j.backend = Local()
j.outputfiles = [LocalFile("out.txt")]
j.splitter = CopySplitter()
j.postprocessors = MergerTester(files=["out.txt"])
j.postprocessors[0].ignorefailed = True
j.postprocessors[0].alwaysfail = True
j.postprocessors[0].wait = 10
j.submit()
run_until_state(j, state="running")
j.remove()
with pytest.raises(KeyError):
jobs(jobID)
def testDatasets(self):
from Ganga.GPI import DiracFile, PhysicalFile, LHCbDataset, Job, LocalFile
# test constructors/setters
ds = LHCbDataset(['lfn:a', 'pfn:b'])
assert len(ds) == 2
print(ds[0])
assert isinstance(ds[0], DiracFile)
assert isinstance(ds[1], PhysicalFile)
ds = LHCbDataset()
ds.files = ['lfn:a', 'pfn:b']
assert isinstance(ds[0], DiracFile)
assert isinstance(ds[1], PhysicalFile)
ds.files.append('lfn:c')
assert isinstance(ds[-1], DiracFile)
d = OutputData(['a', 'b'])
assert isinstance(d.files[0], str)
assert isinstance(d.files[1], str)
# check job assignments
j = Job()
j.inputdata = ['lfn:a', 'pfn:b']
assert isinstance(j.inputdata, LHCbDataset)
j.outputfiles = ['a', DiracFile('b')]
assert isinstance(j.outputfiles[0], LocalFile)
print(type(j.outputfiles[1]))
assert isinstance(j.outputfiles[1], DiracFile)
LFN_DATA = [
'LFN:/lhcb/LHCb/Collision11/DIMUON.DST/00016768/0000/00016768_00000005_1.dimuon.dst',
'LFN:/lhcb/LHCb/Collision11/DIMUON.DST/00016768/0000/00016768_00000006_1.dimuon.dst'
]
ds = LHCbDataset(LFN_DATA)
assert len(ds.getReplicas().keys()) == 2
assert ds.getCatalog()
def testMergeRemoval(self):
from Ganga.GPI import Job, Executable, Local, LocalFile, jobs
# see Savannah 33710
j = Job()
jobID = j.id
# job will run for at least 20 seconds
j.application = Executable(exe='sh',
args=['-c', 'sleep 20; echo foo > out.txt'])
j.backend = Local()
j.outputfiles = [LocalFile('out.txt')]
j.splitter = CopySplitter()
j.postprocessors = MergerTester(files=['out.txt'])
j.postprocessors[0].ignorefailed = True
j.postprocessors[0].alwaysfail = True
j.postprocessors[0].wait = 10
j.submit()
run_until_state(j, state='running')
j.remove()
with pytest.raises(KeyError):
jobs(jobID)
def test_g_MultipleFiles(self):
"""Test that the wildcards work"""
MassStorageFile = self.fileClass
from Ganga.GPI import LocalFile, Job, ArgSplitter
_ext = '.root'
file_1 = generate_unique_temp_file(_ext)
file_2 = generate_unique_temp_file(_ext)
TestMassStorageWN._managed_files.append(file_1)
TestMassStorageWN._managed_files.append(file_2)
j = Job()
j.inputfiles = [LocalFile(file_1), LocalFile(file_2)]
j.splitter = ArgSplitter(args = [[_] for _ in range(0, TestMassStorageWN.sj_len) ])
j.outputfiles = [MassStorageFile(namePattern='*'+_ext, outputfilenameformat=self.customOutputFormat)]
for f in j.outputfiles:
assert f.outputfilenameformat == self.customOutputFormat
j.submit()
for f in j.outputfiles:
assert f.outputfilenameformat == self.customOutputFormat
def test_f_InputAndOutputData(self):
from Ganga.GPI import Job, File, LocalFile, GangaDataset, Local, plugins
# -- INPUTANDOUTPUTDATA BASIC START
# create a script to send
open('my_script2.sh', 'w').write("""#!/bin/bash
ls -ltr
more "my_input.txt"
echo "TESTING" > my_output.txt
""")
import os
os.system('chmod +x my_script2.sh')
# create a script to send
open('my_input.txt', 'w').write('Input Testing works!')
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [ LocalFile('my_input.txt') ]
j.outputfiles = [ LocalFile('my_output.txt') ]
j.submit()
# -- INPUTANDOUTPUTDATA BASIC STOP
# -- INPUTANDOUTPUTDATA PEEKOUTPUT START
j.peek() # list output dir contents
j.peek('my_output.txt')
# -- INPUTANDOUTPUTDATA PEEKOUTPUT STOP
# -- INPUTANDOUTPUTDATA FAILJOB START
# This job will fail
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [ LocalFile('my_input.txt') ]
j.outputfiles = [ LocalFile('my_output_FAIL.txt') ]
j.submit()
# -- INPUTANDOUTPUTDATA FAILJOB STOP
# -- INPUTANDOUTPUTDATA WILDCARD START
# This job will pick up both 'my_input.txt' and 'my_output.txt'
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [LocalFile('my_input.txt')]
j.outputfiles = [LocalFile('*.txt')]
j.submit()
# -- INPUTANDOUTPUTDATA WILDCARD STOP
# -- INPUTANDOUTPUTDATA OUTPUTFILES START
j.outputfiles
# -- INPUTANDOUTPUTDATA OUTPUTFILES STOP
# -- INPUTANDOUTPUTDATA INPUTDATA START
# Create a test script
open('my_script3.sh', 'w').write("""#!/bin/bash
echo $PATH
ls -ltr
more __GangaInputData.txt__
echo "MY TEST FILE" > output_file.txt
""")
import os
os.system('chmod +x my_script3.sh')
# Submit a job
j = Job()
j.application.exe = File('my_script3.sh')
j.inputdata = GangaDataset(files=[LocalFile('*.sh')])
j.backend = Local()
j.submit()
# -- INPUTANDOUTPUTDATA INPUTDATA STOP
# -- INPUTANDOUTPUTDATA GANGAFILES START
plugins('gangafiles')
def test_f_InputAndOutputData(self):
from Ganga.GPI import Job, File, LocalFile, GangaDataset, Local, plugins
# -- INPUTANDOUTPUTDATA BASIC START
# create a script to send
open('my_script2.sh', 'w').write("""#!/bin/bash
ls -ltr
more "my_input.txt"
echo "TESTING" > my_output.txt
""")
import os
os.system('chmod +x my_script2.sh')
# create a script to send
open('my_input.txt', 'w').write('Input Testing works!')
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [LocalFile('my_input.txt')]
j.outputfiles = [LocalFile('my_output.txt')]
j.submit()
# -- INPUTANDOUTPUTDATA BASIC STOP
# -- INPUTANDOUTPUTDATA PEEKOUTPUT START
j.peek() # list output dir contents
j.peek('my_output.txt')
# -- INPUTANDOUTPUTDATA PEEKOUTPUT STOP
# -- INPUTANDOUTPUTDATA FAILJOB START
# This job will fail
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [LocalFile('my_input.txt')]
j.outputfiles = [LocalFile('my_output_FAIL.txt')]
j.submit()
# -- INPUTANDOUTPUTDATA FAILJOB STOP
# -- INPUTANDOUTPUTDATA WILDCARD START
# This job will pick up both 'my_input.txt' and 'my_output.txt'
j = Job()
j.application.exe = File('my_script2.sh')
j.inputfiles = [LocalFile('my_input.txt')]
j.outputfiles = [LocalFile('*.txt')]
j.submit()
# -- INPUTANDOUTPUTDATA WILDCARD STOP
# -- INPUTANDOUTPUTDATA OUTPUTFILES START
j.outputfiles
# -- INPUTANDOUTPUTDATA OUTPUTFILES STOP
# -- INPUTANDOUTPUTDATA INPUTDATA START
# Create a test script
open('my_script3.sh', 'w').write("""#!/bin/bash
echo $PATH
ls -ltr
more __GangaInputData.txt__
echo "MY TEST FILE" > output_file.txt
""")
import os
os.system('chmod +x my_script3.sh')
# Submit a job
j = Job()
j.application.exe = File('my_script3.sh')
j.inputdata = GangaDataset(files=[LocalFile('*.sh')])
j.backend = Local()
j.submit()
# -- INPUTANDOUTPUTDATA INPUTDATA STOP
# -- INPUTANDOUTPUTDATA GANGAFILES START
plugins('gangafiles')
def makeIPResolutionsJob( jobName, dataFile, brunelVersion="", dataType = '2012', extraFiles = [],
ddDBTag = None, condDBTag = None ) :
"""Call this method to make a job that will run Brunel with the IP resolutions
ntupling algorithm, using the given config file (data type, etc) and data file.
Add this method to your ~/.ganga.py file to have it automatically loaded whenever
you run ganga."""
print "Creating an IP resolutions monitoring job named \'%s\'" % jobName
dataFile = os.path.expandvars(dataFile)
if not os.path.exists(dataFile) :
print "Could not find the data file \"%s\"!" % dataFile
return None
dataFile = os.path.abspath(dataFile)
print "Using data file \'%s\'" % dataFile
print "Parsing data file for options."
stepInfos = parseDataFileForSteps(dataFile)
productionOptsFiles = None
if len(stepInfos) > 0 :
for step in stepInfos :
# Get any additional options files used for the Brunel step.
if step['ApplicationName'] == "Brunel" :
productionOptsFiles = []
for optsFile in step['OptionFiles'].split(";") :
productionOptsFiles.append(optsFile.replace(" \n",""))
# Get the DB tags used for the Gauss step. This should be the
# same as for the Brunel step but it seems there's a bug in bkk.
if step['ApplicationName'] == "Gauss" or step['ApplicationName'] == 'DaVinci' or step['ApplicationName'] == 'Brunel' :
if ddDBTag == None :
ddDBTag = step['DDDB']
if condDBTag == None :
condDBTag = step['CONDDB']
if None == ddDBTag and len(extraFiles) == 0 :
print "The DB tags could not be retrieved from the data file and no extra options\
files have been specified! The job cannot be made."
return None
j = Job( name = jobName )
if brunelVersion == "" :
j.application = Brunel()
else :
j.application = Brunel(version=brunelVersion)
j.application.extraopts = 'Brunel().OutputLevel = 5\nBrunel().PrintFreq = 10000\nBrunel().DataType = "{0}"\n'.format(dataType)
if None != ddDBTag :
print "Using DDDBTag \"%s\" and CondDBTag \"%s\"" % (ddDBTag, condDBTag)
j.application.extraopts += "\nBrunel().DDDBtag = \"%s\"\nBrunel().CondDBtag = \"%s\"\n" % (ddDBTag, condDBTag)
if 'sim' in ddDBTag.lower() :
j.application.extraopts += '\nBrunel().Simulation = True\n'
#j.application.extraopts += '\nBrunel().WithMC = True\n'
else :
print "The DB tags could not be retrieved from the data file."
print "If they are not defined in one of the extra options files default values will be used."
j.application.optsfile = [ mainIPConfigFile ] + extraFiles
print "Using options files:"
for optsFile in j.application.optsfile :
print optsFile.name
if None != productionOptsFiles and len(productionOptsFiles) > 0 :
for optsFile in productionOptsFiles :
print optsFile
j.application.extraopts += "\nimportOptions(\"%s\")\n" % optsFile
print "Reading in data ..."
j.inputdata = j.application.readInputData( dataFile )
print "Data read. %s files found." % len(j.inputdata.files)
if len(j.inputdata.files) > 0 :
j.application.extraopts += '\nBrunel().InputType = "{0}"\n'.format(j.inputdata.files[0].name.split('.')[-1].upper())
j.splitter = SplitByFiles( filesPerJob = 10 )
j.backend = Dirac()
j.outputfiles = [DiracFile('*.root')]
return j
))
j.comment = (
'{1} {2} MC {0} ntuple creation for k3pi mixing measurement.'
.format(event_type, year, polarity)
)
j.application = DaVinci(version='v41r3')
j.application.optsfile = [s.format(path=base, year=year) for s in OPTIONS]
if args.test:
# If testing, run over a couple of files locally,
# saving the results to the sandbox
j.inputdata = dataset[0:1]
j.backend = Local()
# Prepend test string to job name
j.name = 'TEST_{0}'.format(j.name)
j.outputfiles = [LocalFile(tfn)]
else:
# If not testing, run over everything on the grid, splitting jobs
# into groups of 10 files, notifying me on job completion/subjob failure,
# and save the results on the grid storage
j.inputdata = dataset
j.backend = Dirac()
j.backend.settings['CPUTime'] = 60*60*24*7
j.do_auto_resubmit = True
j.splitter = SplitByFiles(filesPerJob=5, ignoremissing=True)
j.postprocessors = [Notifier(address=email)]
j.outputfiles = [DiracFile(tfn)]
if not args.inspect_job:
queues.add(j.submit) # noqa
