def setUp(self):
super(TestCustomMerger, self).setUp()
from Ganga.GPI import Job, Executable, Local, File, LocalFile
self.jobslice = []
self.file_name = 'id_echo.sh'
for i in range(2):
j = Job(application=Executable(), backend=Local())
scriptString = '''
#!/bin/sh
echo "Output from job $1." > out.txt
echo "Output from job $2." > out2.txt
'''
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = 'sh'
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile('out.txt'), LocalFile('out2.txt')]
self.jobslice.append(j)
def setUp(self):
for i in range(4):
j = Job(application=Executable(), backend=Local())
scriptString = '''
#!/bin/sh
echo "Output from job $1." > out.txt
echo "Output from job $2." > out2.txt
gzip out.txt
gzip out2.txt
'''
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = 'sh'
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile('out.txt.gz'), LocalFile('out2.txt.gz')]
self.jobslice.append(j)
def setUp(self):
super(TestStructure, self).setUp()
from Ganga.GPI import Job, Executable, Local, File, LocalFile
from GangaTest.Framework.utils import write_file
self.jobslice = []
self.file_name = 'id_echo.sh'
for _ in range(5):
j = Job(application=Executable(), backend=Local())
scriptString = '''
#!/bin/sh
echo "Output from job $1." > out.txt
mkdir -p subdir
echo "Output from job $2." > subdir/out.txt
'''
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = 'sh'
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile('out.txt'), LocalFile('subdir/out.txt')]
self.jobslice.append(j)
def setUp(self):
super(TestSmartMerger, self).setUp()
from Ganga.GPI import Job, Executable, Local, File, LocalFile, config
config['Mergers']['associate'] = {'txt': 'TextMerger'}
self.jobslice = []
self.file_name = 'id_echo.sh'
for i in range(4):
j = Job(application=Executable(), backend=Local())
scriptString = '''
#!/bin/sh
echo "Output from job $1." > out.txt
echo "Output from job $2." > out2.txt
'''
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = 'sh'
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile('out.txt'), LocalFile('out2.txt')]
self.jobslice.append(j)
def setUp(self):
super(TestStructure, self).setUp()
from Ganga.GPI import Job, Executable, Local, File, LocalFile
from GangaTest.Framework.utils import write_file
self.jobslice = []
self.file_name = "id_echo.sh"
for _ in range(5):
j = Job(application=Executable(), backend=Local())
scriptString = """
#!/bin/sh
echo "Output from job $1." > out.txt
mkdir -p subdir
echo "Output from job $2." > subdir/out.txt
"""
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = "sh"
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile("out.txt"), LocalFile("subdir/out.txt")]
self.jobslice.append(j)
def setUp(self):
for i in range(4):
j = Job(application=Executable(), backend=Local())
scriptString = '''
#!/bin/sh
echo "Output from job $1." > out.txt
echo "Output from job $2." > out2.txt
gzip out.txt
gzip out2.txt
'''
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
j.application.exe = 'sh'
j.application.args = [File(fileName), str(j.id), str(j.id * 10)]
j.outputfiles = [LocalFile('out.txt.gz'), LocalFile('out2.txt.gz')]
self.jobslice.append(j)
def testOverWriteProtectionForce(self):
self.runJobSlice()
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
write_file(os.path.join(tmpdir, 'out.txt'), "I shouldn't exist!")
tm = TextMerger()
tm.files = ['out.txt']
assert tm.merge(
self.jobslice, tmpdir, overwrite=True), 'Merge should pass'
def testOverWriteProtection(self):
self.runJobSlice()
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
write_file(os.path.join(tmpdir, 'out.txt'), "I shouldn't exist!")
tm = TextMerger()
tm.files = ['out.txt']
try:
tm.merge(self.jobslice, tmpdir)
except PostProcessException:
pass
else:
assert False, 'Merge should raise exception'
def testScripts(self):
gp = GaudiPython()
#gp.version = configDaVinci['version']
tempDir = tempfile.mkdtemp()
name1 = join(tempDir, 'script1.py')
name2 = join(tempDir, 'script2.py')
write_file(name1, 'print "ABC"\nexecfile("script2.py")\n')
write_file(name2, 'print "DEF"\n')
gp.script = [name1, name2]
j = Job(application=gp, backend=Local())
j.submit()
assert sleep_until_completed(j, 600)
fname = join(j.outputdir, 'stdout')
print('file =', open(fname).read())
assert file_contains(fname, 'ABC'), 'First script file not executed'
assert file_contains(fname, 'DEF'),\
'Inclusion of second script not working'
shutil.rmtree(tempDir)
def setUp(self):
for _ in range(3):
j = Job(application=Root(), backend=Local())
# the following is from the root tutorial
scriptString = """#!/usr/bin/env python
from ROOT import TCanvas, TPad, TFormula, TF1, TPaveLabel, TH1F, TFile
from ROOT import gROOT, gBenchmark
gROOT.Reset()
c1 = TCanvas( 'c1', 'The FillRandom example', 200, 10, 700, 900 )
c1.SetFillColor( 18 )
pad1 = TPad( 'pad1', 'The pad with the function', 0.05, 0.50, 0.95, 0.95, 21 )
pad2 = TPad( 'pad2', 'The pad with the histogram', 0.05, 0.05, 0.95, 0.45, 21 )
pad1.Draw()
pad2.Draw()
pad1.cd()
gBenchmark.Start( 'fillrandom' )
#
# A function (any dimension) or a formula may reference
# an already defined formula
#
form1 = TFormula( 'form1', 'abs(sin(x)/x)' )
sqroot = TF1( 'sqroot', 'x*gaus(0) + [3]*form1', 0, 10 )
sqroot.SetParameters( 10, 4, 1, 20 )
pad1.SetGridx()
pad1.SetGridy()
pad1.GetFrame().SetFillColor( 42 )
pad1.GetFrame().SetBorderMode( -1 )
pad1.GetFrame().SetBorderSize( 5 )
sqroot.SetLineColor( 4 )
sqroot.SetLineWidth( 6 )
sqroot.Draw()
lfunction = TPaveLabel( 5, 39, 9.8, 46, 'The sqroot function' )
lfunction.SetFillColor( 41 )
lfunction.Draw()
c1.Update()
#
# Create a one dimensional histogram (one float per bin)
# and fill it following the distribution in function sqroot.
#
pad2.cd();
pad2.GetFrame().SetFillColor( 42 )
pad2.GetFrame().SetBorderMode( -1 )
pad2.GetFrame().SetBorderSize( 5 )
h1f = TH1F( 'h1f', 'Test random numbers', 200, 0, 10 )
h1f.SetFillColor( 45 )
h1f.FillRandom( 'sqroot', 10000 )
h1f.Draw()
c1.Update()
#
# Open a ROOT file and save the formula, function and histogram
#
myfile = TFile( 'fillrandom.root', 'RECREATE' )
form1.Write()
sqroot.Write()
h1f.Write()
myfile.Close()
gBenchmark.Show( 'fillrandom' )
#make a copy for testing
import shutil
shutil.copyfile('fillrandom.root','fillrandom.foo')
out_log = file('outfile.abc','w')
try:
out_log.write('This is some text output from the job')
finally:
out_log.close()
"""
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
# TODO
j.application.script = File(fileName)
j.outputfiles = [LocalFile(self.output_name)]
self.jobslice.append(j)
def runHistogramEntriesTest(self, file_path, histo_name, expected_entries):
"""Submits a root job that opens the specified histogram and checks that the
number of entries is as expected for the specified histogram."""
scriptString = """#!/usr/bin/env python
from __future__ import print_function
#loads the file specified and fails if the number of elements in the histogram is not as expected
import getopt
import sys
from ROOT import TH1F, TFile
from ROOT import gROOT
if __name__ == '__main__':
#process some options...the b is there for pyroot
opts, args = getopt.getopt(sys.argv[1:],'f:g:e:b',['file=','ghistogram=','entries='])
file = ''
histogram = ''
entries = 0
try:
for o,a in opts:
if o in ('-f','--file'):
file = a
if o in ('-g','--ghistogram'):
histogram = a
if o in ('-e','--entries'):
entries = int(a)
except getopt.error as msg:
print(msg)
sys.exit(2)
#start the test
gROOT.Reset()
myFile = TFile(file, 'READ')
histo = gROOT.FindObject(histogram)
assert histo, 'The histogram was not found.'
assert histo.GetEntries() == entries, 'The number of entries does not match the number specified.'
myFile.Close()
"""
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, 'testforentries.py')
write_file(fileName, scriptString)
# configure a test job
r = Root()
r.args = [
'-f', os.path.basename(file_path), '-g', histo_name, '-e', expected_entries]
r.script = File(fileName)
j = Job(application=r, backend=Local())
from Ganga.Utility.Config import getConfig
if not getConfig('Output')['ForbidLegacyInput']:
j.inputsandbox = [file_path]
else:
j.inputfiles = [LocalFile(file_path)]
j.submit()
sleep_until_completed(j)
return j.status == 'completed'
def setUp(self):
for _ in range(3):
j = Job(application=Root(), backend=Local())
# the following is from the root tutorial
scriptString = """#!/usr/bin/env python
from ROOT import TCanvas, TPad, TFormula, TF1, TPaveLabel, TH1F, TFile
from ROOT import gROOT, gBenchmark
gROOT.Reset()
c1 = TCanvas( 'c1', 'The FillRandom example', 200, 10, 700, 900 )
c1.SetFillColor( 18 )
pad1 = TPad( 'pad1', 'The pad with the function', 0.05, 0.50, 0.95, 0.95, 21 )
pad2 = TPad( 'pad2', 'The pad with the histogram', 0.05, 0.05, 0.95, 0.45, 21 )
pad1.Draw()
pad2.Draw()
pad1.cd()
gBenchmark.Start( 'fillrandom' )
#
# A function (any dimension) or a formula may reference
# an already defined formula
#
form1 = TFormula( 'form1', 'abs(sin(x)/x)' )
sqroot = TF1( 'sqroot', 'x*gaus(0) + [3]*form1', 0, 10 )
sqroot.SetParameters( 10, 4, 1, 20 )
pad1.SetGridx()
pad1.SetGridy()
pad1.GetFrame().SetFillColor( 42 )
pad1.GetFrame().SetBorderMode( -1 )
pad1.GetFrame().SetBorderSize( 5 )
sqroot.SetLineColor( 4 )
sqroot.SetLineWidth( 6 )
sqroot.Draw()
lfunction = TPaveLabel( 5, 39, 9.8, 46, 'The sqroot function' )
lfunction.SetFillColor( 41 )
lfunction.Draw()
c1.Update()
#
# Create a one dimensional histogram (one float per bin)
# and fill it following the distribution in function sqroot.
#
pad2.cd();
pad2.GetFrame().SetFillColor( 42 )
pad2.GetFrame().SetBorderMode( -1 )
pad2.GetFrame().SetBorderSize( 5 )
h1f = TH1F( 'h1f', 'Test random numbers', 200, 0, 10 )
h1f.SetFillColor( 45 )
h1f.FillRandom( 'sqroot', 10000 )
h1f.Draw()
c1.Update()
#
# Open a ROOT file and save the formula, function and histogram
#
myfile = TFile( 'fillrandom.root', 'RECREATE' )
form1.Write()
sqroot.Write()
h1f.Write()
myfile.Close()
gBenchmark.Show( 'fillrandom' )
#make a copy for testing
import shutil
shutil.copyfile('fillrandom.root','fillrandom.foo')
out_log = file('outfile.abc','w')
try:
out_log.write('This is some text output from the job')
finally:
out_log.close()
"""
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, self.file_name)
write_file(fileName, scriptString)
# TODO
j.application.script = File(fileName)
j.outputfiles = [LocalFile(self.output_name)]
self.jobslice.append(j)
def runHistogramEntriesTest(self, file_path, histo_name, expected_entries):
"""Submits a root job that opens the specified histogram and checks that the
number of entries is as expected for the specified histogram."""
scriptString = """#!/usr/bin/env python
from __future__ import print_function
#loads the file specified and fails if the number of elements in the histogram is not as expected
import getopt
import sys
from ROOT import TH1F, TFile
from ROOT import gROOT
if __name__ == '__main__':
#process some options...the b is there for pyroot
opts, args = getopt.getopt(sys.argv[1:],'f:g:e:b',['file=','ghistogram=','entries='])
file = ''
histogram = ''
entries = 0
try:
for o,a in opts:
if o in ('-f','--file'):
file = a
if o in ('-g','--ghistogram'):
histogram = a
if o in ('-e','--entries'):
entries = int(a)
except getopt.error as msg:
print(msg)
sys.exit(2)
#start the test
gROOT.Reset()
myFile = TFile(file, 'READ')
histo = gROOT.FindObject(histogram)
assert histo, 'The histogram was not found.'
assert histo.GetEntries() == entries, 'The number of entries does not match the number specified.'
myFile.Close()
"""
# write string to tmpfile
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
fileName = os.path.join(tmpdir, 'testforentries.py')
write_file(fileName, scriptString)
# configure a test job
r = Root()
r.args = [
'-f',
os.path.basename(file_path), '-g', histo_name, '-e',
expected_entries
]
r.script = File(fileName)
j = Job(application=r, backend=Local())
from Ganga.Utility.Config import getConfig
if not getConfig('Output')['ForbidLegacyInput']:
j.inputsandbox = [file_path]
else:
j.inputfiles = [LocalFile(file_path)]
j.submit()
sleep_until_completed(j)
return j.status == 'completed'
