def testFailJobOnMerge(self):
from Ganga.GPI import CustomMerger
self.runJobSlice()
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
file_name = os.path.join(tmpdir, 'merge.py')
with open(file_name, 'w') as module_file:
module_file.write("""def mergefiles(file_list, output_file):
'''Free script for merging files'''
return False
""")
cm = CustomMerger(module=file_name)
cm.files = ['out.txt', 'out2.txt']
with pytest.raises(PostProcessException):
cm.merge(self.jobslice, tmpdir)
j = self.jobslice[0].copy()
j.splitter = CopySplitter()
j.postprocessors = cm
j.submit()
run_until_state(j, state='failed')
assert j.status == 'failed'
def testSimpleCustomMerge(self):
from Ganga.GPI import CustomMerger
self.runJobSlice()
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
file_name = os.path.join(tmpdir, 'merge.py')
with open(file_name, 'w') as module_file:
module_file.write("""from __future__ import print_function
def mergefiles(file_list, output_file):
'''Free script for merging files'''
with file(output_file,'w') as out:
for f in file_list:
print(f, file=out)
return True
""")
cm = CustomMerger(module=file_name)
cm.files = ['out.txt', 'out2.txt']
assert cm.merge(self.jobslice, tmpdir), 'Merge should complete'
assert os.path.exists(os.path.join(tmpdir, 'out.txt')), 'out.txt must exist'
assert os.path.exists(os.path.join(tmpdir, 'out2.txt')), 'out2.txt must exist'
def testFailJobOnMerge(self):
from Ganga.GPI import CustomMerger
self.runJobSlice()
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
file_name = os.path.join(tmpdir, 'merge.py')
with open(file_name, 'w') as module_file:
module_file.write("""def mergefiles(file_list, output_file):
'''Free script for merging files'''
return False
""")
cm = CustomMerger(module=file_name)
cm.files = ['out.txt', 'out2.txt']
with pytest.raises(PostProcessException):
cm.merge(self.jobslice, tmpdir)
j = self.jobslice[0].copy()
j.splitter = CopySplitter()
j.postprocessors = cm
j.submit()
run_until_state(j, state='failed')
assert j.status == 'failed'
def testSimpleCustomMerge(self):
from Ganga.GPI import CustomMerger
self.runJobSlice()
tmpdir = tempfile.mktemp()
os.mkdir(tmpdir)
file_name = os.path.join(tmpdir, 'merge.py')
with open(file_name, 'w') as module_file:
module_file.write("""from __future__ import print_function
def mergefiles(file_list, output_file):
'''Free script for merging files'''
with file(output_file,'w') as out:
for f in file_list:
print(f, file=out)
return True
""")
cm = CustomMerger(module=file_name)
cm.files = ['out.txt', 'out2.txt']
assert cm.merge(self.jobslice, tmpdir), 'Merge should complete'
assert os.path.exists(os.path.join(tmpdir,
'out.txt')), 'out.txt must exist'
assert os.path.exists(os.path.join(tmpdir,
'out2.txt')), 'out2.txt must exist'
def test_h_PostProcessors(self):
from Ganga.GPI import Job, RootMerger, TextMerger, CustomMerger, SmartMerger, RootFileChecker, FileChecker, \
Notifier, CustomChecker
j = Job()
# -- POSTPROCESSORS APPEND START
j.postprocessors.append(
RootMerger(files=['thesis_data.root'],
ignorefailed=True,
overwrite=True))
# -- POSTPROCESSORS APPEND STOP
# -- POSTPROCESSORS TEXTMERGER START
TextMerger(compress=True)
# -- POSTPROCESSORS TEXTMERGER STOP
# -- POSTPROCESSORS ROOTMERGER START
RootMerger(args='-T')
# -- POSTPROCESSORS ROOTMERGER STOP
# -- POSTPROCESSORS CUSTOMMERGER START
CustomMerger().module = '~/mymerger.py'
# -- POSTPROCESSORS CUSTOMMERGER STOP
# -- POSTPROCESSORS SMARTMERGER START
SmartMerger(files=['thesis_data.root', 'stdout'], overwrite=True)
# -- POSTPROCESSORS SMARTMERGER STOP
# -- POSTPROCESSORS SMARTMERGERAPPEND START
j.postprocessors.append(
SmartMerger(files=['thesis_data.root', 'stdout'], overwrite=True))
# -- POSTPROCESSORS SMARTMERGERAPPEND STOP
# -- POSTPROCESSORS SMARTMERGERAPPEND2 START
j.postprocessors.append(TextMerger(files=['stdout'], overwrite=True))
j.postprocessors.append(
RootMerger(files=['thesis_data.root'], overwrite=False))
# -- POSTPROCESSORS SMARTMERGERAPPEND2 STOP
# -- POSTPROCESSORS FILECHECKER START
fc = FileChecker(files=['stdout'], searchStrings=['Segmentation'])
# -- POSTPROCESSORS FILECHECKER STOP
# -- POSTPROCESSORS FILECHECKERMUSTEXIST START
fc.filesMustExist = True
# -- POSTPROCESSORS FILECHECKERMUSTEXIST STOP
# -- POSTPROCESSORS FILECHECKEROPTS START
fc.searchStrings = ['SUCCESS']
fc.failIfFound = False
# -- POSTPROCESSORS FILECHECKEROPTS STOP
# -- POSTPROCESSORS FILECHECKEROPTS START
rfc = RootFileChecker(files=["*.root"])
rfc.files = ["*.root"]
j.postprocessors.append(rfc)
# -- POSTPROCESSORS FILECHECKEROPTS STOP
# -- POSTPROCESSORS CUSTOMCHECKER START
cc = CustomChecker(module='~/mychecker.py')
# -- POSTPROCESSORS CUSTOMCHECKER STOP
# -- POSTPROCESSORS NOTIFIER START
n = Notifier(address='myaddress.cern.ch')
# -- POSTPROCESSORS NOTIFIER STOP
# -- POSTPROCESSORS NOTIFIEROPTS START
n.verbose = True
# -- POSTPROCESSORS NOTIFIEROPTS STOP
# -- POSTPROCESSORS MULTIPLE START
tm = TextMerger(files=['stdout'], compress=True)
rm = RootMerger(files=['thesis_data.root'], args='-f6')
fc = FileChecker(files=['stdout'], searchStrings=['Segmentation'])
cc = CustomChecker(module='~/mychecker.py')
n = Notifier(address='myadress.cern.ch')
j.postprocessors = [tm, rm, fc, cc, n]
# -- POSTPROCESSORS MULTIPLE STOP
# -- POSTPROCESSORS MULTIPLE2 START
j.postprocessors.append(fc)
j.postprocessors.append(tm)
j.postprocessors.append(rm)
j.postprocessors.append(cc)
j.postprocessors.append(n)
# -- POSTPROCESSORS MULTIPLE2 STOP
j.postprocessors.remove(FileChecker())