Esempio n. 1
0
    def testFailJobOnMerge(self):
        from Ganga.GPI import CustomMerger

        self.runJobSlice()
        tmpdir = tempfile.mktemp()
        os.mkdir(tmpdir)

        file_name = os.path.join(tmpdir, 'merge.py')
        with open(file_name, 'w') as module_file:
            module_file.write("""def mergefiles(file_list, output_file):
    '''Free script for merging files'''
    return False
        """)

        cm = CustomMerger(module=file_name)
        cm.files = ['out.txt', 'out2.txt']
        with pytest.raises(PostProcessException):
            cm.merge(self.jobslice, tmpdir)

        j = self.jobslice[0].copy()
        j.splitter = CopySplitter()
        j.postprocessors = cm
        j.submit()

        run_until_state(j, state='failed')
        assert j.status == 'failed'
Esempio n. 2
0
    def testSimpleCustomMerge(self):
        from Ganga.GPI import CustomMerger

        self.runJobSlice()
        tmpdir = tempfile.mktemp()
        os.mkdir(tmpdir)

        file_name = os.path.join(tmpdir, 'merge.py')
        with open(file_name, 'w') as module_file:
            module_file.write("""from __future__ import print_function
def mergefiles(file_list, output_file):
    '''Free script for merging files'''
    
    with file(output_file,'w') as out:
        for f in file_list:
            print(f, file=out)
    
    return True
        """)

        cm = CustomMerger(module=file_name)
        cm.files = ['out.txt', 'out2.txt']
        assert cm.merge(self.jobslice, tmpdir), 'Merge should complete'
        assert os.path.exists(os.path.join(tmpdir, 'out.txt')), 'out.txt must exist'
        assert os.path.exists(os.path.join(tmpdir, 'out2.txt')), 'out2.txt must exist'
Esempio n. 3
0
    def testFailJobOnMerge(self):
        from Ganga.GPI import CustomMerger

        self.runJobSlice()
        tmpdir = tempfile.mktemp()
        os.mkdir(tmpdir)

        file_name = os.path.join(tmpdir, 'merge.py')
        with open(file_name, 'w') as module_file:
            module_file.write("""def mergefiles(file_list, output_file):
    '''Free script for merging files'''
    return False
        """)

        cm = CustomMerger(module=file_name)
        cm.files = ['out.txt', 'out2.txt']
        with pytest.raises(PostProcessException):
            cm.merge(self.jobslice, tmpdir)

        j = self.jobslice[0].copy()
        j.splitter = CopySplitter()
        j.postprocessors = cm
        j.submit()

        run_until_state(j, state='failed')
        assert j.status == 'failed'
Esempio n. 4
0
    def testSimpleCustomMerge(self):
        from Ganga.GPI import CustomMerger

        self.runJobSlice()
        tmpdir = tempfile.mktemp()
        os.mkdir(tmpdir)

        file_name = os.path.join(tmpdir, 'merge.py')
        with open(file_name, 'w') as module_file:
            module_file.write("""from __future__ import print_function
def mergefiles(file_list, output_file):
    '''Free script for merging files'''
    
    with file(output_file,'w') as out:
        for f in file_list:
            print(f, file=out)
    
    return True
        """)

        cm = CustomMerger(module=file_name)
        cm.files = ['out.txt', 'out2.txt']
        assert cm.merge(self.jobslice, tmpdir), 'Merge should complete'
        assert os.path.exists(os.path.join(tmpdir,
                                           'out.txt')), 'out.txt must exist'
        assert os.path.exists(os.path.join(tmpdir,
                                           'out2.txt')), 'out2.txt must exist'
Esempio n. 5
0
    def test_h_PostProcessors(self):

        from Ganga.GPI import Job, RootMerger, TextMerger, CustomMerger, SmartMerger, RootFileChecker, FileChecker, \
            Notifier, CustomChecker

        j = Job()

        # -- POSTPROCESSORS APPEND START
        j.postprocessors.append(
            RootMerger(files=['thesis_data.root'],
                       ignorefailed=True,
                       overwrite=True))
        # -- POSTPROCESSORS APPEND STOP

        # -- POSTPROCESSORS TEXTMERGER START
        TextMerger(compress=True)
        # -- POSTPROCESSORS TEXTMERGER STOP

        # -- POSTPROCESSORS ROOTMERGER START
        RootMerger(args='-T')
        # -- POSTPROCESSORS ROOTMERGER STOP

        # -- POSTPROCESSORS CUSTOMMERGER START
        CustomMerger().module = '~/mymerger.py'
        # -- POSTPROCESSORS CUSTOMMERGER STOP

        # -- POSTPROCESSORS SMARTMERGER START
        SmartMerger(files=['thesis_data.root', 'stdout'], overwrite=True)
        # -- POSTPROCESSORS SMARTMERGER STOP

        # -- POSTPROCESSORS SMARTMERGERAPPEND START
        j.postprocessors.append(
            SmartMerger(files=['thesis_data.root', 'stdout'], overwrite=True))
        # -- POSTPROCESSORS SMARTMERGERAPPEND STOP

        # -- POSTPROCESSORS SMARTMERGERAPPEND2 START
        j.postprocessors.append(TextMerger(files=['stdout'], overwrite=True))
        j.postprocessors.append(
            RootMerger(files=['thesis_data.root'], overwrite=False))
        # -- POSTPROCESSORS SMARTMERGERAPPEND2 STOP

        # -- POSTPROCESSORS FILECHECKER START
        fc = FileChecker(files=['stdout'], searchStrings=['Segmentation'])
        # -- POSTPROCESSORS FILECHECKER STOP

        # -- POSTPROCESSORS FILECHECKERMUSTEXIST START
        fc.filesMustExist = True
        # -- POSTPROCESSORS FILECHECKERMUSTEXIST STOP

        # -- POSTPROCESSORS FILECHECKEROPTS START
        fc.searchStrings = ['SUCCESS']
        fc.failIfFound = False
        # -- POSTPROCESSORS FILECHECKEROPTS STOP

        # -- POSTPROCESSORS FILECHECKEROPTS START
        rfc = RootFileChecker(files=["*.root"])
        rfc.files = ["*.root"]
        j.postprocessors.append(rfc)
        # -- POSTPROCESSORS FILECHECKEROPTS STOP

        # -- POSTPROCESSORS CUSTOMCHECKER START
        cc = CustomChecker(module='~/mychecker.py')
        # -- POSTPROCESSORS CUSTOMCHECKER STOP

        # -- POSTPROCESSORS NOTIFIER START
        n = Notifier(address='myaddress.cern.ch')
        # -- POSTPROCESSORS NOTIFIER STOP

        # -- POSTPROCESSORS NOTIFIEROPTS START
        n.verbose = True
        # -- POSTPROCESSORS NOTIFIEROPTS STOP

        # -- POSTPROCESSORS MULTIPLE START
        tm = TextMerger(files=['stdout'], compress=True)
        rm = RootMerger(files=['thesis_data.root'], args='-f6')
        fc = FileChecker(files=['stdout'], searchStrings=['Segmentation'])
        cc = CustomChecker(module='~/mychecker.py')
        n = Notifier(address='myadress.cern.ch')

        j.postprocessors = [tm, rm, fc, cc, n]
        # -- POSTPROCESSORS MULTIPLE STOP

        # -- POSTPROCESSORS MULTIPLE2 START
        j.postprocessors.append(fc)
        j.postprocessors.append(tm)
        j.postprocessors.append(rm)
        j.postprocessors.append(cc)
        j.postprocessors.append(n)
        # -- POSTPROCESSORS MULTIPLE2 STOP

        j.postprocessors.remove(FileChecker())