コード例 #1
0
 def setUp(self):
     j = Job(application=DaVinci())
     j.prepare()
     from Ganga.Utility.Config import getConfig
     if getConfig('Output')['ForbidLegacyInput']:
         j.inputfiles = [LocalFile(name='dummy.in')]
     else:
         j.inputsandbox = [File(name='dummy.in')]
     self.app = j.application._impl
     #self.extra = GaudiExtras()
     # self.extra.master_input_buffers['master.buffer'] = '###MASTERBUFFER###'
     #self.extra.master_input_files = [File(name='master.in')]
     # self.extra.input_buffers['subjob.buffer'] = '###SUBJOBBUFFER###'
     self.input_files = [
         File(name='subjob.in'),
         File(
             FileBuffer('subjob.buffer',
                        '###SUBJOBBUFFER###').create().name)
     ]
     self.appmasterconfig = StandardJobConfig(inputbox=[
         File(name='master.in'),
         File(
             FileBuffer('master.buffer',
                        '###MASTERBUFFER###').create().name)
     ])
     j.outputfiles = ['dummy1.out', 'dummy2.out', 'dummy3.out']
     self.rth = LHCbGaudiRunTimeHandler()
コード例 #2
0
class TestGaudiRunTimeHandler(GangaGPITestCase):
    def setUp(self):
        j = Job(application=DaVinci())
        j.prepare()
        from Ganga.Utility.Config import getConfig
        if getConfig('Output')['ForbidLegacyInput']:
            j.inputfiles = [LocalFile(name='dummy.in')]
        else:
            j.inputsandbox = [File(name='dummy.in')]
        self.app = j.application._impl
        #self.extra = GaudiExtras()
        # self.extra.master_input_buffers['master.buffer'] = '###MASTERBUFFER###'
        #self.extra.master_input_files = [File(name='master.in')]
        # self.extra.input_buffers['subjob.buffer'] = '###SUBJOBBUFFER###'
        self.input_files = [
            File(name='subjob.in'),
            File(
                FileBuffer('subjob.buffer',
                           '###SUBJOBBUFFER###').create().name)
        ]
        self.appmasterconfig = StandardJobConfig(inputbox=[
            File(name='master.in'),
            File(
                FileBuffer('master.buffer',
                           '###MASTERBUFFER###').create().name)
        ])
        j.outputfiles = ['dummy1.out', 'dummy2.out', 'dummy3.out']
        self.rth = LHCbGaudiRunTimeHandler()

    def test_GaudiRunTimeHandler_master_prepare(self):
        stdjobconfig = self.rth.master_prepare(self.app, self.appmasterconfig)
        # should have master.buffer, master.in and options.pkl and dummy.in
        print("sandbox =", stdjobconfig.getSandboxFiles())
        print("sandbox =",
              [file.name for file in stdjobconfig.getSandboxFiles()])
        assert len(stdjobconfig.getSandboxFiles()) == 3

    def test_GaudiRunTimeHandler_prepare(self):
        sjc = StandardJobConfig(
            inputbox=self.input_files,
            outputbox=['dummy1.out', 'dummy2.out', 'dummy3.out'])
        jobmasterconfig = StandardJobConfig()
        jobmasterconfig.outputdata = []
        stdjobconfig = self.rth.prepare(self.app, sjc, self.appmasterconfig,
                                        jobmasterconfig)
        # should have subjob.in(buffer), data.opts and gaudiscript.py
        print("sandbox =", stdjobconfig.getSandboxFiles())
        print("sandbox =",
              [file.name for file in stdjobconfig.getSandboxFiles()])
        assert len(stdjobconfig.getSandboxFiles()) == 4, 'inputsandbox error'
        l = len(stdjobconfig.getOutputSandboxFiles())
        print("outputsandbox =", stdjobconfig.getOutputSandboxFiles())
        assert l == 4, 'outputsandbox error'
コード例 #3
0
class TestGaudiRunTimeHandler(GangaGPITestCase):

    def setUp(self):
        j = Job(application=DaVinci())
        j.prepare()
        from Ganga.Utility.Config import getConfig
        if getConfig('Output')['ForbidLegacyInput']:
            j.inputfiles = [LocalFile(name='dummy.in')]
        else:
            j.inputsandbox = [File(name='dummy.in')]
        self.app = j.application._impl
        #self.extra = GaudiExtras()
        # self.extra.master_input_buffers['master.buffer'] = '###MASTERBUFFER###'
        #self.extra.master_input_files = [File(name='master.in')]
        # self.extra.input_buffers['subjob.buffer'] = '###SUBJOBBUFFER###'
        self.input_files = [File(name='subjob.in'), File(
            FileBuffer('subjob.buffer', '###SUBJOBBUFFER###').create().name)]
        self.appmasterconfig = StandardJobConfig(inputbox=[File(name='master.in'), File(
            FileBuffer('master.buffer', '###MASTERBUFFER###').create().name)])
        j.outputfiles = ['dummy1.out', 'dummy2.out', 'dummy3.out']
        self.rth = LHCbGaudiRunTimeHandler()

    def test_GaudiRunTimeHandler_master_prepare(self):
        stdjobconfig = self.rth.master_prepare(self.app, self.appmasterconfig)
        # should have master.buffer, master.in and options.pkl and dummy.in
        print("sandbox =", stdjobconfig.getSandboxFiles())
        print(
            "sandbox =", [file.name for file in stdjobconfig.getSandboxFiles()])
        assert len(stdjobconfig.getSandboxFiles()) == 3

    def test_GaudiRunTimeHandler_prepare(self):
        sjc = StandardJobConfig(
            inputbox=self.input_files, outputbox=['dummy1.out', 'dummy2.out', 'dummy3.out'])
        jobmasterconfig = StandardJobConfig()
        jobmasterconfig.outputdata = []
        stdjobconfig = self.rth.prepare(
            self.app, sjc, self.appmasterconfig, jobmasterconfig)
        # should have subjob.in(buffer), data.opts and gaudiscript.py
        print("sandbox =", stdjobconfig.getSandboxFiles())
        print(
            "sandbox =", [file.name for file in stdjobconfig.getSandboxFiles()])
        assert len(stdjobconfig.getSandboxFiles()) == 4, 'inputsandbox error'
        l = len(stdjobconfig.getOutputSandboxFiles())
        print("outputsandbox =", stdjobconfig.getOutputSandboxFiles())
        assert l == 4, 'outputsandbox error'
コード例 #4
0
 def setUp(self):
     j = Job(application=DaVinci())
     j.prepare()
     from Ganga.Utility.Config import getConfig
     if getConfig('Output')['ForbidLegacyInput']:
         j.inputfiles = [LocalFile(name='dummy.in')]
     else:
         j.inputsandbox = [File(name='dummy.in')]
     self.app = j.application._impl
     #self.extra = GaudiExtras()
     # self.extra.master_input_buffers['master.buffer'] = '###MASTERBUFFER###'
     #self.extra.master_input_files = [File(name='master.in')]
     # self.extra.input_buffers['subjob.buffer'] = '###SUBJOBBUFFER###'
     self.input_files = [File(name='subjob.in'), File(
         FileBuffer('subjob.buffer', '###SUBJOBBUFFER###').create().name)]
     self.appmasterconfig = StandardJobConfig(inputbox=[File(name='master.in'), File(
         FileBuffer('master.buffer', '###MASTERBUFFER###').create().name)])
     j.outputfiles = ['dummy1.out', 'dummy2.out', 'dummy3.out']
     self.rth = LHCbGaudiRunTimeHandler()
コード例 #5
0
ファイル: TestSplitters.py プロジェクト: slangrock/ganga
    def test_SplitByFiles_split(self):
        job = Job(application=DaVinci())
        #job.application.extra = GaudiExtras()
        job.splitter = GPI.SplitByFiles(filesPerJob=2)
        dummy_files = ['pfn:f1.dst', 'pfn:f2.dst', 'pfn:f3.dst', 'pfn:f4.dst',
                       'pfn:f5.dst']
        job.inputdata = LHCbDataset(dummy_files)
        job.prepare()
        subjobs = job.splitter.split(job)
        assert len(subjobs) == 3, 'incorrect number of split jobs'
        # for i in range(0,3):
        jobconfigs = [LHCbGaudiRunTimeHandler().prepare(
            subjobs[i].application, None, None, None) for i in range(0, 3)]
        # job 0
        dataopts = [file for file in jobconfigs[
            0].inputbox if file.name.find('data.py') >= 0][0].getContents()
        #dataopts = subjobs[0].application.extra.input_buffers['data.py']
        ok = dataopts.rfind('f1.dst') >= 0 and dataopts.rfind('f2.dst') >= 0 \
            and len(subjobs[0].inputdata) == 2
        print(len(subjobs[0].inputdata))
        assert ok, 'problem w/ subjob 0 input data'
        # job 1
        dataopts = [file for file in jobconfigs[
            1].inputbox if file.name.find('data.py') >= 0][0].getContents()
        #dataopts = subjobs[1].application.extra.input_buffers['data.py']
        ok = dataopts.rfind('f3.dst') >= 0 and dataopts.rfind('f4.dst') >= 0 \
            and len(subjobs[1].inputdata) == 2
        assert ok, 'problem w/ subjob 1 input data'
        # job 2
        dataopts = [file for file in jobconfigs[
            2].inputbox if file.name.find('data.py') >= 0][0].getContents()
       #dataopts = subjobs[2].application.extra.input_buffers['data.py']
        ok = dataopts.rfind('f5.dst') >= 0 and \
            len(subjobs[2].inputdata) == 1
        assert ok, 'problem w/ subjob 2 input data'

        # Check also that data in the optsfiles was picked up.
        job = Job(application=DaVinci())
        #job.application.extra = GaudiExtras()
        job.splitter = GPI.SplitByFiles(filesPerJob=2)
        dummy_files = ['pfn:f1.dst', 'pfn:f2.dst', 'pfn:f3.dst', 'pfn:f4.dst',
                       'pfn:f5.dst']
        l = LHCbDataset(dummy_files)
        tdir = mkdtemp()
        f = open(os.path.join(tdir, 'data.py'), 'w')
        f.write(l.optionsString())
        f.close()
        job.application.optsfile = [f.name]
        job.prepare()
        subjobs = job.splitter.split(job)
        assert len(
            subjobs) == 3, 'incorrect number of split jobs, for data in optsfile'
        # for i in range(0,3):
        jobconfigs = [LHCbGaudiRunTimeHandler().prepare(
            subjobs[i].application, None, None, None) for i in range(0, 3)]
        # job 0
        dataopts = [file for file in jobconfigs[
            0].inputbox if file.name.find('data.py') >= 0][0].getContents()
        #dataopts = subjobs[0].application.extra.input_buffers['data.py']
        ok = dataopts.rfind('f1.dst') >= 0 and dataopts.rfind('f2.dst') >= 0 \
            and len(subjobs[0].inputdata) == 2
        print(len(subjobs[0].inputdata))
        assert ok, 'problem w/ subjob 0 input data, for data in optsfile'
        # job 1
        dataopts = [file for file in jobconfigs[
            1].inputbox if file.name.find('data.py') >= 0][0].getContents()
        #dataopts = subjobs[1].application.extra.input_buffers['data.py']
        ok = dataopts.rfind('f3.dst') >= 0 and dataopts.rfind('f4.dst') >= 0 \
            and len(subjobs[1].inputdata) == 2
        assert ok, 'problem w/ subjob 1 input data, for data in optsfile'
        # job 2
        dataopts = [file for file in jobconfigs[
            2].inputbox if file.name.find('data.py') >= 0][0].getContents()
       #dataopts = subjobs[2].application.extra.input_buffers['data.py']
        ok = dataopts.rfind('f5.dst') >= 0 and \
            len(subjobs[2].inputdata) == 1
        assert ok, 'problem w/ subjob 2 input data, for data in optsfile'