def setUp(self): j = Job(application=DaVinci()) j.prepare() from Ganga.Utility.Config import getConfig if getConfig('Output')['ForbidLegacyInput']: j.inputfiles = [LocalFile(name='dummy.in')] else: j.inputsandbox = [File(name='dummy.in')] self.app = j.application._impl #self.extra = GaudiExtras() # self.extra.master_input_buffers['master.buffer'] = '###MASTERBUFFER###' #self.extra.master_input_files = [File(name='master.in')] # self.extra.input_buffers['subjob.buffer'] = '###SUBJOBBUFFER###' self.input_files = [ File(name='subjob.in'), File( FileBuffer('subjob.buffer', '###SUBJOBBUFFER###').create().name) ] self.appmasterconfig = StandardJobConfig(inputbox=[ File(name='master.in'), File( FileBuffer('master.buffer', '###MASTERBUFFER###').create().name) ]) j.outputfiles = ['dummy1.out', 'dummy2.out', 'dummy3.out'] self.rth = LHCbGaudiRunTimeHandler()
class TestGaudiRunTimeHandler(GangaGPITestCase): def setUp(self): j = Job(application=DaVinci()) j.prepare() from Ganga.Utility.Config import getConfig if getConfig('Output')['ForbidLegacyInput']: j.inputfiles = [LocalFile(name='dummy.in')] else: j.inputsandbox = [File(name='dummy.in')] self.app = j.application._impl #self.extra = GaudiExtras() # self.extra.master_input_buffers['master.buffer'] = '###MASTERBUFFER###' #self.extra.master_input_files = [File(name='master.in')] # self.extra.input_buffers['subjob.buffer'] = '###SUBJOBBUFFER###' self.input_files = [ File(name='subjob.in'), File( FileBuffer('subjob.buffer', '###SUBJOBBUFFER###').create().name) ] self.appmasterconfig = StandardJobConfig(inputbox=[ File(name='master.in'), File( FileBuffer('master.buffer', '###MASTERBUFFER###').create().name) ]) j.outputfiles = ['dummy1.out', 'dummy2.out', 'dummy3.out'] self.rth = LHCbGaudiRunTimeHandler() def test_GaudiRunTimeHandler_master_prepare(self): stdjobconfig = self.rth.master_prepare(self.app, self.appmasterconfig) # should have master.buffer, master.in and options.pkl and dummy.in print("sandbox =", stdjobconfig.getSandboxFiles()) print("sandbox =", [file.name for file in stdjobconfig.getSandboxFiles()]) assert len(stdjobconfig.getSandboxFiles()) == 3 def test_GaudiRunTimeHandler_prepare(self): sjc = StandardJobConfig( inputbox=self.input_files, outputbox=['dummy1.out', 'dummy2.out', 'dummy3.out']) jobmasterconfig = StandardJobConfig() jobmasterconfig.outputdata = [] stdjobconfig = self.rth.prepare(self.app, sjc, self.appmasterconfig, jobmasterconfig) # should have subjob.in(buffer), data.opts and gaudiscript.py print("sandbox =", stdjobconfig.getSandboxFiles()) print("sandbox =", [file.name for file in stdjobconfig.getSandboxFiles()]) assert len(stdjobconfig.getSandboxFiles()) == 4, 'inputsandbox error' l = len(stdjobconfig.getOutputSandboxFiles()) print("outputsandbox =", stdjobconfig.getOutputSandboxFiles()) assert l == 4, 'outputsandbox error'
class TestGaudiRunTimeHandler(GangaGPITestCase): def setUp(self): j = Job(application=DaVinci()) j.prepare() from Ganga.Utility.Config import getConfig if getConfig('Output')['ForbidLegacyInput']: j.inputfiles = [LocalFile(name='dummy.in')] else: j.inputsandbox = [File(name='dummy.in')] self.app = j.application._impl #self.extra = GaudiExtras() # self.extra.master_input_buffers['master.buffer'] = '###MASTERBUFFER###' #self.extra.master_input_files = [File(name='master.in')] # self.extra.input_buffers['subjob.buffer'] = '###SUBJOBBUFFER###' self.input_files = [File(name='subjob.in'), File( FileBuffer('subjob.buffer', '###SUBJOBBUFFER###').create().name)] self.appmasterconfig = StandardJobConfig(inputbox=[File(name='master.in'), File( FileBuffer('master.buffer', '###MASTERBUFFER###').create().name)]) j.outputfiles = ['dummy1.out', 'dummy2.out', 'dummy3.out'] self.rth = LHCbGaudiRunTimeHandler() def test_GaudiRunTimeHandler_master_prepare(self): stdjobconfig = self.rth.master_prepare(self.app, self.appmasterconfig) # should have master.buffer, master.in and options.pkl and dummy.in print("sandbox =", stdjobconfig.getSandboxFiles()) print( "sandbox =", [file.name for file in stdjobconfig.getSandboxFiles()]) assert len(stdjobconfig.getSandboxFiles()) == 3 def test_GaudiRunTimeHandler_prepare(self): sjc = StandardJobConfig( inputbox=self.input_files, outputbox=['dummy1.out', 'dummy2.out', 'dummy3.out']) jobmasterconfig = StandardJobConfig() jobmasterconfig.outputdata = [] stdjobconfig = self.rth.prepare( self.app, sjc, self.appmasterconfig, jobmasterconfig) # should have subjob.in(buffer), data.opts and gaudiscript.py print("sandbox =", stdjobconfig.getSandboxFiles()) print( "sandbox =", [file.name for file in stdjobconfig.getSandboxFiles()]) assert len(stdjobconfig.getSandboxFiles()) == 4, 'inputsandbox error' l = len(stdjobconfig.getOutputSandboxFiles()) print("outputsandbox =", stdjobconfig.getOutputSandboxFiles()) assert l == 4, 'outputsandbox error'
def setUp(self): j = Job(application=DaVinci()) j.prepare() from Ganga.Utility.Config import getConfig if getConfig('Output')['ForbidLegacyInput']: j.inputfiles = [LocalFile(name='dummy.in')] else: j.inputsandbox = [File(name='dummy.in')] self.app = j.application._impl #self.extra = GaudiExtras() # self.extra.master_input_buffers['master.buffer'] = '###MASTERBUFFER###' #self.extra.master_input_files = [File(name='master.in')] # self.extra.input_buffers['subjob.buffer'] = '###SUBJOBBUFFER###' self.input_files = [File(name='subjob.in'), File( FileBuffer('subjob.buffer', '###SUBJOBBUFFER###').create().name)] self.appmasterconfig = StandardJobConfig(inputbox=[File(name='master.in'), File( FileBuffer('master.buffer', '###MASTERBUFFER###').create().name)]) j.outputfiles = ['dummy1.out', 'dummy2.out', 'dummy3.out'] self.rth = LHCbGaudiRunTimeHandler()
def test_SplitByFiles_split(self): job = Job(application=DaVinci()) #job.application.extra = GaudiExtras() job.splitter = GPI.SplitByFiles(filesPerJob=2) dummy_files = ['pfn:f1.dst', 'pfn:f2.dst', 'pfn:f3.dst', 'pfn:f4.dst', 'pfn:f5.dst'] job.inputdata = LHCbDataset(dummy_files) job.prepare() subjobs = job.splitter.split(job) assert len(subjobs) == 3, 'incorrect number of split jobs' # for i in range(0,3): jobconfigs = [LHCbGaudiRunTimeHandler().prepare( subjobs[i].application, None, None, None) for i in range(0, 3)] # job 0 dataopts = [file for file in jobconfigs[ 0].inputbox if file.name.find('data.py') >= 0][0].getContents() #dataopts = subjobs[0].application.extra.input_buffers['data.py'] ok = dataopts.rfind('f1.dst') >= 0 and dataopts.rfind('f2.dst') >= 0 \ and len(subjobs[0].inputdata) == 2 print(len(subjobs[0].inputdata)) assert ok, 'problem w/ subjob 0 input data' # job 1 dataopts = [file for file in jobconfigs[ 1].inputbox if file.name.find('data.py') >= 0][0].getContents() #dataopts = subjobs[1].application.extra.input_buffers['data.py'] ok = dataopts.rfind('f3.dst') >= 0 and dataopts.rfind('f4.dst') >= 0 \ and len(subjobs[1].inputdata) == 2 assert ok, 'problem w/ subjob 1 input data' # job 2 dataopts = [file for file in jobconfigs[ 2].inputbox if file.name.find('data.py') >= 0][0].getContents() #dataopts = subjobs[2].application.extra.input_buffers['data.py'] ok = dataopts.rfind('f5.dst') >= 0 and \ len(subjobs[2].inputdata) == 1 assert ok, 'problem w/ subjob 2 input data' # Check also that data in the optsfiles was picked up. job = Job(application=DaVinci()) #job.application.extra = GaudiExtras() job.splitter = GPI.SplitByFiles(filesPerJob=2) dummy_files = ['pfn:f1.dst', 'pfn:f2.dst', 'pfn:f3.dst', 'pfn:f4.dst', 'pfn:f5.dst'] l = LHCbDataset(dummy_files) tdir = mkdtemp() f = open(os.path.join(tdir, 'data.py'), 'w') f.write(l.optionsString()) f.close() job.application.optsfile = [f.name] job.prepare() subjobs = job.splitter.split(job) assert len( subjobs) == 3, 'incorrect number of split jobs, for data in optsfile' # for i in range(0,3): jobconfigs = [LHCbGaudiRunTimeHandler().prepare( subjobs[i].application, None, None, None) for i in range(0, 3)] # job 0 dataopts = [file for file in jobconfigs[ 0].inputbox if file.name.find('data.py') >= 0][0].getContents() #dataopts = subjobs[0].application.extra.input_buffers['data.py'] ok = dataopts.rfind('f1.dst') >= 0 and dataopts.rfind('f2.dst') >= 0 \ and len(subjobs[0].inputdata) == 2 print(len(subjobs[0].inputdata)) assert ok, 'problem w/ subjob 0 input data, for data in optsfile' # job 1 dataopts = [file for file in jobconfigs[ 1].inputbox if file.name.find('data.py') >= 0][0].getContents() #dataopts = subjobs[1].application.extra.input_buffers['data.py'] ok = dataopts.rfind('f3.dst') >= 0 and dataopts.rfind('f4.dst') >= 0 \ and len(subjobs[1].inputdata) == 2 assert ok, 'problem w/ subjob 1 input data, for data in optsfile' # job 2 dataopts = [file for file in jobconfigs[ 2].inputbox if file.name.find('data.py') >= 0][0].getContents() #dataopts = subjobs[2].application.extra.input_buffers['data.py'] ok = dataopts.rfind('f5.dst') >= 0 and \ len(subjobs[2].inputdata) == 1 assert ok, 'problem w/ subjob 2 input data, for data in optsfile'