def setUp(self): job = Job(application=Gauss()) job.application.platform = 'x86_64-slc6-gcc48-opt' gauss = job.application._impl gauss._getshell() optsfiles = ['./TestGaudi/Gauss-Job.py'] self.extraopts = 'EventSelector(Input=[\"DATAFILE=\'LFN:dummy.dst\' TYP=\'POOL_ROOTTREE\' OPT=\'READ\'\"])' self.parser = PythonOptionsParser(optsfiles, self.extraopts, gauss.env) self.job = job
def readInputData(self, optsfiles, extraopts=False): '''Returns a LHCbDataSet object from a list of options files. The optional argument extraopts will decide if the extraopts string inside the application is considered or not. Usage examples: # Create an LHCbDataset object with the data found in the optionsfile l=DaVinci(version='v22r0p2').readInputData([\"~/cmtuser/\" \ \"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"]) # Get the data from an options file and assign it to the jobs inputdata field j.inputdata = j.application.readInputData([\"~/cmtuser/\" \ \"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"]) # Assuming you have data in your extraopts, you can use the extraopts. # In this case your extraopts need to be fully parseable by gaudirun.py # So you must make sure that you have the proper import statements. # e.g. from Gaudi.Configuration import * # If you mix optionsfiles and extraopts, as usual extraopts may # overwright your options # # Use this to create a new job with data from extraopts of an old job j=Job(inputdata=jobs[-1].application.readInputData([],True)) ''' def dummyfile(): temp_fd, temp_filename = tempfile.mkstemp(text=True, suffix='.py') os.write(temp_fd, "Dummy file to keep the Optionsparser happy") os.close(temp_fd) return temp_filename if type(optsfiles) != type([]): optsfiles = [optsfiles] # use a dummy file to keep the parser happy if len(optsfiles) == 0: optsfiles.append(dummyfile()) if extraopts: extraopts = self.extraopts else: extraopts = "" # parser = check_inputs(optsfiles, extraopts, self.env) try: parser = PythonOptionsParser(optsfiles, extraopts, self.getenv(False)) except Exception as err: msg = 'Unable to parse the job options. Please check options ' \ 'files and extraopts.' logger.error("PythonOptionsParserError:\n%s" % str(err)) raise ApplicationConfigurationError(msg) return GPIProxyObjectFactory(parser.get_input_data())
def readInputData(self, optsfiles, extraopts=False): '''Returns a LHCbDataSet object from a list of options files. The optional argument extraopts will decide if the extraopts string inside the application is considered or not. Usage examples: # Create an LHCbDataset object with the data found in the optionsfile l=DaVinci(version='v22r0p2').readInputData([\"~/cmtuser/\" \ \"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"]) # Get the data from an options file and assign it to the jobs inputdata field j.inputdata = j.application.readInputData([\"~/cmtuser/\" \ \"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"]) # Assuming you have data in your extraopts, you can use the extraopts. # In this case your extraopts need to be fully parseable by gaudirun.py # So you must make sure that you have the proper import statements. # e.g. from Gaudi.Configuration import * # If you mix optionsfiles and extraopts, as usual extraopts may # overwright your options # # Use this to create a new job with data from extraopts of an old job j=Job(inputdata=jobs[-1].application.readInputData([],True)) ''' def dummyfile(): temp_fd, temp_filename = tempfile.mkstemp(text=True, suffix='.py') os.write(temp_fd, "Dummy file to keep the Optionsparser happy") os.close(temp_fd) return temp_filename if type(optsfiles) != type([]): optsfiles = [optsfiles] # use a dummy file to keep the parser happy if len(optsfiles) == 0: optsfiles.append(dummyfile()) if extraopts: extraopts = self.extraopts else: extraopts = "" # parser = check_inputs(optsfiles, extraopts, self.env) try: parser = PythonOptionsParser( optsfiles, extraopts, self.getenv(False)) except Exception as err: msg = 'Unable to parse the job options. Please check options ' \ 'files and extraopts.' logger.error("PythonOptionsParserError:\n%s" % str(err)) raise ApplicationConfigurationError(None, msg) return GPIProxyObjectFactory(parser.get_input_data())
def _get_parser(self): optsfiles = [] for fileitem in self.optsfile: if type(fileitem) is str: optsfiles.append(fileitem) else: optsfiles.append(fileitem.name) #optsfiles = [fileitem.name for fileitem in self.optsfile] # add on XML summary extraopts = '' if self.extraopts: extraopts += self.extraopts if self.extraopts.find('LHCbApp().XMLSummary') is -1: extraopts += "\nfrom Gaudi.Configuration import *" extraopts += "\nfrom Configurables import LHCbApp" extraopts += "\nLHCbApp().XMLSummary='summary.xml'" else: extraopts += "\nfrom Gaudi.Configuration import *" extraopts += "\nfrom Configurables import LHCbApp" extraopts += "\nLHCbApp().XMLSummary='summary.xml'" try: parser = PythonOptionsParser(optsfiles, extraopts, self.getenv(False)) except ApplicationConfigurationError as err: logger.error("PythonOptionsParserError:\n%s" % str(err)) # fix this when preparing not attached to job msg2 = '' try: debug_dir = self.getJobObject().getDebugWorkspace().getPath() msg2 += 'You can also view this from within ganga '\ 'by doing job.peek(\'../debug/gaudirun.<whatever>\').' except Exception, err: logger.debug("path Error:\n%s" % str(err)) debug_dir = tempfile.mkdtemp() messages = err.message.split('###SPLIT###') if len(messages) is 2: stdout = open(debug_dir + '/gaudirun.stdout', 'w') stderr = open(debug_dir + '/gaudirun.stderr', 'w') stdout.write(messages[0]) stderr.write(messages[1]) stdout.close() stderr.close() msg = 'Unable to parse job options! Please check options ' \ 'files and extraopts. The output and error streams from gaudirun.py can be ' \ 'found in %s and %s respectively . ' % (stdout.name, stderr.name) else: f = open(debug_dir + '/gaudirun.out', 'w') f.write(err.message) f.close() msg = 'Unable to parse job options! Please check options ' \ 'files and extraopts. The output from gaudirun.py can be ' \ 'found in %s . ' % f.name msg += msg2 logger.debug(msg) raise ApplicationConfigurationError(msg)
class TestPythonOptionsParser(GangaGPITestCase): def setUp(self): job = Job(application=Gauss()) job.application.platform = 'x86_64-slc6-gcc48-opt' gauss = job.application._impl gauss._getshell() optsfiles = ['./TestGaudi/Gauss-Job.py'] self.extraopts = 'EventSelector(Input=[\"DATAFILE=\'LFN:dummy.dst\' TYP=\'POOL_ROOTTREE\' OPT=\'READ\'\"])' self.parser = PythonOptionsParser(optsfiles, self.extraopts, gauss.env) self.job = job def test_PythonOptionsParser__get_opts_dict_and_pkl_string(self): options, opts_pkl_string = self.parser._get_opts_dict_and_pkl_string() assert options, 'failed to get options' assert opts_pkl_string, 'failed to get pkl string' def test_PythonOptionsParser__join_opts_files(self): joined_str = self.parser._join_opts_files() ok = joined_str.rfind(self.extraopts) assert ok, 'extraopts not added to options' def test_PythonOptionsParser_get_input_data(self): input = self.parser.get_input_data() assert len(input) == 1, 'problem collecting input data files' def test_PythonOptionsParser_get_output_files(self): config = Ganga.Utility.Config.getConfig('LHCb') # start w/ default config.setUserValue( 'outputsandbox_types', ['NTupleSvc', 'HistogramPersistencySvc', 'MicroDSTStream']) sandbox, data = self.parser.get_output_files() ok = sandbox.count('GaussHistos.root') == 1 and \ data.count('Gauss.sim') == 1 assert ok, 'collecting/sorting of output files failed (default)' # move the .sim file config.setUserValue('outputsandbox_types', [ 'NTupleSvc', 'HistogramPersistencySvc', 'MicroDSTStream', 'GaussTape' ]) sandbox, data = self.parser.get_output_files() ok = sandbox.count('GaussHistos.root') == 1 and \ sandbox.count('Gauss.sim') == 1 and \ data.count('Gauss.sim') == 0 assert ok, 'collecting/sorting of output files failed (.sim->sandbox)' # move the .root files to data config.setUserValue('outputsandbox_types', ['GaussTape']) sandbox, data = self.parser.get_output_files() ok = data.count('GaussHistos.root') == 1 and \ sandbox.count('Gauss.sim') == 1 and \ len(data) == 1 and len(sandbox) == 1 assert ok, 'collecting/sorting of output files failed (.root->data)' def test_PythonOptionsParser_get_output(self): config = Ganga.Utility.Config.getConfig('LHCb') # previous method tests the configure stuff...let's just make sure we # can properly override it in a job config.setUserValue( 'outputsandbox_types', ['NTupleSvc', 'HistogramPersistencySvc', 'MicroDSTStream']) j = self.job sandbox, data = self.parser.get_output(j) ok = sandbox.count('GaussHistos.root') == 1 and \ data.count('Gauss.sim') == 1 assert ok, 'collecting/sorting of output files failed (default)' j.outputfiles = [] j.outputfiles = ['Gauss.sim'] sandbox, data = self.parser.get_output(j) ok = sandbox.count('Gauss.sim') == 1 and len(data) == 0 print type(sandbox) assert ok, 'collecting/sorting of output files failed (.sim->sandbox) %s -> %s' % ( str(sandbox.count('Gauss.sim')), str(len(data))) j.outputfiles = [] j.outputfiles = [DiracFile('*.root')] sandbox, data = self.parser.get_output(j) ok = len(sandbox) == 0 and data.count('GaussHistos.root') == 1 assert ok, 'collecting/sorting of output files failed (.root->data)' # make sure if matches both goes to data j.outputfiles = ['*.sim'] j.outputfiles = [DiracFile('*.sim')] sandbox, data = self.parser.get_output(j) ok = data.count('Gauss.sim') == 1 and sandbox.count('Gauss.sim') == 0 assert ok, 'collecting/sorting of output files failed (matches both)'
class TestPythonOptionsParser(GangaGPITestCase): def setUp(self): job = Job(application=Gauss()) job.application.platform = 'x86_64-slc6-gcc48-opt' gauss = job.application._impl gauss._getshell() optsfiles = ['./TestGaudi/Gauss-Job.py'] self.extraopts = 'EventSelector(Input=[\"DATAFILE=\'LFN:dummy.dst\' TYP=\'POOL_ROOTTREE\' OPT=\'READ\'\"])' self.parser = PythonOptionsParser(optsfiles, self.extraopts, gauss.env) self.job = job def test_PythonOptionsParser__get_opts_dict_and_pkl_string(self): options, opts_pkl_string = self.parser._get_opts_dict_and_pkl_string() assert options, 'failed to get options' assert opts_pkl_string, 'failed to get pkl string' def test_PythonOptionsParser__join_opts_files(self): joined_str = self.parser._join_opts_files() ok = joined_str.rfind(self.extraopts) assert ok, 'extraopts not added to options' def test_PythonOptionsParser_get_input_data(self): input = self.parser.get_input_data() assert len(input) == 1, 'problem collecting input data files' def test_PythonOptionsParser_get_output_files(self): config = Ganga.Utility.Config.getConfig('LHCb') # start w/ default config.setUserValue('outputsandbox_types', ['NTupleSvc', 'HistogramPersistencySvc', 'MicroDSTStream']) sandbox, data = self.parser.get_output_files() ok = sandbox.count('GaussHistos.root') == 1 and \ data.count('Gauss.sim') == 1 assert ok, 'collecting/sorting of output files failed (default)' # move the .sim file config.setUserValue('outputsandbox_types', ['NTupleSvc', 'HistogramPersistencySvc', 'MicroDSTStream', 'GaussTape']) sandbox, data = self.parser.get_output_files() ok = sandbox.count('GaussHistos.root') == 1 and \ sandbox.count('Gauss.sim') == 1 and \ data.count('Gauss.sim') == 0 assert ok, 'collecting/sorting of output files failed (.sim->sandbox)' # move the .root files to data config.setUserValue('outputsandbox_types', ['GaussTape']) sandbox, data = self.parser.get_output_files() ok = data.count('GaussHistos.root') == 1 and \ sandbox.count('Gauss.sim') == 1 and \ len(data) == 1 and len(sandbox) == 1 assert ok, 'collecting/sorting of output files failed (.root->data)' def test_PythonOptionsParser_get_output(self): config = Ganga.Utility.Config.getConfig('LHCb') # previous method tests the configure stuff...let's just make sure we # can properly override it in a job config.setUserValue('outputsandbox_types', ['NTupleSvc', 'HistogramPersistencySvc', 'MicroDSTStream']) j = self.job sandbox, data = self.parser.get_output(j) ok = sandbox.count('GaussHistos.root') == 1 and \ data.count('Gauss.sim') == 1 assert ok, 'collecting/sorting of output files failed (default)' j.outputfiles = [] j.outputfiles = ['Gauss.sim'] sandbox, data = self.parser.get_output(j) ok = sandbox.count('Gauss.sim') == 1 and len(data) == 0 print type(sandbox) assert ok, 'collecting/sorting of output files failed (.sim->sandbox) %s -> %s' % ( str(sandbox.count('Gauss.sim')), str(len(data))) j.outputfiles = [] j.outputfiles = [DiracFile('*.root')] sandbox, data = self.parser.get_output(j) ok = len(sandbox) == 0 and data.count('GaussHistos.root') == 1 assert ok, 'collecting/sorting of output files failed (.root->data)' # make sure if matches both goes to data j.outputfiles = ['*.sim'] j.outputfiles = [DiracFile('*.sim')] sandbox, data = self.parser.get_output(j) ok = data.count('Gauss.sim') == 1 and sandbox.count('Gauss.sim') == 0 assert ok, 'collecting/sorting of output files failed (matches both)'