def test_Gauss_Production_PR33857_1(self): """ Using ProdConf (production style). systemConfig = 'x86_64-slc6-gcc48-opt' This is taken from PR 33857 (and would fall back to SetupProject) """ gLogger.always("**** GAUSS v49r5") ra = RunApplication() ra.applicationName = 'Gauss' ra.applicationVersion = 'v49r5' ra.systemConfig = 'x86_64-slc6-gcc48-opt' ra.commandOptions = ['$APPCONFIGOPTS/Gauss/Sim08-Beam4000GeV-mu100-2012-nu2.5.py', '$APPCONFIGOPTS/Gauss/DataType-2012.py', '$APPCONFIGOPTS/Gauss/RICHRandomHits.py', '$APPCONFIGOPTS/Gauss/NoPacking.py', '$DECFILESROOT/options/12877041.py', '$LBPYTHIA8ROOT/options/Pythia8.py', '$APPCONFIGOPTS/Gauss/G4PL_FTFP_BERT_EmNoCuts.py', '$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py'] ra.extraPackages = [('AppConfig', 'v3r277'), ('Gen/DecFiles', 'v29r10'), ('ProdConf', '') ] ra.prodConfFileName = find_all('test_prodConf_gauss_v49r5.py', '..')[0] ra.applicationLog = '00033857_00000001_1_log.txt' ra.stdError = '00033857_00000001_1_err.txt' res = ra.run() self.assertTrue(res['OK']) self.assertEqual(res['Value'], (0, '', ''))
def test_DaVinci_2_Production_PR33857_7(self): """ Using ProdConf (production style). systemConfig = 'x86_64-slc6-gcc49-opt' This is taken from PR 33857 """ gLogger.always("**** DAVINCI v41r3") ra = RunApplication() ra.applicationName = 'DaVinci' ra.applicationVersion = 'v41r3' ra.systemConfig = 'x86_64-slc6-gcc49-opt' ra.commandOptions = ['$APPCONFIGOPTS/Merging/DVMergeDST.py', '$APPCONFIGOPTS/DaVinci/DataType-2012.py', '$APPCONFIGOPTS/Merging/WriteFSR.py', '$APPCONFIGOPTS/Merging/MergeFSR.py'] ra.extraPackages = [('AppConfig', 'v3r305'), ('ProdConf', '') ] ra.step_Number = 1 ra.prodConfFileName = find_all('test_prodConf_davinci_v41r3.py', '..')[0] ra.applicationLog = '00033857_00000007_7_log.txt' ra.stdError = '00033857_00000007_7_err.txt' res = ra.run() self.assertTrue(res['OK']) self.assertEqual(res['Value'], (0, '', ''))
def test_DaVinci_new_gcc62(self): """ Using ProdConf (production style). systemConfig = 'x86_64-slc6-gcc62-opt' This is taken from step 130847 """ gLogger.always("**** DAVINCI v42r2") ra = RunApplication() ra.applicationName = 'DaVinci' ra.applicationVersion = 'v42r2' ra.systemConfig = 'x86_64-slc6-gcc62-opt' ra.commandOptions = ['$APPCONFIGOPTS/DaVinci/DV-Stripping27-Stripping.py', '$APPCONFIGOPTS/DaVinci/DataType-2016.py', '$APPCONFIGOPTS/DaVinci/InputType-RDST.py', '$APPCONFIGOPTS/DaVinci/DV-RawEventJuggler-0_3-to-4_2.py'] ra.extraPackages = [('AppConfig', 'v3r308'), ('Det/SQLDDDB', 'v7r11'), ('ProdConf', '') ] ra.step_Number = 1 ra.prodConfFileName = find_all('test_prodConf_davinci_v42r2.py', '..')[0] ra.applicationLog = '0daVinci_000v42r2_62_log.txt' ra.stdError = '0daVinci_000v42r2_62_err.txt' res = ra.run() self.assertTrue(res['OK']) self.assertEqual(res['Value'], (0, '', ''))
def test_Brunel_Production_PR33857_5(self): """ Using ProdConf (production style). systemConfig = 'x86_64-slc5-gcc46-opt' This is taken from PR 33857 """ gLogger.always("**** BRUNEL v43r2p11") ra = RunApplication() ra.applicationName = 'Brunel' ra.applicationVersion = 'v43r2p11' ra.systemConfig = 'x86_64-slc5-gcc46-opt' ra.commandOptions = ['$APPCONFIGOPTS/Brunel/DataType-2012.py', '$APPCONFIGOPTS/Brunel/MC-WithTruth.py', '$APPCONFIGOPTS/Brunel/Sim09-Run1.py', '$APPCONFIGOPTS/Persistency/DST-multipleTCK-2012.py', '$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py'] ra.extraPackages = [('AppConfig', 'v3r302'), ('ProdConf', '') ] ra.prodConfFileName = find_all('test_prodConf_brunel_v43r2p11.py', '..')[0] ra.applicationLog = '00033857_00000005_5_log.txt' ra.stdError = '00033857_00000005_5_err.txt' res = ra.run() self.assertTrue(res['OK']) self.assertEqual(res['Value'], (0, '', ''))
def test_DaVinci_1_Production_PR33857_6(self): """ Using ProdConf (production style). systemConfig = 'x86_64-slc5-gcc46-opt' This is taken from PR 33857 """ gLogger.always("**** DAVINCI v32r2p1") ra = RunApplication() ra.applicationName = 'DaVinci' ra.applicationVersion = 'v32r2p1' ra.systemConfig = 'x86_64-slc5-gcc46-opt' ra.commandOptions = ['$CHARMCONFIGOPTS/MCFiltering/D02K3PiFromB2DstmunuXStripTrigFiltering_2012.py', '$APPCONFIGOPTS/DaVinci/DataType-2012.py', '$APPCONFIGOPTS/DaVinci/InputType-DST.py', '$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py'] ra.extraPackages = [('AppConfig', 'v3r305'), ('CharmConfig', 'v3r30'), ('ProdConf', '') ] ra.step_Number = 1 ra.prodConfFileName = find_all('test_prodConf_davinci_v32r2p1.py', '..')[0] ra.applicationLog = '00033857_00000006_6_log.txt' ra.stdError = '00033857_00000006_6_err.txt' res = ra.run() self.assertTrue(res['OK']) self.assertEqual(res['Value'], (0, '', ''))
def test_Boole_2_Production_PR33857(self): """ Using ProdConf (production style). Taken from PR 33857. Same as before but using "ANY" as CMT config """ gLogger.always("**** Boole v30r1") ra = RunApplication() ra.applicationName = 'Boole' ra.applicationVersion = 'v30r1' ra.commandOptions = ['$APPCONFIGOPTS/Boole/Default.py', '$APPCONFIGOPTS/Boole/DataType-2012.py', '$APPCONFIGOPTS/Boole/NoPacking.py', '$APPCONFIGOPTS/Boole/Boole-SetOdinRndTrigger.py', '$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py'] ra.extraPackages = [('AppConfig', 'v3r266'), ('ProdConf', '') ] ra.step_Number = 1 ra.prodConfFileName = find_all('test_prodConf_boole_v30r1_2.py', '..')[0] ra.applicationLog = '00033857_00000002_3_log.txt' ra.stdError = '00033857_00000002_3_err.txt' res = ra.run() self.assertTrue(res['OK']) self.assertEqual(res['Value'], (0, '', ''))
def test_Gauss_User_step133294(self): """ Not using ProdConf (users style). systemConfig = 'x86_64-slc6-gcc48-opt' This is taken from PR 33857 (and would fall back to SetupProject) """ gLogger.always("**** GAUSS v49r5") ra = RunApplication() ra.applicationName = 'Gauss' ra.applicationVersion = 'v49r5' ra.systemConfig = 'x86_64-slc6-gcc48-opt' # ra.commandOptions = ['$APPCONFIGOPTS/Gauss/Sim08-Beam4000GeV-mu100-2012-nu2.5.py', # '$APPCONFIGOPTS/Gauss/DataType-2012.py', # '$APPCONFIGOPTS/Gauss/RICHRandomHits.py', # '$APPCONFIGOPTS/Gauss/NoPacking.py', # '$DECFILESROOT/options/12877041.py', # '$LBPYTHIA8ROOT/options/Pythia8.py', # '$APPCONFIGOPTS/Gauss/G4PL_FTFP_BERT_EmNoCuts.py', # '$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py'] ra.extraPackages = [('AppConfig', 'v3r277'), ('Gen/DecFiles', 'v29r10'), ] generatedOpts = 'gaudi_extra_options.py' if os.path.exists(generatedOpts): os.remove(generatedOpts) inputDataOpts = getDataOptions('Gauss', [], None, 'pool_xml_catalog.xml')['Value'] # always OK projectOpts = getModuleOptions('Gauss', 1, inputDataOpts, '', 5732353, 4340993, 'User')['Value'] # always OK options = open(generatedOpts, 'w') options.write(projectOpts) options.close() ra.applicationLog = 'user_133294_log.txt' ra.stdError = 'user_133294_err.txt' res = ra.run() # this won't really do anything but it doesn't matters self.assertTrue(res['OK']) self.assertEqual(res['Value'], (0, '', ''))
def _mergeRootFiles(outputFile, inputFiles): """ Merge ROOT files Args: outputFile (str): output file name """ cmd = "hadd -f %s " % outputFile + ' '.join(inputFiles) ra = RunApplication() ra.applicationName = 'ROOT' ra.command = cmd try: ra.run() return S_OK() except RuntimeError: return _errorReport("Failed to merge root files")
def test_Brunel_new_bestCMT(self): """ Using ProdConf (production style). systemConfig = 'ANY' This is taken from step 131287 (Reco17) """ gLogger.always("**** Brunel v52r2") ra = RunApplication() ra.applicationName = 'Brunel' ra.applicationVersion = 'v52r2' ra.commandOptions = ['$APPCONFIGOPTS/Brunel/DataType-2017.py', '$APPCONFIGOPTS/Brunel/rdst.py'] ra.extraPackages = [('AppConfig', 'v3r323'), ('Det/SQLDDDB', 'v7r11'), ('ProdConf', '') ] ra.step_Number = 1 ra.prodConfFileName = find_all('test_prodConf_brunel_v52r2.py', '..')[0] ra.applicationLog = '0brunel_000v52r2_ANY_62_log.txt' ra.stdError = '0brunel_000v52r2_ANY_62_err.txt' self.assertRaises(LHCbApplicationError, ra.run) # This will fail as there's no input file
def test_Moore_2_Production_PR33857_4(self): """ Using ProdConf (production style). systemConfig = 'x86_64-slc5-gcc46-opt' This is taken from PR 33857 """ gLogger.always("**** MOORE v14r8p1") ra = RunApplication() ra.applicationName = 'Moore' ra.applicationVersion = 'v14r8p1' ra.systemConfig = 'x86_64-slc5-gcc46-opt' ra.commandOptions = ['$APPCONFIGOPTS/Moore/MooreSimProductionForSeparateL0AppStep.py', '$APPCONFIGOPTS/Conditions/TCK-0x409f0045.py', '$APPCONFIGOPTS/Moore/DataType-2012.py'] ra.extraPackages = [('AppConfig', 'v3r241'), ('ProdConf', '') ] ra.prodConfFileName = find_all('test_prodConf_moore_v14r8p1.py', '..')[0] ra.applicationLog = '00033857_00000004_4_log.txt' ra.stdError = '00033857_00000004_4_err.txt' res = ra.run() self.assertTrue(res['OK']) self.assertEqual(res['Value'], (0, '', ''))
def test_Moore_1_Production_PR33857_3(self): """ Using ProdConf (production style). systemConfig = 'x86_64-slc6-gcc48-opt' This is taken from PR 33857 """ gLogger.always("**** MOORE v20r4") ra = RunApplication() ra.applicationName = 'Moore' ra.applicationVersion = 'v20r4' ra.systemConfig = 'x86_64-slc6-gcc48-opt' ra.commandOptions = ['$APPCONFIGOPTS/L0App/L0AppSimProduction.py', '$APPCONFIGOPTS/L0App/L0AppTCK-0x0045.py', '$APPCONFIGOPTS/L0App/DataType-2012.py'] ra.extraPackages = [('AppConfig', 'v3r200'), ('ProdConf', '') ] ra.prodConfFileName = find_all('test_prodConf_moore_v20r4.py', '..')[0] ra.applicationLog = '00033857_00000003_3_log.txt' ra.stdError = '00033857_00000003_3_err.txt' res = ra.run() self.assertTrue(res['OK']) self.assertEqual(res['Value'], (0, '', ''))
def execute(self, production_id=None, prod_job_id=None, wms_job_id=None, workflowStatus=None, stepStatus=None, wf_commons=None, step_commons=None, step_number=None, step_id=None): """ The main execution method of the module. It runs a gaudi script app using RunApplication module. This is usually the main module run by user jobs. """ try: super(GaudiApplicationScript, self).execute(self.version, production_id, prod_job_id, wms_job_id, workflowStatus, stepStatus, wf_commons, step_commons, step_number, step_id) self._resolveInputVariables() if not self.applicationName or not self.applicationVersion: raise RuntimeError('No Gaudi Application defined') if not self.systemConfig: raise RuntimeError('No binary tag selected') if not self.script: raise RuntimeError('No script defined') if not self.applicationLog: self.applicationLog = '%s.log' % (os.path.basename( self.script)) self.log.info("Executing application %s %s for binary tag %s" % (self.applicationName, self.applicationVersion, self.systemConfig)) gaudiCmd = [] if re.search('.py$', self.script): gaudiCmd.append('python') gaudiCmd.append(os.path.basename(self.script)) gaudiCmd.append(self.arguments) else: gaudiCmd.append(os.path.basename(self.script)) gaudiCmd.append(self.arguments) command = ' '.join(gaudiCmd) # How to run the application ra = RunApplication() # lb-run stuff ra.applicationName = self.applicationName ra.applicationVersion = self.applicationVersion ra.systemConfig = self.systemConfig # actual stuff to run ra.command = command ra.applicationLog = self.applicationLog # Now really running self.setApplicationStatus(self.applicationName) ra.run( ) # This would trigger an exception in case of failure, or application status != 0 self.setApplicationStatus('%s Successful' % os.path.basename(self.script)) return S_OK('%s Successful' % os.path.basename(self.script)) except LbRunError as lbre: # This is the case for lb-run/environment errors self.setApplicationStatus(repr(lbre)) return S_ERROR(DErrno.EWMSRESC, str(lbre)) except LHCbApplicationError as lbae: # This is the case for real application errors self.setApplicationStatus(repr(lbae)) return S_ERROR(str(lbae)) except LHCbDIRACError as lbde: # This is the case for LHCbDIRAC errors (e.g. subProcess call failed) self.setApplicationStatus(repr(lbde)) return S_ERROR(str(lbde)) except Exception as e: # pylint:disable=broad-except self.log.exception( "Failure in GaudiApplicationScript execute module", lException=e) self.setApplicationStatus("Error in GaudiApplicationScript module") return S_ERROR(str(e)) finally: super(GaudiApplicationScript, self).finalize(self.version)
def execute(self, production_id=None, prod_job_id=None, wms_job_id=None, workflowStatus=None, stepStatus=None, wf_commons=None, step_commons=None, step_id=None, step_number=None): """ The main execution method of GaudiApplication. It runs a gaudirun app using RunApplication module. This is the module used for each and every job of productions. It can also be used by users. """ try: super(GaudiApplication, self).execute(__RCSID__, production_id, prod_job_id, wms_job_id, workflowStatus, stepStatus, wf_commons, step_commons, step_number, step_id) if not self._checkWFAndStepStatus(): return S_OK() self._resolveInputVariables() self.log.info("Executing application %s %s for binary tag %s" % (self.applicationName, self.applicationVersion, self.systemConfig)) if self.jobType.lower() == 'merge' in self.siteName: self._disableWatchdogCPUCheck() # Resolve options files commandOptions = [] if self.optionsFile and self.optionsFile != "None": for fileopt in self.optionsFile.split(';'): if os.path.exists( '%s/%s' % (os.getcwd(), os.path.basename(fileopt))): commandOptions.append(fileopt) # Otherwise take the one from the application options directory elif re.search(r'\$', fileopt): self.log.info( 'Found options file containing environment variable: %s' % fileopt) commandOptions.append(fileopt) else: self.log.error( 'Cannot process options: "%s" not found via environment variable or in local directory' % (fileopt)) self.log.info('Final options files: %s' % (', '.join(commandOptions))) runNumberGauss = 0 firstEventNumberGauss = 0 if self.applicationName.lower( ) == "gauss" and self.production_id and self.prod_job_id: if self.jobType.lower() == 'user': eventsMax = self.numberOfEvents else: # maintaining backward compatibility eventsMax = self.maxNumberOfEvents if self.maxNumberOfEvents else self.numberOfEvents runNumberGauss = int(self.production_id) * 100 + int( self.prod_job_id) firstEventNumberGauss = eventsMax * (int(self.prod_job_id) - 1) + 1 if self.optionsLine or self.jobType.lower() == 'user': self.log.debug("Won't get any step outputs (USER job)") stepOutputs = [] stepOutputTypes = [] histogram = False else: self.log.debug("Getting the step outputs") stepOutputs, stepOutputTypes, histogram = self._determineOutputs( ) self.log.debug( "stepOutputs, stepOutputTypes, histogram ==> %s, %s, %s" % (stepOutputs, stepOutputTypes, histogram)) prodConfFileName = '' if self.optionsLine or self.jobType.lower() == 'user': # Prepare standard project run time options generatedOpts = 'gaudi_extra_options.py' if os.path.exists(generatedOpts): os.remove(generatedOpts) inputDataOpts = getDataOptions( self.applicationName, self.stepInputData, self.inputDataType, self.poolXMLCatName)['Value'] # always OK projectOpts = getModuleOptions( self.applicationName, self.numberOfEvents, inputDataOpts, self.optionsLine, runNumberGauss, firstEventNumberGauss, self.jobType)['Value'] # always OK self.log.info('Extra options generated for %s %s step:' % (self.applicationName, self.applicationVersion)) print projectOpts # Always useful to see in the logs (don't use gLogger as we often want to cut n' paste) with open(generatedOpts, 'w') as options: options.write(projectOpts) commandOptions.append(generatedOpts) else: prodConfFileName = self.createProdConfFile( stepOutputTypes, histogram, runNumberGauss, firstEventNumberGauss) # How to run the application ra = RunApplication() # lb-run stuff ra.applicationName = self.applicationName ra.applicationVersion = self.applicationVersion ra.systemConfig = self.systemConfig ra.extraPackages = self.extraPackages ra.runTimeProject = self.runTimeProjectName ra.runTimeProjectVersion = self.runTimeProjectVersion # actual stuff to run ra.command = self.executable ra.extraOptionsLine = self.extraOptionsLine ra.commandOptions = commandOptions if self.multicoreStep.upper() == 'Y': ra.multicore = self.multicoreJob ra.prodConfFileName = prodConfFileName if self.applicationLog: ra.applicationLog = self.applicationLog ra.stdError = self.stdError # Now really running try: self.setApplicationStatus( '%s step %s' % (self.applicationName, self.step_number)) ra.run( ) # This would trigger an exception in case of failure, or application status != 0 except LHCbApplicationError as appError: # Running gdb in case of core dump if 'core' in [ fileProduced.split('.')[0] for fileProduced in os.listdir('.') ]: # getting the environment where the application executed app = ra.applicationName + '/' + ra.applicationVersion envCommand = ra.lbrunCommand.split( app)[0] + ' --py -A ' + app # FIXME: The following may raise CalledProcessError if the application is not lb-run native. # The exception will be caught at the end of the module. # We enter here iff the application fails, and there's a core dump, so it's ~OKish. # Nevertheless, it would be better if it was handled correctly. lhcbApplicationEnv = eval( subprocess.check_output(shlex.split(envCommand))) # now running the GDB command gdbCommand = "gdb python core.* >> %s_Step%s_coredump.log" % ( self.applicationName, self.step_number) rg = RunApplication() rg._runApp(gdbCommand, lhcbApplicationEnv) raise appError self.log.info("Going to manage %s output" % self.applicationName) self._manageAppOutput(stepOutputs) # Still have to set the application status e.g. user job case. self.setApplicationStatus( '%s %s Successful' % (self.applicationName, self.applicationVersion)) return S_OK("%s %s Successful" % (self.applicationName, self.applicationVersion)) except LbRunError as lbre: # This is the case for lb-run/environment errors self.setApplicationStatus(repr(lbre)) return S_ERROR(DErrno.EWMSRESC, str(lbre)) except LHCbApplicationError as lbae: # This is the case for real application errors self.setApplicationStatus(repr(lbae)) return S_ERROR(str(lbae)) except LHCbDIRACError as lbde: # This is the case for LHCbDIRAC errors (e.g. subProcess call failed) self.setApplicationStatus(repr(lbde)) return S_ERROR(str(lbde)) except Exception as exc: # pylint:disable=broad-except self.log.exception("Failure in GaudiApplication execute module", lException=exc, lExcInfo=True) self.setApplicationStatus("Error in GaudiApplication module") return S_ERROR(str(exc)) finally: super(GaudiApplication, self).finalize(__RCSID__)
def execute(self, production_id=None, prod_job_id=None, wms_job_id=None, workflowStatus=None, stepStatus=None, wf_commons=None, step_commons=None, step_id=None, step_number=None): """The main execution method of the RootApplication module: runs a ROOT app using RunApplication module """ try: super(RootApplication, self).execute(self.version, production_id, prod_job_id, wms_job_id, workflowStatus, stepStatus, wf_commons, step_commons, step_number, step_id) self._resolveInputVariables() if not self.applicationVersion: raise RuntimeError('No Root Version defined') if not self.systemConfig: raise RuntimeError('No system configuration selected') if not self.applicationName: raise RuntimeError('No script defined') if not self.applicationLog: self.applicationLog = '%s.log' % self.applicationName if self.rootType.lower() not in ('c', 'py', 'bin', 'exe'): raise RuntimeError('Wrong root type defined') self.setApplicationStatus('Initializing RootApplication module') self.log.info("Executing application Root %s with CMT config %s " % (self.applicationVersion, self.systemConfig)) if not os.path.exists(self.applicationName): self.log.error('rootScript not Found at %s' % os.path.abspath('.')) return S_ERROR('rootScript not Found') if self.rootType.lower() == 'c': rootCmd = ['root'] rootCmd.append('-b') rootCmd.append('-f') if self.arguments: escapedArgs = [] for arg in self.arguments: if isinstance(arg, str): escapedArgs.append('\'"%s"\'' % (arg)) else: escapedArgs.append('%s' % (arg)) macroArgs = r'%s\(%s\)' % (self.applicationName, ','.join(escapedArgs)) rootCmd.append(macroArgs) else: rootCmd.append(self.applicationName) elif self.rootType.lower() == 'py': rootCmd = ['python'] rootCmd = [os.path.abspath(self.applicationName)] if self.arguments: rootCmd += self.arguments if os.path.exists(self.applicationLog): os.remove(self.applicationLog) # How to run the application ra = RunApplication() # lb-run stuff ra.applicationName = self.applicationName ra.applicationVersion = self.applicationVersion ra.systemConfig = self.systemConfig # actual stuff to run ra.command = rootCmd ra.applicationLog = self.applicationLog # Now really running ra.run( ) # This would trigger an exception in case of failure, or application status != 0 # Return OK assuming that subsequent module will spot problems self.setApplicationStatus('%s (Root) Successful' % self.applicationName) return S_OK() except LbRunError as lbre: # This is the case for lb-run/environment errors self.setApplicationStatus(repr(lbre)) return S_ERROR(DErrno.EWMSRESC, str(lbre)) except LHCbApplicationError as lbae: # This is the case for real application errors self.setApplicationStatus(repr(lbae)) return S_ERROR(str(lbae)) except LHCbDIRACError as lbde: # This is the case for LHCbDIRAC errors (e.g. subProcess call failed) self.setApplicationStatus(repr(lbde)) return S_ERROR(str(lbde)) except Exception as e: # pylint:disable=broad-except self.log.exception("Failure in RootApplication execute module", lException=e) return S_ERROR("Error in RootApplication module") finally: super(RootApplication, self).finalize(self.version)