def test_DaVinci_new_gcc62(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc6-gcc62-opt'
This is taken from step 130847
"""
gLogger.always("**** DAVINCI v42r2")
ra = RunApplication()
ra.applicationName = 'DaVinci'
ra.applicationVersion = 'v42r2'
ra.systemConfig = 'x86_64-slc6-gcc62-opt'
ra.commandOptions = ['$APPCONFIGOPTS/DaVinci/DV-Stripping27-Stripping.py',
'$APPCONFIGOPTS/DaVinci/DataType-2016.py',
'$APPCONFIGOPTS/DaVinci/InputType-RDST.py',
'$APPCONFIGOPTS/DaVinci/DV-RawEventJuggler-0_3-to-4_2.py']
ra.extraPackages = [('AppConfig', 'v3r308'),
('Det/SQLDDDB', 'v7r11'),
('ProdConf', '')
]
ra.step_Number = 1
ra.prodConfFileName = find_all('test_prodConf_davinci_v42r2.py', '..')[0]
ra.applicationLog = '0daVinci_000v42r2_62_log.txt'
ra.stdError = '0daVinci_000v42r2_62_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def test_Gauss_Production_PR33857_1(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc6-gcc48-opt'
This is taken from PR 33857 (and would fall back to SetupProject)
"""
gLogger.always("**** GAUSS v49r5")
ra = RunApplication()
ra.applicationName = 'Gauss'
ra.applicationVersion = 'v49r5'
ra.systemConfig = 'x86_64-slc6-gcc48-opt'
ra.commandOptions = ['$APPCONFIGOPTS/Gauss/Sim08-Beam4000GeV-mu100-2012-nu2.5.py',
'$APPCONFIGOPTS/Gauss/DataType-2012.py',
'$APPCONFIGOPTS/Gauss/RICHRandomHits.py',
'$APPCONFIGOPTS/Gauss/NoPacking.py',
'$DECFILESROOT/options/12877041.py',
'$LBPYTHIA8ROOT/options/Pythia8.py',
'$APPCONFIGOPTS/Gauss/G4PL_FTFP_BERT_EmNoCuts.py',
'$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py']
ra.extraPackages = [('AppConfig', 'v3r277'),
('Gen/DecFiles', 'v29r10'),
('ProdConf', '')
]
ra.prodConfFileName = find_all('test_prodConf_gauss_v49r5.py', '..')[0]
ra.applicationLog = '00033857_00000001_1_log.txt'
ra.stdError = '00033857_00000001_1_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def test_DaVinci_1_Production_PR33857_6(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc5-gcc46-opt'
This is taken from PR 33857
"""
gLogger.always("**** DAVINCI v32r2p1")
ra = RunApplication()
ra.applicationName = 'DaVinci'
ra.applicationVersion = 'v32r2p1'
ra.systemConfig = 'x86_64-slc5-gcc46-opt'
ra.commandOptions = ['$CHARMCONFIGOPTS/MCFiltering/D02K3PiFromB2DstmunuXStripTrigFiltering_2012.py',
'$APPCONFIGOPTS/DaVinci/DataType-2012.py',
'$APPCONFIGOPTS/DaVinci/InputType-DST.py',
'$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py']
ra.extraPackages = [('AppConfig', 'v3r305'),
('CharmConfig', 'v3r30'),
('ProdConf', '')
]
ra.step_Number = 1
ra.prodConfFileName = find_all('test_prodConf_davinci_v32r2p1.py', '..')[0]
ra.applicationLog = '00033857_00000006_6_log.txt'
ra.stdError = '00033857_00000006_6_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def test_DaVinci_2_Production_PR33857_7(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc6-gcc49-opt'
This is taken from PR 33857
"""
gLogger.always("**** DAVINCI v41r3")
ra = RunApplication()
ra.applicationName = 'DaVinci'
ra.applicationVersion = 'v41r3'
ra.systemConfig = 'x86_64-slc6-gcc49-opt'
ra.commandOptions = ['$APPCONFIGOPTS/Merging/DVMergeDST.py',
'$APPCONFIGOPTS/DaVinci/DataType-2012.py',
'$APPCONFIGOPTS/Merging/WriteFSR.py',
'$APPCONFIGOPTS/Merging/MergeFSR.py']
ra.extraPackages = [('AppConfig', 'v3r305'),
('ProdConf', '')
]
ra.step_Number = 1
ra.prodConfFileName = find_all('test_prodConf_davinci_v41r3.py', '..')[0]
ra.applicationLog = '00033857_00000007_7_log.txt'
ra.stdError = '00033857_00000007_7_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def test_Brunel_Production_PR33857_5(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc5-gcc46-opt'
This is taken from PR 33857
"""
gLogger.always("**** BRUNEL v43r2p11")
ra = RunApplication()
ra.applicationName = 'Brunel'
ra.applicationVersion = 'v43r2p11'
ra.systemConfig = 'x86_64-slc5-gcc46-opt'
ra.commandOptions = ['$APPCONFIGOPTS/Brunel/DataType-2012.py',
'$APPCONFIGOPTS/Brunel/MC-WithTruth.py',
'$APPCONFIGOPTS/Brunel/Sim09-Run1.py',
'$APPCONFIGOPTS/Persistency/DST-multipleTCK-2012.py',
'$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py']
ra.extraPackages = [('AppConfig', 'v3r302'),
('ProdConf', '')
]
ra.prodConfFileName = find_all('test_prodConf_brunel_v43r2p11.py', '..')[0]
ra.applicationLog = '00033857_00000005_5_log.txt'
ra.stdError = '00033857_00000005_5_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def test_Boole_2_Production_PR33857(self):
""" Using ProdConf (production style). Taken from PR 33857.
Same as before but using "ANY" as CMT config
"""
gLogger.always("**** Boole v30r1")
ra = RunApplication()
ra.applicationName = 'Boole'
ra.applicationVersion = 'v30r1'
ra.commandOptions = ['$APPCONFIGOPTS/Boole/Default.py',
'$APPCONFIGOPTS/Boole/DataType-2012.py',
'$APPCONFIGOPTS/Boole/NoPacking.py',
'$APPCONFIGOPTS/Boole/Boole-SetOdinRndTrigger.py',
'$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py']
ra.extraPackages = [('AppConfig', 'v3r266'),
('ProdConf', '')
]
ra.step_Number = 1
ra.prodConfFileName = find_all('test_prodConf_boole_v30r1_2.py', '..')[0]
ra.applicationLog = '00033857_00000002_3_log.txt'
ra.stdError = '00033857_00000002_3_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def _mergeRootFiles(outputFile, inputFiles):
""" Merge ROOT files
Args:
outputFile (str): output file name
"""
cmd = "hadd -f %s " % outputFile + ' '.join(inputFiles)
ra = RunApplication()
ra.applicationName = 'ROOT'
ra.command = cmd
try:
ra.run()
return S_OK()
except RuntimeError:
return _errorReport("Failed to merge root files")
def test_Gauss_User_step133294(self):
""" Not using ProdConf (users style). systemConfig = 'x86_64-slc6-gcc48-opt'
This is taken from PR 33857 (and would fall back to SetupProject)
"""
gLogger.always("**** GAUSS v49r5")
ra = RunApplication()
ra.applicationName = 'Gauss'
ra.applicationVersion = 'v49r5'
ra.systemConfig = 'x86_64-slc6-gcc48-opt'
# ra.commandOptions = ['$APPCONFIGOPTS/Gauss/Sim08-Beam4000GeV-mu100-2012-nu2.5.py',
# '$APPCONFIGOPTS/Gauss/DataType-2012.py',
# '$APPCONFIGOPTS/Gauss/RICHRandomHits.py',
# '$APPCONFIGOPTS/Gauss/NoPacking.py',
# '$DECFILESROOT/options/12877041.py',
# '$LBPYTHIA8ROOT/options/Pythia8.py',
# '$APPCONFIGOPTS/Gauss/G4PL_FTFP_BERT_EmNoCuts.py',
# '$APPCONFIGOPTS/Persistency/Compression-ZLIB-1.py']
ra.extraPackages = [('AppConfig', 'v3r277'),
('Gen/DecFiles', 'v29r10'),
]
generatedOpts = 'gaudi_extra_options.py'
if os.path.exists(generatedOpts):
os.remove(generatedOpts)
inputDataOpts = getDataOptions('Gauss',
[],
None,
'pool_xml_catalog.xml')['Value'] # always OK
projectOpts = getModuleOptions('Gauss',
1,
inputDataOpts,
'',
5732353,
4340993,
'User')['Value'] # always OK
options = open(generatedOpts, 'w')
options.write(projectOpts)
options.close()
ra.applicationLog = 'user_133294_log.txt'
ra.stdError = 'user_133294_err.txt'
res = ra.run() # this won't really do anything but it doesn't matters
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def test_Moore_2_Production_PR33857_4(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc5-gcc46-opt'
This is taken from PR 33857
"""
gLogger.always("**** MOORE v14r8p1")
ra = RunApplication()
ra.applicationName = 'Moore'
ra.applicationVersion = 'v14r8p1'
ra.systemConfig = 'x86_64-slc5-gcc46-opt'
ra.commandOptions = ['$APPCONFIGOPTS/Moore/MooreSimProductionForSeparateL0AppStep.py',
'$APPCONFIGOPTS/Conditions/TCK-0x409f0045.py',
'$APPCONFIGOPTS/Moore/DataType-2012.py']
ra.extraPackages = [('AppConfig', 'v3r241'),
('ProdConf', '')
]
ra.prodConfFileName = find_all('test_prodConf_moore_v14r8p1.py', '..')[0]
ra.applicationLog = '00033857_00000004_4_log.txt'
ra.stdError = '00033857_00000004_4_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def test_Moore_1_Production_PR33857_3(self):
""" Using ProdConf (production style). systemConfig = 'x86_64-slc6-gcc48-opt'
This is taken from PR 33857
"""
gLogger.always("**** MOORE v20r4")
ra = RunApplication()
ra.applicationName = 'Moore'
ra.applicationVersion = 'v20r4'
ra.systemConfig = 'x86_64-slc6-gcc48-opt'
ra.commandOptions = ['$APPCONFIGOPTS/L0App/L0AppSimProduction.py',
'$APPCONFIGOPTS/L0App/L0AppTCK-0x0045.py',
'$APPCONFIGOPTS/L0App/DataType-2012.py']
ra.extraPackages = [('AppConfig', 'v3r200'),
('ProdConf', '')
]
ra.prodConfFileName = find_all('test_prodConf_moore_v20r4.py', '..')[0]
ra.applicationLog = '00033857_00000003_3_log.txt'
ra.stdError = '00033857_00000003_3_err.txt'
res = ra.run()
self.assertTrue(res['OK'])
self.assertEqual(res['Value'], (0, '', ''))
def execute(self,
production_id=None,
prod_job_id=None,
wms_job_id=None,
workflowStatus=None,
stepStatus=None,
wf_commons=None,
step_commons=None,
step_number=None,
step_id=None):
""" The main execution method of the module. It runs a gaudi script app using RunApplication module.
This is usually the main module run by user jobs.
"""
try:
super(GaudiApplicationScript,
self).execute(self.version, production_id, prod_job_id,
wms_job_id, workflowStatus, stepStatus,
wf_commons, step_commons, step_number, step_id)
self._resolveInputVariables()
if not self.applicationName or not self.applicationVersion:
raise RuntimeError('No Gaudi Application defined')
if not self.systemConfig:
raise RuntimeError('No binary tag selected')
if not self.script:
raise RuntimeError('No script defined')
if not self.applicationLog:
self.applicationLog = '%s.log' % (os.path.basename(
self.script))
self.log.info("Executing application %s %s for binary tag %s" %
(self.applicationName, self.applicationVersion,
self.systemConfig))
gaudiCmd = []
if re.search('.py$', self.script):
gaudiCmd.append('python')
gaudiCmd.append(os.path.basename(self.script))
gaudiCmd.append(self.arguments)
else:
gaudiCmd.append(os.path.basename(self.script))
gaudiCmd.append(self.arguments)
command = ' '.join(gaudiCmd)
# How to run the application
ra = RunApplication()
# lb-run stuff
ra.applicationName = self.applicationName
ra.applicationVersion = self.applicationVersion
ra.systemConfig = self.systemConfig
# actual stuff to run
ra.command = command
ra.applicationLog = self.applicationLog
# Now really running
self.setApplicationStatus(self.applicationName)
ra.run(
) # This would trigger an exception in case of failure, or application status != 0
self.setApplicationStatus('%s Successful' %
os.path.basename(self.script))
return S_OK('%s Successful' % os.path.basename(self.script))
except LbRunError as lbre: # This is the case for lb-run/environment errors
self.setApplicationStatus(repr(lbre))
return S_ERROR(DErrno.EWMSRESC, str(lbre))
except LHCbApplicationError as lbae: # This is the case for real application errors
self.setApplicationStatus(repr(lbae))
return S_ERROR(str(lbae))
except LHCbDIRACError as lbde: # This is the case for LHCbDIRAC errors (e.g. subProcess call failed)
self.setApplicationStatus(repr(lbde))
return S_ERROR(str(lbde))
except Exception as e: # pylint:disable=broad-except
self.log.exception(
"Failure in GaudiApplicationScript execute module",
lException=e)
self.setApplicationStatus("Error in GaudiApplicationScript module")
return S_ERROR(str(e))
finally:
super(GaudiApplicationScript, self).finalize(self.version)
def execute(self,
production_id=None,
prod_job_id=None,
wms_job_id=None,
workflowStatus=None,
stepStatus=None,
wf_commons=None,
step_commons=None,
step_id=None,
step_number=None):
""" The main execution method of GaudiApplication. It runs a gaudirun app using RunApplication module.
This is the module used for each and every job of productions. It can also be used by users.
"""
try:
super(GaudiApplication,
self).execute(__RCSID__, production_id, prod_job_id,
wms_job_id, workflowStatus, stepStatus,
wf_commons, step_commons, step_number, step_id)
if not self._checkWFAndStepStatus():
return S_OK()
self._resolveInputVariables()
self.log.info("Executing application %s %s for binary tag %s" %
(self.applicationName, self.applicationVersion,
self.systemConfig))
if self.jobType.lower() == 'merge' in self.siteName:
self._disableWatchdogCPUCheck()
# Resolve options files
commandOptions = []
if self.optionsFile and self.optionsFile != "None":
for fileopt in self.optionsFile.split(';'):
if os.path.exists(
'%s/%s' %
(os.getcwd(), os.path.basename(fileopt))):
commandOptions.append(fileopt)
# Otherwise take the one from the application options directory
elif re.search(r'\$', fileopt):
self.log.info(
'Found options file containing environment variable: %s'
% fileopt)
commandOptions.append(fileopt)
else:
self.log.error(
'Cannot process options: "%s" not found via environment variable or in local directory'
% (fileopt))
self.log.info('Final options files: %s' %
(', '.join(commandOptions)))
runNumberGauss = 0
firstEventNumberGauss = 0
if self.applicationName.lower(
) == "gauss" and self.production_id and self.prod_job_id:
if self.jobType.lower() == 'user':
eventsMax = self.numberOfEvents
else:
# maintaining backward compatibility
eventsMax = self.maxNumberOfEvents if self.maxNumberOfEvents else self.numberOfEvents
runNumberGauss = int(self.production_id) * 100 + int(
self.prod_job_id)
firstEventNumberGauss = eventsMax * (int(self.prod_job_id) -
1) + 1
if self.optionsLine or self.jobType.lower() == 'user':
self.log.debug("Won't get any step outputs (USER job)")
stepOutputs = []
stepOutputTypes = []
histogram = False
else:
self.log.debug("Getting the step outputs")
stepOutputs, stepOutputTypes, histogram = self._determineOutputs(
)
self.log.debug(
"stepOutputs, stepOutputTypes, histogram ==> %s, %s, %s"
% (stepOutputs, stepOutputTypes, histogram))
prodConfFileName = ''
if self.optionsLine or self.jobType.lower() == 'user':
# Prepare standard project run time options
generatedOpts = 'gaudi_extra_options.py'
if os.path.exists(generatedOpts):
os.remove(generatedOpts)
inputDataOpts = getDataOptions(
self.applicationName, self.stepInputData,
self.inputDataType,
self.poolXMLCatName)['Value'] # always OK
projectOpts = getModuleOptions(
self.applicationName, self.numberOfEvents, inputDataOpts,
self.optionsLine, runNumberGauss, firstEventNumberGauss,
self.jobType)['Value'] # always OK
self.log.info('Extra options generated for %s %s step:' %
(self.applicationName, self.applicationVersion))
print projectOpts # Always useful to see in the logs (don't use gLogger as we often want to cut n' paste)
with open(generatedOpts, 'w') as options:
options.write(projectOpts)
commandOptions.append(generatedOpts)
else:
prodConfFileName = self.createProdConfFile(
stepOutputTypes, histogram, runNumberGauss,
firstEventNumberGauss)
# How to run the application
ra = RunApplication()
# lb-run stuff
ra.applicationName = self.applicationName
ra.applicationVersion = self.applicationVersion
ra.systemConfig = self.systemConfig
ra.extraPackages = self.extraPackages
ra.runTimeProject = self.runTimeProjectName
ra.runTimeProjectVersion = self.runTimeProjectVersion
# actual stuff to run
ra.command = self.executable
ra.extraOptionsLine = self.extraOptionsLine
ra.commandOptions = commandOptions
if self.multicoreStep.upper() == 'Y':
ra.multicore = self.multicoreJob
ra.prodConfFileName = prodConfFileName
if self.applicationLog:
ra.applicationLog = self.applicationLog
ra.stdError = self.stdError
# Now really running
try:
self.setApplicationStatus(
'%s step %s' % (self.applicationName, self.step_number))
ra.run(
) # This would trigger an exception in case of failure, or application status != 0
except LHCbApplicationError as appError:
# Running gdb in case of core dump
if 'core' in [
fileProduced.split('.')[0]
for fileProduced in os.listdir('.')
]:
# getting the environment where the application executed
app = ra.applicationName + '/' + ra.applicationVersion
envCommand = ra.lbrunCommand.split(
app)[0] + ' --py -A ' + app
# FIXME: The following may raise CalledProcessError if the application is not lb-run native.
# The exception will be caught at the end of the module.
# We enter here iff the application fails, and there's a core dump, so it's ~OKish.
# Nevertheless, it would be better if it was handled correctly.
lhcbApplicationEnv = eval(
subprocess.check_output(shlex.split(envCommand)))
# now running the GDB command
gdbCommand = "gdb python core.* >> %s_Step%s_coredump.log" % (
self.applicationName, self.step_number)
rg = RunApplication()
rg._runApp(gdbCommand, lhcbApplicationEnv)
raise appError
self.log.info("Going to manage %s output" % self.applicationName)
self._manageAppOutput(stepOutputs)
# Still have to set the application status e.g. user job case.
self.setApplicationStatus(
'%s %s Successful' %
(self.applicationName, self.applicationVersion))
return S_OK("%s %s Successful" %
(self.applicationName, self.applicationVersion))
except LbRunError as lbre: # This is the case for lb-run/environment errors
self.setApplicationStatus(repr(lbre))
return S_ERROR(DErrno.EWMSRESC, str(lbre))
except LHCbApplicationError as lbae: # This is the case for real application errors
self.setApplicationStatus(repr(lbae))
return S_ERROR(str(lbae))
except LHCbDIRACError as lbde: # This is the case for LHCbDIRAC errors (e.g. subProcess call failed)
self.setApplicationStatus(repr(lbde))
return S_ERROR(str(lbde))
except Exception as exc: # pylint:disable=broad-except
self.log.exception("Failure in GaudiApplication execute module",
lException=exc,
lExcInfo=True)
self.setApplicationStatus("Error in GaudiApplication module")
return S_ERROR(str(exc))
finally:
super(GaudiApplication, self).finalize(__RCSID__)
def execute(self,
production_id=None,
prod_job_id=None,
wms_job_id=None,
workflowStatus=None,
stepStatus=None,
wf_commons=None,
step_commons=None,
step_id=None,
step_number=None):
"""The main execution method of the RootApplication module: runs a ROOT app using RunApplication module
"""
try:
super(RootApplication,
self).execute(self.version, production_id, prod_job_id,
wms_job_id, workflowStatus, stepStatus,
wf_commons, step_commons, step_number, step_id)
self._resolveInputVariables()
if not self.applicationVersion:
raise RuntimeError('No Root Version defined')
if not self.systemConfig:
raise RuntimeError('No system configuration selected')
if not self.applicationName:
raise RuntimeError('No script defined')
if not self.applicationLog:
self.applicationLog = '%s.log' % self.applicationName
if self.rootType.lower() not in ('c', 'py', 'bin', 'exe'):
raise RuntimeError('Wrong root type defined')
self.setApplicationStatus('Initializing RootApplication module')
self.log.info("Executing application Root %s with CMT config %s " %
(self.applicationVersion, self.systemConfig))
if not os.path.exists(self.applicationName):
self.log.error('rootScript not Found at %s' %
os.path.abspath('.'))
return S_ERROR('rootScript not Found')
if self.rootType.lower() == 'c':
rootCmd = ['root']
rootCmd.append('-b')
rootCmd.append('-f')
if self.arguments:
escapedArgs = []
for arg in self.arguments:
if isinstance(arg, str):
escapedArgs.append('\'"%s"\'' % (arg))
else:
escapedArgs.append('%s' % (arg))
macroArgs = r'%s\(%s\)' % (self.applicationName,
','.join(escapedArgs))
rootCmd.append(macroArgs)
else:
rootCmd.append(self.applicationName)
elif self.rootType.lower() == 'py':
rootCmd = ['python']
rootCmd = [os.path.abspath(self.applicationName)]
if self.arguments:
rootCmd += self.arguments
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
# How to run the application
ra = RunApplication()
# lb-run stuff
ra.applicationName = self.applicationName
ra.applicationVersion = self.applicationVersion
ra.systemConfig = self.systemConfig
# actual stuff to run
ra.command = rootCmd
ra.applicationLog = self.applicationLog
# Now really running
ra.run(
) # This would trigger an exception in case of failure, or application status != 0
# Return OK assuming that subsequent module will spot problems
self.setApplicationStatus('%s (Root) Successful' %
self.applicationName)
return S_OK()
except LbRunError as lbre: # This is the case for lb-run/environment errors
self.setApplicationStatus(repr(lbre))
return S_ERROR(DErrno.EWMSRESC, str(lbre))
except LHCbApplicationError as lbae: # This is the case for real application errors
self.setApplicationStatus(repr(lbae))
return S_ERROR(str(lbae))
except LHCbDIRACError as lbde: # This is the case for LHCbDIRAC errors (e.g. subProcess call failed)
self.setApplicationStatus(repr(lbde))
return S_ERROR(str(lbde))
except Exception as e: # pylint:disable=broad-except
self.log.exception("Failure in RootApplication execute module",
lException=e)
return S_ERROR("Error in RootApplication module")
finally:
super(RootApplication, self).finalize(self.version)