def GetNewLDLibs(systemConfig, application, applicationVersion):
""" Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included
@param systemConfig: System config used for the job
@param application: name of the application considered
@param applicationVersion: version of the application considered
@return: new LD_LIBRARY_PATH
"""
new_ld_lib_path = ""
deps = resolveDepsTar(systemConfig, application, applicationVersion)
for dep in deps:
depfolder = dep.replace(".tgz","").replace(".tar.gz","")
res = getSoftwareFolder(depfolder)
if not res['OK']:
continue
basedepfolder = res['Value']
if os.path.exists(os.path.join(basedepfolder, "lib")):
gLogger.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "lib")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if os.path.exists(os.path.join(basedepfolder, "LDLibs")):
gLogger.verbose("Found lib folder in %s" % (depfolder))
newlibdir = os.path.join(basedepfolder, "LDLibs")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if os.environ.has_key("LD_LIBRARY_PATH"):
if new_ld_lib_path:
new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ["LD_LIBRARY_PATH"]
else:
new_ld_lib_path = os.environ["LD_LIBRARY_PATH"]
return new_ld_lib_path
def execute(self):
""" Run the module
"""
self.result = self.resolveInputVariables()
if not self.systemConfig:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationName:
self.result = S_ERROR("Pythia version name not given")
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
appDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall'% (self.systemConfig,
self.applicationName,
self.applicationVersion), '')
appDir = appDir.replace(".tgz","").replace(".tar.gz","")
res = getSoftwareFolder(appDir)
if not res['OK']:
self.setApplicationStatus('Pythia: Could not find neither local area not shared area install')
return res
myappDir = res['Value']
deptar = resolveDepsTar(self.systemConfig, self.applicationName, self.applicationVersion)[0]
depdir = deptar.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(depdir)
if not res['OK']:
return res
path = res['Value']
if not os.path.exists(path + "/%s.ep" % depdir):
return S_ERROR("Lumi files not found")
originpath = os.path.join(path, "/%s.ep" % depdir)
randomName = '/tmp/LumiFile-' + GenRandString(8)
try:
os.mkdir(randomName)
except Exception, x:
return S_ERROR("Could not create temp dir: %s %s" % (Exception, x))
def GetNewPATH(systemConfig, application, applicationVersion):
""" Same as L{GetNewLDLibs},but for the PATH
"""
new_path = ""
deps = resolveDepsTar(systemConfig, application, applicationVersion)
for dep in deps:
depfolder = dep.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(depfolder)
if not res['OK']:
continue
depfolder = res['Value']
if os.path.exists(os.path.join(depfolder, "bin")):
gLogger.verbose("Found bin folder in %s" % (depfolder))
newpathdir = os.path.join(depfolder, "bin")
new_path = newpathdir
if os.environ.has_key("PATH"):
if new_path:
new_path = new_path + ":%s" % os.environ["PATH"]
else:
new_path = os.environ["PATH"]
return new_path
def execute(self):
""" Run the module
"""
self.result = self.resolveInputVariables()
if not self.systemConfig:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
tomatoDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig, "tomato",
self.applicationVersion), '')
if not tomatoDir:
self.log.error('Could not get Tomato tar ball name, cannot proceed')
return S_ERROR('Problem accessing CS')
tomatoDir = tomatoDir.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(tomatoDir)
if not res['Value']:
self.setApplicationStatus('Tomato: Could not find neither local area not shared area install')
return res
myTomatoDir = res['Value']
res = self.prepareMARLIN_DLL(myTomatoDir)
if not res['OK']:
self.log.error('Failed building MARLIN_DLL: %s' % res['Message'])
self.setApplicationStatus('Failed to setup MARLIN_DLL')
return S_ERROR('Something wrong with software installation')
self.envdict['MARLIN_DLL'] = res['Value']
deps = resolveDepsTar(self.systemConfig, "tomato", self.applicationVersion)
for dep in deps:
if dep.lower().count('marlin'):
marlindir = dep.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(marlindir)
if not res['OK']:
self.log.error('Marlin was not found in software directory')
return res
else:
self.envdict['MarlinDIR'] = res['Value']
break
new_ldlibs = ''
if os.environ.has_key('LD_LIBRARY_PATH'):
new_ldlibs = os.path.join(myTomatoDir, 'LDLibs') + ":%s" % os.environ['LD_LIBRARY_PATH']
else:
new_ldlibs = os.path.join(myTomatoDir, 'LDLibs')
self.envdict['LD_LIB_PATH'] = new_ldlibs
res = self.GetInputFiles()
if not res['OK']:
self.log.error(res['Message'])
return res
listofslcio = res['Value']
finalXML = 'tomato.xml'
res = PrepareTomatoSalad(None, finalXML, self.OutputFile, listofslcio, self.collection)
if not res['OK']:
self.log.error('Could not prepare the Tomato XML: %s' % res['Message'])
self.setApplicationStatus('Failed to setup Tomato')
return S_ERROR('Failed to setup Tomato')
self.result = self.runMarlin(finalXML, self.envdict)
if not self.result['OK']:
self.log.error('Something wrong during running: %s' % self.result['Message'])
self.setApplicationStatus('Error during running %s' % self.applicationName)
return S_ERROR('Failed to run %s' % self.applicationName)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
self.log.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' % (self.applicationName))
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
self.log.info( "Status after the application execution is %s" % str( status ) )
return self.finalStatusReport(status)
def execute(self):
""" Called by Agent
Executes the following
- resolve input variables
- resolve installation location
- resolve dependencies location (beam_spectra)
- get processlist if needed
- define output file name
- prepare whizard.in
- make magic
@return: S_OK(), S_ERROR()
"""
self.result = self.resolveInputVariables()
if not self.systemConfig:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('Whizard should not proceed as previous step did not end properly')
#if self.debug:
# self.excludeAllButEventString = False
whizardDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall'%(self.systemConfig, "whizard",
self.applicationVersion), '')
if not whizardDir:
self.log.error('Could not get info from CS')
self.setApplicationStatus('Failed finding info from CS')
return S_ERROR('Failed finding info from CS')
whizardDir = whizardDir.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(whizardDir)
if not res['OK']:
self.log.error("Failed getting software folder", res['Message'])
self.setApplicationStatus('Failed finding software')
return res
mySoftDir = res['Value']
###Remove libc
removeLibc(mySoftDir + "/lib")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = GetNewLDLibs(self.systemConfig, "whizard", self.applicationVersion)
#Don't forget to prepend the application's libs
new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path
### Resolve dependencies (look for beam_spectra)
deps = resolveDepsTar(self.systemConfig, "whizard", self.applicationVersion)
path_to_beam_spectra = ""
path_to_gridfiles = ""
for dep in deps:
depfolder = dep.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(depfolder)
if not res['OK']:
self.log.error("Failed getting software folder", res['Message'])
self.setApplicationStatus('Failed finding software')
return res
depfolder = res['Value']
if depfolder.count("beam_spectra"):
path_to_beam_spectra = depfolder
elif depfolder.count("gridfiles"):
path_to_gridfiles = depfolder
##Env variables needed to run whizard: avoids hard coded locations
os.environ['LUMI_LINKER'] = path_to_beam_spectra + "/lumi_linker_000"
os.environ['PHOTONS_B1'] = path_to_beam_spectra + "/photons_beam1_linker_000"
os.environ['PHOTONS_B2'] = path_to_beam_spectra + "/photons_beam2_linker_000"
os.environ['EBEAM'] = path_to_beam_spectra + "/ebeam_in_linker_000"
os.environ['PBEAM'] = path_to_beam_spectra + "/pbeam_in_linker_000"
os.environ['LUMI_EE_LINKER'] = path_to_beam_spectra + "/lumi_ee_linker_000"
os.environ['LUMI_EG_LINKER'] = path_to_beam_spectra + "/lumi_eg_linker_000"
os.environ['LUMI_GE_LINKER'] = path_to_beam_spectra + "/lumi_ge_linker_000"
os.environ['LUMI_GG_LINKER'] = path_to_beam_spectra + "/lumi_gg_linker_000"
list_of_gridfiles = []
if path_to_gridfiles and self.useGridFiles:
tmp_list_of_gridfiles = [os.path.join(path_to_gridfiles, item) for item in os.listdir(path_to_gridfiles)]
gridfilesfound = False
for path in tmp_list_of_gridfiles:
if os.path.isdir(path) and path.count(str(self.energy)): #Here look for a sub directory for the energy related grid files
list_of_gridfiles = [os.path.join(path, item) for item in os.listdir(path)]
gridfilesfound = True
self.log.info('Found grid files specific for energy %s' % self.energy)
break
if not gridfilesfound:
self.log.info("Will use generic grid files found, hope the energy is set right")
list_of_gridfiles = [item for item in glob.glob(os.path.join(path_to_gridfiles, "*.grb")) + glob.glob(os.path.join(path_to_gridfiles, "*.grc"))]
template = False
if self.SteeringFile.count("template"):
template = True
## Get from process file the proper whizard.in file
if self.getProcessInFile:
whizardin = ""
res = self.obtainProcessList()
if not res['OK']:
self.log.error("Could not obtain process list")
self.setApplicationStatus('Failed getting processlist')
return res
whizardin = self.processlist.getInFile(self.evttype)
if not whizardin:
self.log.error("Whizard input file was not found in process list, cannot proceed")
self.setApplicationStatus('Whizard input file was not found')
return S_ERROR("Error while resolving whizard input file")
if whizardin.count("template"):
template = True
try:
shutil.copy("%s/%s" % (mySoftDir, whizardin), "./whizardnew.in")
self.SteeringFile = "whizardnew.in"
except:
self.log.error("Could not copy %s from %s" % (whizardin, mySoftDir))
self.setApplicationStatus('Failed getting whizard.in file')
return S_ERROR("Failed to obtain %s" % whizardin)
##Check existence of Les Houches input file
leshouchesfiles = ''
if not os.path.exists("LesHouches.msugra_1.in"):
if self.susymodel:
if self.susymodel == 1:
if os.path.exists("%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir)):
leshouchesfiles = "%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir)
if self.susymodel == 2:
if os.path.exists("%s/LesHouches_chne.msugra_1.in" % (mySoftDir)):
leshouchesfiles = "%s/LesHouches_chne.msugra_1.in" % (mySoftDir)
if self.Model:
if self.genmodel.hasModel(self.Model)['OK']:
if self.genmodel.getFile(self.Model)['OK']:
if os.path.exists("%s/%s" % (mySoftDir, self.genmodel.getFile(self.Model)['Value'])):
leshouchesfiles = "%s/%s" % (mySoftDir, self.genmodel.getFile(self.Model)['Value'])
else:
self.log.error("Request LesHouches file is missing, cannot proceed")
self.setApplicationStatus("LesHouches file missing")
return S_ERROR("The LesHouches file was not found. Probably you are using a wrong version of whizard.")
else:
self.log.warn("No file found attached to model %s" % self.Model)
else:
self.log.error("Model undefined:",self.Model)
self.setApplicationStatus("Model undefined")
return S_ERROR("No Model %s defined" % self.Model)
else:
leshouchesfiles = "LesHouches.msugra_1.in"
outputfilename = self.evttype
if self.jobindex:
outputfilename = "%s_%s" % (outputfilename, self.jobindex)
if self.optionsdict:
self.log.info("Using: %s" % self.optionsdict)
self.options = WhizardOptions(self.Model)
res = self.options.changeAndReturn(self.optionsdict)
if not res['OK']:
return res
res = self.options.toWhizardDotIn("whizard.in")
elif not template:
res = PrepareWhizardFile(self.SteeringFile, outputfilename, self.energy,
self.RandomSeed, self.NumberOfEvents, self.Lumi,
"whizard.in")
else:
res = PrepareWhizardFileTemplate(self.SteeringFile, outputfilename,
self.parameters, "whizard.in")
if not res['OK']:
self.log.error('Something went wrong with input file generation')
self.setApplicationStatus('Whizard: something went wrong with input file generation')
return S_ERROR('Something went wrong with whizard.in file generation')
foundproceesinwhizardin = res['Value']
scriptName = 'Whizard_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/sh \n')
script.write('#####################################################################\n')
script.write('# Dynamically generated script to run a production or analysis job. #\n')
script.write('#####################################################################\n')
script.write('declare -x PATH=%s:$PATH\n' % mySoftDir)
script.write('declare -x LD_LIBRARY_PATH=%s\n' % new_ld_lib_path)
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write('echo Printing content of whizard.in \n')
script.write('cat whizard.in\n')
script.write('echo =============================\n')
script.write('cp %s/whizard.mdl ./\n' % mySoftDir)
if leshouchesfiles:
if not leshouchesfiles == 'LesHouches.msugra_1.in':
script.write('cp %s ./LesHouches.msugra_1.in\n' % (leshouchesfiles))
script.write('ln -s LesHouches.msugra_1.in fort.71\n')
if len(list_of_gridfiles):
for gridfile in list_of_gridfiles:
script.write('cp %s ./\n' % (gridfile))
script.write('cp %s/whizard.prc ./\n' % mySoftDir)
if self.genlevelcuts:
res = self.makeWhizardDotCut1(self.genlevelcuts)
if not res['OK']:
script.close()
self.log.error("Could not create the cut1 file")
return S_ERROR("Could not create the cut1 file")
script.write('echo =============================\n')
script.write('echo Printing content of whizard.prc \n')
script.write('cat whizard.prc\n')
script.write('echo =============================\n')
extracmd = ""
if not self.debug:
extracmd = "2>/dev/null"
comm = ""
if foundproceesinwhizardin:
comm = 'whizard --simulation_input \'write_events_file = \"%s\"\' %s\n' % (outputfilename, extracmd)
else:
comm = 'whizard --process_input \'process_id =\"%s\"\' --simulation_input \'write_events_file = \"%s\"\' %s\n' % (self.evttype,
outputfilename,
extracmd)
self.log.info("Will run %s" % comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('Whizard %s step %s' %(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit=209715200)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
if not self.result['OK']:
self.log.error("Failed with error %s" % self.result['Message'])
if not os.path.exists(self.applicationLog):
self.log.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' % (self.applicationName))
lumi = ''
message = ""
success = False
###Analyse log file
logfile = file(self.applicationLog)
for line in logfile:
if line.count('! Event sample corresponds to luminosity'):
elems = line.split()
lumi = elems[-1]
if line.count("*** Fatal error:"):
status = 1
message = line
break
elif line.count("PYSTOP"):
status = 1
message = line
break
elif line.count("No matrix element available"):
status = 1
message = line
break
elif line.count("Floating point exception"):
status = 1
message = line
break
elif line.count("Event generation finished."):
success = True
else:
status = 0
if success:
status = 0
else:
status = 1
self.log.info('The sample generated has an equivalent luminosity of %s' % lumi)
if lumi:
self.workflow_commons['Luminosity'] = float(lumi)
else:
status = 1
##Now care for the cross sections
info = {}
res = self.options.getAsDict()
if os.path.exists("whizard.out") and res['OK']:
full_opts_dict = res['Value']
processes = full_opts_dict['process_input']['process_id'].split()
info = {}
info['xsection'] = {}
processes.append('sum')
inf = open("whizard.out", "r")
for line in inf:
line = line.rstrip()
for process in processes:
if not process:
continue
if line.count(" %s " % process):
info['xsection'][process] = {}
line = line.lstrip()
crosssection = line.split()[1]
err_crosssection = line.split()[2]
frac = line.split()[4]
info['xsection'][process]['xsection'] = float(crosssection)
info['xsection'][process]['err_xsection'] = float(err_crosssection)
info['xsection'][process]['fraction'] = float(frac)
inf.close()
if info:
if 'Info' not in self.workflow_commons:
self.workflow_commons['Info'] = info
else:
self.workflow_commons['Info'].update(info)
self.log.info( "Status after the application execution is %s" % str( status ) )
messageout = 'Whizard %s Successful' % (self.applicationVersion)
failed = False
if status != 0:
self.log.error( "Whizard execution completed with errors:" )
failed = True
else:
self.log.info( "Whizard execution completed successfully")
###Deal with output file
if len(self.OutputFile):
if os.path.exists(outputfilename + ".001.stdhep"):
ofnames = glob.glob('*.stdhep')
if len(ofnames) > 1:
basename = self.OutputFile.split(".stdhep")[0]
i = 0
for f in ofnames:
i += 1
name = basename + "_" + str(i) + ".stdhep"
os.rename(f, name)
else:
os.rename(outputfilename + ".001.stdhep", self.OutputFile)
else:
self.log.error( "Whizard execution did not produce a stdhep file" )
self.setApplicationStatus('Whizard %s Failed to produce STDHEP file' % (self.applicationVersion))
messageout = 'Whizard Failed to produce STDHEP file'
if not self.ignoreapperrors:
return S_ERROR(messageout)
if failed == True:
self.log.error( "==================================\n StdError:\n" )
self.log.error( message )
self.setApplicationStatus('%s Exited With Status %s' % (self.applicationName, status))
self.log.error('Whizard Exited With Status %s' % (status))
messageout = 'Whizard Exited With Status %s' % (status)
if not self.ignoreapperrors:
return S_ERROR(messageout)
else:
self.setApplicationStatus(messageout)
return S_OK( { "OutputFile": self.OutputFile } )