def getNewLDLibs(platform, application, applicationVersion):
""" Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included
:param string platform: System config used for the job
:param string application: name of the application considered
:param string applicationVersion: version of the application considered
:return: new LD_LIBRARY_PATH
"""
log = gLogger.getSubLogger("GetLDLibs")
log.verbose("Getting all lib folders")
new_ld_lib_path = ""
deps = resolveDeps(platform, application, applicationVersion)
for dep in deps:
res = getSoftwareFolder(platform, dep["app"], dep['version'])
if not res['OK']:
continue
basedepfolder = res['Value']
if os.path.exists(os.path.join(basedepfolder, "lib")):
log.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "lib")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if os.path.exists(os.path.join(basedepfolder, "LDLibs")):
log.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "LDLibs")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if "LD_LIBRARY_PATH" in os.environ:
if new_ld_lib_path:
new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ["LD_LIBRARY_PATH"]
else:
new_ld_lib_path = os.environ["LD_LIBRARY_PATH"]
return new_ld_lib_path
def getNewPATH(platform, application, applicationVersion):
""" Same as :func:`getNewLDLibs`,but for the PATH
:param string platform: System config used for the job
:param string application: name of the application considered
:param string applicationVersion: version of the application considered
:return: new PATH
"""
log = gLogger.getSubLogger("GetPaths")
log.verbose("Getting all PATH folders")
new_path = ""
deps = resolveDeps(platform, application, applicationVersion)
for dep in deps:
res = getSoftwareFolder(platform, dep['app'], dep['version'])
if not res['OK']:
continue
depfolder = res['Value']
if os.path.exists(os.path.join(depfolder, "bin")):
log.verbose("Found bin folder in %s" % (depfolder))
newpathdir = os.path.join(depfolder, "bin")
new_path = newpathdir
if "PATH" in os.environ:
if new_path:
new_path = new_path + ":%s" % os.environ["PATH"]
else:
new_path = os.environ["PATH"]
return new_path
def getNewPATH(platform, application, applicationVersion):
""" Same as :func:`getNewLDLibs`,but for the PATH
:param string platform: System config used for the job
:param string application: name of the application considered
:param string applicationVersion: version of the application considered
:return: new PATH
"""
log = gLogger.getSubLogger("GetPaths")
log.verbose("Getting all PATH folders")
new_path = ""
deps = resolveDeps(platform, application, applicationVersion)
for dep in deps:
res = getSoftwareFolder(platform, dep['app'], dep['version'])
if not res['OK']:
continue
depfolder = res['Value']
if os.path.exists(os.path.join(depfolder, "bin")):
log.verbose("Found bin folder in %s" % (depfolder))
newpathdir = os.path.join(depfolder, "bin")
new_path = newpathdir
if "PATH" in os.environ:
if new_path:
new_path = new_path + ":%s" % os.environ["PATH"]
else:
new_path = os.environ["PATH"]
return new_path
def getEnvScript(self, sysconfig, appname, appversion):
""" Called if CVMFS install is not here
"""
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['Value']:
self.setApplicationStatus('Tomato: Could not find neither local area not shared area install')
return res
myTomatoDir = res['Value']
deps = resolveDeps(sysconfig, "tomato", appversion)
for dep in deps:
if dep["app"].lower() == 'marlin':
res = getSoftwareFolder(sysconfig, "marlin", dep["version"])
if not res['OK']:
LOG.error('Marlin was not found in software directory')
return res
else:
myMarlinDir = res['Value']
break
env_script_name = "TomatoEnv.sh"
script = open(env_script_name, "w")
script.write("#!/bin/sh\n")
script.write('###########################################################\n')
script.write('# Dynamically generated script to get the Env for Tomato. #\n')
script.write('###########################################################\n')
script.write("declare -x PATH=%s/Executable:$PATH\n" % myMarlinDir)
script.write('declare -x ROOTSYS=%s/ROOT\n' % (myMarlinDir))
script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs\n' % (myMarlinDir))
script.write("declare -x LD_LIBRARY_PATH=%s/LDLibs:$LD_LIBRARY_PATH\n" % myTomatoDir)
script.close()
return S_OK(os.path.abspath(env_script_name))
def __init__(self, argumentsDict):
""" Standard constructor
Defines, from dictionary of job parameters passed, a set of members to hold e.g. the
applications and the system config.
Also determines the SharedArea and LocalArea.
"""
self.ops = Operations()
self.job = argumentsDict.get('Job', {})
self.ce = argumentsDict.get('CE', {})
self.source = argumentsDict.get('Source', {})
apps = []
if self.job.has_key('SoftwarePackages'):
if type( self.job['SoftwarePackages'] ) == type(''):
apps = [self.job['SoftwarePackages']]
elif type( self.job['SoftwarePackages'] ) == type([]):
apps = self.job['SoftwarePackages']
self.jobConfig = ''
if 'SystemConfig' in self.job:
self.jobConfig = self.job['SystemConfig']
elif 'Platform' in self.job:
self.jobConfig = self.job['Platform']
else:
self.jobConfig = natOS.CMTSupportedConfig()[0]
self.apps = []
for app in apps:
DIRAC.gLogger.verbose( 'Requested Package %s' % app )
app = tuple(app.split('.'))
if len(app) > 2:
tempapp = app
app = []
app.append(tempapp[0])
app.append(".".join(tempapp[1:]))
deps = resolveDeps(self.jobConfig, app[0], app[1])
for dep in deps:
depapp = (dep['app'], dep['version'])
self.apps.append(depapp)
self.apps.append(app)
self.ceConfigs = []
if 'CompatiblePlatforms' in self.ce:
self.ceConfigs = self.ce['CompatiblePlatforms']
if type(self.ceConfigs) == type(''):
self.ceConfigs = [self.ceConfigs]
#else:
### Use always the list of compatible platform.
self.ceConfigs = natOS.CMTSupportedConfig()
self.sharedArea = getSharedAreaLocation()
DIRAC.gLogger.info("SharedArea is %s" % self.sharedArea)
self.localArea = getLocalAreaLocation()
DIRAC.gLogger.info("LocalArea is %s" % self.localArea)
def __init__(self, argumentsDict):
""" Standard constructor
Defines, from dictionary of job parameters passed, a set of members to hold e.g. the
applications and the system config.
Also determines the SharedArea and LocalArea.
"""
self.ops = Operations()
self.job = argumentsDict.get('Job', {})
self.ce = argumentsDict.get('CE', {})
self.source = argumentsDict.get('Source', {})
apps = []
if 'SoftwarePackages' in self.job:
if isinstance(self.job['SoftwarePackages'], basestring):
apps = [self.job['SoftwarePackages']]
elif isinstance(self.job['SoftwarePackages'], list):
apps = self.job['SoftwarePackages']
self.jobConfig = ''
if 'SystemConfig' in self.job:
self.jobConfig = self.job['SystemConfig']
elif 'Platform' in self.job:
self.jobConfig = self.job['Platform']
else:
self.jobConfig = natOS.CMTSupportedConfig()[0]
self.apps = []
for app in apps:
DIRAC.gLogger.verbose('Requested Package %s' % app)
app = tuple(app.split('.'))
if len(app) > 2:
tempapp = app
app = []
app.append(tempapp[0])
app.append(".".join(tempapp[1:]))
deps = resolveDeps(self.jobConfig, app[0], app[1])
for dep in deps:
depapp = (dep['app'], dep['version'])
self.apps.append(depapp)
self.apps.append(app)
self.ceConfigs = []
if 'CompatiblePlatforms' in self.ce:
self.ceConfigs = self.ce['CompatiblePlatforms']
if isinstance(self.ceConfigs, basestring):
self.ceConfigs = [self.ceConfigs]
#else:
### Use always the list of compatible platform.
self.ceConfigs = natOS.CMTSupportedConfig()
self.sharedArea = getSharedAreaLocation()
DIRAC.gLogger.info("SharedArea is %s" % self.sharedArea)
self.localArea = getLocalAreaLocation()
DIRAC.gLogger.info("LocalArea is %s" % self.localArea)
def installDependencies(app, config, areas):
"""install dependencies for application"""
appName = app[0].lower()
appVersion = app[1]
deps = resolveDeps(config, appName, appVersion)
for dep in deps:
depapp = [ dep["app"], dep["version"] ]
resDep = installInAnyArea(areas, depapp, config)
if not resDep['OK']:
return S_ERROR("Failed to install dependency: %s" % str(depapp))
return S_OK()
def installDependencies(app, config, areas):
"""install dependencies for application"""
appName = app[0].lower()
appVersion = app[1]
deps = resolveDeps(config, appName, appVersion)
for dep in deps:
depapp = [dep["app"], dep["version"]]
resDep = installInAnyArea(areas, depapp, config)
if not resDep['OK']:
return S_ERROR("Failed to install dependency: %s" % str(depapp))
return S_OK()
def execute(self):
""" Run the module
"""
self.result = self.resolveInputVariables()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationName:
self.result = S_ERROR("Pythia version name not given")
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
self.log.error("Failed to resolve the input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
self.log.error('Failed finding the software area')
self.setApplicationStatus('Could find neither local area nor shared area install')
return res
myappDir = res['Value']
deptar = resolveDeps(self.platform, self.applicationName, self.applicationVersion)[0]
res = getSoftwareFolder(self.platform, deptar["app"], deptar["version"])
if not res['OK']:
self.log.error("Failed finding the dependency location")
return res
path = res['Value']
if not os.path.exists("%s.ep" % path):
self.log.error('Could not find the lumi files!')
return S_ERROR("Lumi files not found")
originpath = "%s.ep" % path
randomName = '/tmp/LumiFile-' + generateRandomString(8)
try:
os.mkdir(randomName)
except EnvironmentError, x:
self.log.error("Failed setting up the temp directory")
return S_ERROR("Could not create temp dir: %s" % str(x))
def execute(self):
""" Run the module
"""
self.result = self.resolveInputVariables()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationName:
self.result = S_ERROR("Pythia version name not given")
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
LOG.error("Failed to resolve the input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
LOG.error('Failed finding the software area')
self.setApplicationStatus('Could find neither local area nor shared area install')
return res
myappDir = res['Value']
deptar = resolveDeps(self.platform, self.applicationName, self.applicationVersion)[0]
res = getSoftwareFolder(self.platform, deptar["app"], deptar["version"])
if not res['OK']:
LOG.error("Failed finding the dependency location")
return res
path = res['Value']
if not os.path.exists("%s.ep" % path):
LOG.error('Could not find the lumi files!')
return S_ERROR("Lumi files not found")
originpath = "%s.ep" % path
randomName = '/tmp/LumiFile-' + generateRandomString(8)
try:
os.mkdir(randomName)
except EnvironmentError, x:
LOG.error("Failed setting up the temp directory")
return S_ERROR("Could not create temp dir: %s" % str(x))
def getEnvScript(self, sysconfig, appname, appversion):
""" Called if CVMFS install is not here
"""
res = getSoftwareFolder(sysconfig, appname, appversion)
if not res['Value']:
self.setApplicationStatus(
'Tomato: Could not find neither local area not shared area install'
)
return res
myTomatoDir = res['Value']
deps = resolveDeps(sysconfig, "tomato", appversion)
for dep in deps:
if dep["app"].lower() == 'marlin':
res = getSoftwareFolder(sysconfig, "marlin", dep["version"])
if not res['OK']:
self.log.error(
'Marlin was not found in software directory')
return res
else:
myMarlinDir = res['Value']
break
env_script_name = "TomatoEnv.sh"
script = open(env_script_name, "w")
script.write("#!/bin/sh\n")
script.write(
'###########################################################\n')
script.write(
'# Dynamically generated script to get the Env for Tomato. #\n')
script.write(
'###########################################################\n')
script.write("declare -x PATH=%s/Executable:$PATH\n" % myMarlinDir)
script.write('declare -x ROOTSYS=%s/ROOT\n' % (myMarlinDir))
script.write('declare -x LD_LIBRARY_PATH=$ROOTSYS/lib:%s/LDLibs\n' %
(myMarlinDir))
script.write(
"declare -x LD_LIBRARY_PATH=%s/LDLibs:$LD_LIBRARY_PATH\n" %
myTomatoDir)
script.close()
return S_OK(os.path.abspath(env_script_name))
def getNewLDLibs(platform, application, applicationVersion):
""" Prepare the LD_LIBRARY_PATH environment variable: make sure all lib folder are included
:param string platform: System config used for the job
:param string application: name of the application considered
:param string applicationVersion: version of the application considered
:return: new LD_LIBRARY_PATH
"""
log = gLogger.getSubLogger("GetLDLibs")
log.verbose("Getting all lib folders")
new_ld_lib_path = ""
deps = resolveDeps(platform, application, applicationVersion)
for dep in deps:
res = getSoftwareFolder(platform, dep["app"], dep['version'])
if not res['OK']:
continue
basedepfolder = res['Value']
if os.path.exists(os.path.join(basedepfolder, "lib")):
log.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "lib")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if os.path.exists(os.path.join(basedepfolder, "LDLibs")):
log.verbose("Found lib folder in %s" % (basedepfolder))
newlibdir = os.path.join(basedepfolder, "LDLibs")
new_ld_lib_path = newlibdir
####Remove the libc
removeLibc(new_ld_lib_path)
if "LD_LIBRARY_PATH" in os.environ:
if new_ld_lib_path:
new_ld_lib_path = new_ld_lib_path + ":%s" % os.environ[
"LD_LIBRARY_PATH"]
else:
new_ld_lib_path = os.environ["LD_LIBRARY_PATH"]
return new_ld_lib_path
def runIt(self):
""" Called by Agent
Executes the following
- resolve input variables
- resolve installation location
- resolve dependencies location (beam_spectra)
- get processlist if needed
- define output file name
- prepare whizard.in
- make magic
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
LOG.error("Failed to resolve input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('Whizard should not proceed as previous step did not end properly')
#if self.debug:
# self.excludeAllButEventString = False
res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
LOG.error("Failed getting software folder", res['Message'])
self.setApplicationStatus('Failed finding software')
return res
mySoftDir = res['Value']
###Remove libc
removeLibc(mySoftDir + "/lib")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName, self.applicationVersion)
#Don't forget to prepend the application's libs
new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path
### Resolve dependencies (look for beam_spectra)
deps = resolveDeps(self.platform, self.applicationName, self.applicationVersion)
path_to_beam_spectra = ""
path_to_gridfiles = ""
for dep in deps:
res = getSoftwareFolder(self.platform, dep[ "app" ], dep['version'])
if not res['OK']:
LOG.error("Failed getting software folder", res['Message'])
self.setApplicationStatus('Failed finding software')
return res
depfolder = res['Value']
if dep["app"] == "beam_spectra":
path_to_beam_spectra = depfolder
elif dep["app"] == "gridfiles":
path_to_gridfiles = depfolder
##Env variables needed to run whizard: avoids hard coded locations
os.environ['LUMI_LINKER'] = path_to_beam_spectra + "/lumi_linker_000"
os.environ['PHOTONS_B1'] = path_to_beam_spectra + "/photons_beam1_linker_000"
os.environ['PHOTONS_B2'] = path_to_beam_spectra + "/photons_beam2_linker_000"
os.environ['EBEAM'] = path_to_beam_spectra + "/ebeam_in_linker_000"
os.environ['PBEAM'] = path_to_beam_spectra + "/pbeam_in_linker_000"
os.environ['LUMI_EE_LINKER'] = path_to_beam_spectra + "/lumi_ee_linker_000"
os.environ['LUMI_EG_LINKER'] = path_to_beam_spectra + "/lumi_eg_linker_000"
os.environ['LUMI_GE_LINKER'] = path_to_beam_spectra + "/lumi_ge_linker_000"
os.environ['LUMI_GG_LINKER'] = path_to_beam_spectra + "/lumi_gg_linker_000"
list_of_gridfiles = []
if path_to_gridfiles and self.useGridFiles:
tmp_list_of_gridfiles = [os.path.join(path_to_gridfiles, item) for item in os.listdir(path_to_gridfiles)]
gridfilesfound = False
for path in tmp_list_of_gridfiles:
if os.path.isdir(path) and path.count(str(self.energy)):
#Here look for a sub directory for the energy related grid files
list_of_gridfiles = [os.path.join(path, item) for item in os.listdir(path)]
gridfilesfound = True
LOG.info('Found grid files specific for energy %s' % self.energy)
break
if not gridfilesfound:
LOG.info("Will use generic grid files found, hope the energy is set right")
list_of_gridfiles = [item for item in glob.glob(os.path.join(path_to_gridfiles, "*.grb")) + glob.glob(os.path.join(path_to_gridfiles, "*.grc"))]
template = False
if self.SteeringFile.count("template"):
template = True
## Get from process file the proper whizard.in file
if self.getProcessInFile:
whizardin = ""
res = self.obtainProcessList()
if not res['OK']:
LOG.error("Could not obtain process list")
self.setApplicationStatus('Failed getting processlist')
return res
whizardin = self.processlist.getInFile(self.evttype)
if not whizardin:
LOG.error("Whizard input file was not found in process list, cannot proceed")
self.setApplicationStatus('Whizard input file was not found')
return S_ERROR("Error while resolving whizard input file")
if whizardin.count("template"):
template = True
try:
shutil.copy("%s/%s" % (mySoftDir, whizardin), "./whizardnew.in")
self.SteeringFile = "whizardnew.in"
except EnvironmentError:
LOG.error("Could not copy %s from %s" % (whizardin, mySoftDir))
self.setApplicationStatus('Failed getting whizard.in file')
return S_ERROR("Failed to obtain %s" % whizardin)
##Check existence of Les Houches input file
leshouchesfiles = ''
if not os.path.exists("LesHouches.msugra_1.in"):
if self.susymodel:
if self.susymodel == 1:
if os.path.exists("%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir)):
leshouchesfiles = "%s/LesHouches_slsqhh.msugra_1.in" % (mySoftDir)
if self.susymodel == 2:
if os.path.exists("%s/LesHouches_chne.msugra_1.in" % (mySoftDir)):
leshouchesfiles = "%s/LesHouches_chne.msugra_1.in" % (mySoftDir)
if self.Model:
if self.genmodel.hasModel(self.Model)['OK']:
if self.genmodel.getFile(self.Model)['OK']:
if os.path.exists("%s/%s" % (mySoftDir, self.genmodel.getFile(self.Model)['Value'])):
leshouchesfiles = "%s/%s" % (mySoftDir, self.genmodel.getFile(self.Model)['Value'])
else:
LOG.error("Request LesHouches file is missing, cannot proceed")
self.setApplicationStatus("LesHouches file missing")
return S_ERROR("The LesHouches file was not found. Probably you are using a wrong version of whizard.")
else:
LOG.warn("No file found attached to model %s" % self.Model)
else:
LOG.error("Model undefined:", self.Model)
self.setApplicationStatus("Model undefined")
return S_ERROR("No Model %s defined" % self.Model)
else:
leshouchesfiles = "LesHouches.msugra_1.in"
outputfilename = self.evttype
if self.optionsdict:
LOG.info("Using: %s" % self.optionsdict)
self.options = WhizardOptions(self.Model)
res = self.options.changeAndReturn(self.optionsdict)
if not res['OK']:
return res
res = self.options.toWhizardDotIn("whizard.in")
elif not template:
res = prepareWhizardFile(self.SteeringFile, outputfilename, self.energy,
self.RandomSeed, self.NumberOfEvents, self.Lumi,
"whizard.in")
else:
res = prepareWhizardFileTemplate(self.SteeringFile, outputfilename,
self.parameters, "whizard.in")
if not res['OK']:
LOG.error('Something went wrong with input file generation')
self.setApplicationStatus('Whizard: something went wrong with input file generation')
return S_ERROR('Something went wrong with whizard.in file generation')
foundproceesinwhizardin = res['Value']
scriptName = 'Whizard_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/sh \n')
script.write('#####################################################################\n')
script.write('# Dynamically generated script to run a production or analysis job. #\n')
script.write('#####################################################################\n')
script.write('declare -x PATH=%s:$PATH\n' % mySoftDir)
script.write('declare -x LD_LIBRARY_PATH=%s\n' % new_ld_lib_path)
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write('echo Printing content of whizard.in \n')
script.write('cat whizard.in\n')
script.write('echo =============================\n')
script.write('cp %s/whizard.mdl ./\n' % mySoftDir)
if leshouchesfiles:
if not leshouchesfiles == 'LesHouches.msugra_1.in':
script.write('cp %s ./LesHouches.msugra_1.in\n' % (leshouchesfiles))
script.write('ln -s LesHouches.msugra_1.in fort.71\n')
if len(list_of_gridfiles):
for gridfile in list_of_gridfiles:
script.write('cp %s ./\n' % (gridfile))
script.write('cp %s/whizard.prc ./\n' % mySoftDir)
if self.genlevelcuts:
res = self.makeWhizardDotCut1()
if not res['OK']:
script.close()
LOG.error("Could not create the cut1 file")
return S_ERROR("Could not create the cut1 file")
script.write('echo =============================\n')
script.write('echo Printing content of whizard.prc \n')
script.write('cat whizard.prc\n')
script.write('echo =============================\n')
extracmd = ""
if not self.debug:
extracmd = "2>/dev/null"
comm = ""
if foundproceesinwhizardin:
comm = 'whizard --simulation_input \'write_events_file = \"%s\"\'' % (outputfilename)
else:
comm = 'whizard --process_input \'process_id =\"%s\"\' --simulation_input \'write_events_file = \"%s\"\' ' % (self.evttype,
outputfilename)
comm = "%s %s %s\n" % (comm, self.extraCLIarguments, extracmd)
LOG.info("Will run %s" % comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0o755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('Whizard %s step %s' %(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit=209715200)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
if not self.result['OK']:
LOG.error("Failed with error %s" % self.result['Message'])
if not os.path.exists(self.applicationLog):
LOG.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' % (self.applicationName))
lumi = ''
message = ""
success = False
###Analyse log file
with open(self.applicationLog) as logfile:
for line in logfile:
if line.count('! Event sample corresponds to luminosity'):
elems = line.split()
lumi = elems[-1]
if line.count("*** Fatal error:"):
status = 1
message = line
break
elif line.count("PYSTOP"):
status = 1
message = line
break
elif line.count("No matrix element available"):
status = 1
message = line
break
elif line.count("Floating point exception"):
status = 1
message = line
break
elif line.count("Event generation finished."):
success = True
else:
status = 0
if success:
status = 0
else:
status = 1
LOG.info('The sample generated has an equivalent luminosity of %s' % lumi)
if lumi:
self.workflow_commons['Luminosity'] = float(lumi)
else:
status = 1
##Now care for the cross sections
info = {}
res = self.options.getAsDict()
if os.path.exists("whizard.out") and res['OK']:
full_opts_dict = res['Value']
processes = full_opts_dict['process_input']['process_id'].split()
info = {}
info['xsection'] = {}
processes.append('sum')
with open("whizard.out", "r") as inf:
for line in inf:
line = line.rstrip()
for process in processes:
if not process:
continue
if line.count(" %s " % process):
info['xsection'][process] = {}
line = line.lstrip()
crosssection = line.split()[1]
err_crosssection = line.split()[2]
frac = line.split()[4]
info['xsection'][process]['xsection'] = float(crosssection)
info['xsection'][process]['err_xsection'] = float(err_crosssection)
info['xsection'][process]['fraction'] = float(frac)
if info:
if 'Info' not in self.workflow_commons:
self.workflow_commons['Info'] = info
else:
self.workflow_commons['Info'].update(info)
LOG.info("Status after the application execution is %s" % str(status))
messageout = 'Whizard %s Successful' % (self.applicationVersion)
failed = False
if status != 0:
LOG.error("Whizard execution completed with errors:")
failed = True
else:
LOG.info("Whizard execution completed successfully")
###Deal with output file
if len(self.OutputFile):
if os.path.exists(outputfilename + ".001.stdhep"):
LOG.notice("Looking for output files")
ofnames = glob.glob(outputfilename+'*.stdhep')
if len(ofnames) > 1:
basename = self.OutputFile.split(".stdhep")[0]
i = 0
for of in ofnames:
i += 1
name = basename + "_" + str(i) + ".stdhep"
os.rename(of, name)
else:
os.rename(outputfilename + ".001.stdhep", self.OutputFile)
else:
LOG.error("Whizard execution did not produce a stdhep file")
self.setApplicationStatus('Whizard %s Failed to produce STDHEP file' % (self.applicationVersion))
messageout = 'Whizard Failed to produce STDHEP file'
if not self.ignoreapperrors:
return S_ERROR(messageout)
if failed is True:
LOG.error("==================================\n StdError:\n")
LOG.error(message)
self.setApplicationStatus('%s Exited With Status %s' % (self.applicationName, status))
LOG.error('Whizard Exited With Status %s' % (status))
messageout = 'Whizard Exited With Status %s' % (status)
if not self.ignoreapperrors:
return S_ERROR(messageout)
else:
self.setApplicationStatus(messageout)
return S_OK( { "OutputFile": self.OutputFile } )
def runIt(self):
""" Called by Agent
Executes the following
- resolve input variables
- resolve installation location
- resolve dependencies location (beam_spectra)
- get processlist if needed
- define output file name
- prepare whizard.in
- make magic
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:",
self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'Whizard should not proceed as previous step did not end properly'
)
#if self.debug:
# self.excludeAllButEventString = False
res = getSoftwareFolder(self.platform, self.applicationName,
self.applicationVersion)
if not res['OK']:
self.log.error("Failed getting software folder", res['Message'])
self.setApplicationStatus('Failed finding software')
return res
mySoftDir = res['Value']
###Remove libc
removeLibc(mySoftDir + "/lib")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName,
self.applicationVersion)
#Don't forget to prepend the application's libs
new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path
### Resolve dependencies (look for beam_spectra)
deps = resolveDeps(self.platform, self.applicationName,
self.applicationVersion)
path_to_beam_spectra = ""
path_to_gridfiles = ""
for dep in deps:
res = getSoftwareFolder(self.platform, dep["app"], dep['version'])
if not res['OK']:
self.log.error("Failed getting software folder",
res['Message'])
self.setApplicationStatus('Failed finding software')
return res
depfolder = res['Value']
if dep["app"] == "beam_spectra":
path_to_beam_spectra = depfolder
elif dep["app"] == "gridfiles":
path_to_gridfiles = depfolder
##Env variables needed to run whizard: avoids hard coded locations
os.environ['LUMI_LINKER'] = path_to_beam_spectra + "/lumi_linker_000"
os.environ[
'PHOTONS_B1'] = path_to_beam_spectra + "/photons_beam1_linker_000"
os.environ[
'PHOTONS_B2'] = path_to_beam_spectra + "/photons_beam2_linker_000"
os.environ['EBEAM'] = path_to_beam_spectra + "/ebeam_in_linker_000"
os.environ['PBEAM'] = path_to_beam_spectra + "/pbeam_in_linker_000"
os.environ[
'LUMI_EE_LINKER'] = path_to_beam_spectra + "/lumi_ee_linker_000"
os.environ[
'LUMI_EG_LINKER'] = path_to_beam_spectra + "/lumi_eg_linker_000"
os.environ[
'LUMI_GE_LINKER'] = path_to_beam_spectra + "/lumi_ge_linker_000"
os.environ[
'LUMI_GG_LINKER'] = path_to_beam_spectra + "/lumi_gg_linker_000"
list_of_gridfiles = []
if path_to_gridfiles and self.useGridFiles:
tmp_list_of_gridfiles = [
os.path.join(path_to_gridfiles, item)
for item in os.listdir(path_to_gridfiles)
]
gridfilesfound = False
for path in tmp_list_of_gridfiles:
if os.path.isdir(path) and path.count(str(self.energy)):
#Here look for a sub directory for the energy related grid files
list_of_gridfiles = [
os.path.join(path, item) for item in os.listdir(path)
]
gridfilesfound = True
self.log.info('Found grid files specific for energy %s' %
self.energy)
break
if not gridfilesfound:
self.log.info(
"Will use generic grid files found, hope the energy is set right"
)
list_of_gridfiles = [
item for item in
glob.glob(os.path.join(path_to_gridfiles, "*.grb")) +
glob.glob(os.path.join(path_to_gridfiles, "*.grc"))
]
template = False
if self.SteeringFile.count("template"):
template = True
## Get from process file the proper whizard.in file
if self.getProcessInFile:
whizardin = ""
res = self.obtainProcessList()
if not res['OK']:
self.log.error("Could not obtain process list")
self.setApplicationStatus('Failed getting processlist')
return res
whizardin = self.processlist.getInFile(self.evttype)
if not whizardin:
self.log.error(
"Whizard input file was not found in process list, cannot proceed"
)
self.setApplicationStatus('Whizard input file was not found')
return S_ERROR("Error while resolving whizard input file")
if whizardin.count("template"):
template = True
try:
shutil.copy("%s/%s" % (mySoftDir, whizardin),
"./whizardnew.in")
self.SteeringFile = "whizardnew.in"
except EnvironmentError:
self.log.error("Could not copy %s from %s" %
(whizardin, mySoftDir))
self.setApplicationStatus('Failed getting whizard.in file')
return S_ERROR("Failed to obtain %s" % whizardin)
##Check existence of Les Houches input file
leshouchesfiles = ''
if not os.path.exists("LesHouches.msugra_1.in"):
if self.susymodel:
if self.susymodel == 1:
if os.path.exists("%s/LesHouches_slsqhh.msugra_1.in" %
(mySoftDir)):
leshouchesfiles = "%s/LesHouches_slsqhh.msugra_1.in" % (
mySoftDir)
if self.susymodel == 2:
if os.path.exists("%s/LesHouches_chne.msugra_1.in" %
(mySoftDir)):
leshouchesfiles = "%s/LesHouches_chne.msugra_1.in" % (
mySoftDir)
if self.Model:
if self.genmodel.hasModel(self.Model)['OK']:
if self.genmodel.getFile(self.Model)['OK']:
if os.path.exists(
"%s/%s" %
(mySoftDir, self.genmodel.getFile(
self.Model)['Value'])):
leshouchesfiles = "%s/%s" % (
mySoftDir, self.genmodel.getFile(
self.Model)['Value'])
else:
self.log.error(
"Request LesHouches file is missing, cannot proceed"
)
self.setApplicationStatus(
"LesHouches file missing")
return S_ERROR(
"The LesHouches file was not found. Probably you are using a wrong version of whizard."
)
else:
self.log.warn("No file found attached to model %s" %
self.Model)
else:
self.log.error("Model undefined:", self.Model)
self.setApplicationStatus("Model undefined")
return S_ERROR("No Model %s defined" % self.Model)
else:
leshouchesfiles = "LesHouches.msugra_1.in"
outputfilename = self.evttype
if self.optionsdict:
self.log.info("Using: %s" % self.optionsdict)
self.options = WhizardOptions(self.Model)
res = self.options.changeAndReturn(self.optionsdict)
if not res['OK']:
return res
res = self.options.toWhizardDotIn("whizard.in")
elif not template:
res = prepareWhizardFile(self.SteeringFile, outputfilename,
self.energy, self.RandomSeed,
self.NumberOfEvents, self.Lumi,
"whizard.in")
else:
res = prepareWhizardFileTemplate(self.SteeringFile, outputfilename,
self.parameters, "whizard.in")
if not res['OK']:
self.log.error('Something went wrong with input file generation')
self.setApplicationStatus(
'Whizard: something went wrong with input file generation')
return S_ERROR(
'Something went wrong with whizard.in file generation')
foundproceesinwhizardin = res['Value']
scriptName = 'Whizard_%s_Run_%s.sh' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/sh \n')
script.write(
'#####################################################################\n'
)
script.write(
'# Dynamically generated script to run a production or analysis job. #\n'
)
script.write(
'#####################################################################\n'
)
script.write('declare -x PATH=%s:$PATH\n' % mySoftDir)
script.write('declare -x LD_LIBRARY_PATH=%s\n' % new_ld_lib_path)
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write('echo Printing content of whizard.in \n')
script.write('cat whizard.in\n')
script.write('echo =============================\n')
script.write('cp %s/whizard.mdl ./\n' % mySoftDir)
if leshouchesfiles:
if not leshouchesfiles == 'LesHouches.msugra_1.in':
script.write('cp %s ./LesHouches.msugra_1.in\n' %
(leshouchesfiles))
script.write('ln -s LesHouches.msugra_1.in fort.71\n')
if len(list_of_gridfiles):
for gridfile in list_of_gridfiles:
script.write('cp %s ./\n' % (gridfile))
script.write('cp %s/whizard.prc ./\n' % mySoftDir)
if self.genlevelcuts:
res = self.makeWhizardDotCut1()
if not res['OK']:
script.close()
self.log.error("Could not create the cut1 file")
return S_ERROR("Could not create the cut1 file")
script.write('echo =============================\n')
script.write('echo Printing content of whizard.prc \n')
script.write('cat whizard.prc\n')
script.write('echo =============================\n')
extracmd = ""
if not self.debug:
extracmd = "2>/dev/null"
comm = ""
if foundproceesinwhizardin:
comm = 'whizard --simulation_input \'write_events_file = \"%s\"\'' % (
outputfilename)
else:
comm = 'whizard --process_input \'process_id =\"%s\"\' --simulation_input \'write_events_file = \"%s\"\' ' % (
self.evttype, outputfilename)
comm = "%s %s %s\n" % (comm, self.extraCLIarguments, extracmd)
self.log.info("Will run %s" % comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('Whizard %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
comm,
callbackFunction=self.redirectLogOutput,
bufferLimit=209715200)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
if not self.result['OK']:
self.log.error("Failed with error %s" % self.result['Message'])
if not os.path.exists(self.applicationLog):
self.log.error(
"Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' %
(self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' %
(self.applicationName))
lumi = ''
message = ""
success = False
###Analyse log file
with open(self.applicationLog) as logfile:
for line in logfile:
if line.count('! Event sample corresponds to luminosity'):
elems = line.split()
lumi = elems[-1]
if line.count("*** Fatal error:"):
status = 1
message = line
break
elif line.count("PYSTOP"):
status = 1
message = line
break
elif line.count("No matrix element available"):
status = 1
message = line
break
elif line.count("Floating point exception"):
status = 1
message = line
break
elif line.count("Event generation finished."):
success = True
else:
status = 0
if success:
status = 0
else:
status = 1
self.log.info(
'The sample generated has an equivalent luminosity of %s' % lumi)
if lumi:
self.workflow_commons['Luminosity'] = float(lumi)
else:
status = 1
##Now care for the cross sections
info = {}
res = self.options.getAsDict()
if os.path.exists("whizard.out") and res['OK']:
full_opts_dict = res['Value']
processes = full_opts_dict['process_input']['process_id'].split()
info = {}
info['xsection'] = {}
processes.append('sum')
with open("whizard.out", "r") as inf:
for line in inf:
line = line.rstrip()
for process in processes:
if not process:
continue
if line.count(" %s " % process):
info['xsection'][process] = {}
line = line.lstrip()
crosssection = line.split()[1]
err_crosssection = line.split()[2]
frac = line.split()[4]
info['xsection'][process]['xsection'] = float(
crosssection)
info['xsection'][process]['err_xsection'] = float(
err_crosssection)
info['xsection'][process]['fraction'] = float(frac)
if info:
if 'Info' not in self.workflow_commons:
self.workflow_commons['Info'] = info
else:
self.workflow_commons['Info'].update(info)
self.log.info("Status after the application execution is %s" %
str(status))
messageout = 'Whizard %s Successful' % (self.applicationVersion)
failed = False
if status != 0:
self.log.error("Whizard execution completed with errors:")
failed = True
else:
self.log.info("Whizard execution completed successfully")
###Deal with output file
if len(self.OutputFile):
if os.path.exists(outputfilename + ".001.stdhep"):
self.log.notice("Looking for output files")
ofnames = glob.glob(outputfilename + '*.stdhep')
if len(ofnames) > 1:
basename = self.OutputFile.split(".stdhep")[0]
i = 0
for of in ofnames:
i += 1
name = basename + "_" + str(i) + ".stdhep"
os.rename(of, name)
else:
os.rename(outputfilename + ".001.stdhep",
self.OutputFile)
else:
self.log.error(
"Whizard execution did not produce a stdhep file")
self.setApplicationStatus(
'Whizard %s Failed to produce STDHEP file' %
(self.applicationVersion))
messageout = 'Whizard Failed to produce STDHEP file'
if not self.ignoreapperrors:
return S_ERROR(messageout)
if failed is True:
self.log.error("==================================\n StdError:\n")
self.log.error(message)
self.setApplicationStatus('%s Exited With Status %s' %
(self.applicationName, status))
self.log.error('Whizard Exited With Status %s' % (status))
messageout = 'Whizard Exited With Status %s' % (status)
if not self.ignoreapperrors:
return S_ERROR(messageout)
else:
self.setApplicationStatus(messageout)
return S_OK({"OutputFile": self.OutputFile})
def TARinstall(app, config, area):
""" For the specified app, install all dependencies
"""
curdir = os.getcwd()
appName = app[0].lower()
appVersion = app[1]
deps = resolveDeps(config, appName, appVersion)
for dep in deps:
depapp = []
depapp.append(dep["app"])
depapp.append(dep["version"])
gLogger.info("Installing dependency %s %s" % (dep["app"], dep["version"]))
res = getTarBallLocation(depapp, config, area)
if not res['OK']:
gLogger.error("Could not install dependency %s %s: %s" % (dep["app"], dep["version"], res['Message']))
return S_ERROR('Failed to install software')
res_from_getTarBall = res['Value']
app_tar = res_from_getTarBall[0]
TarBallURL = res_from_getTarBall[1]
overwrite = res_from_getTarBall[2]
md5sum = res_from_getTarBall[3]
res = install(depapp, app_tar, TarBallURL, overwrite, md5sum, area)
os.chdir(curdir)
if not res['OK']:
gLogger.error("Could not install dependency %s %s: %s" % (dep["app"], dep["version"], res['Message']))
return S_ERROR('Failed to install software')
res_from_install = res['Value']
res = check(depapp, area, res_from_install)
os.chdir(curdir)
if not res['OK']:
gLogger.error("Failed to check dependency %s %s" % (dep["app"], dep["version"]))
return S_ERROR('Failed to check integrity of software')
res_from_check = res['Value']
res = configure(depapp, area, res_from_check)
os.chdir(curdir)
if not res['OK']:
gLogger.error("Failed to configure dependency %s %s" % (dep["app"], dep["version"]))
return S_ERROR('Failed to configure software')
res = clean(area, res_from_install)
if not res['OK']:
gLogger.error("Failed to clean useless tar balls, deal with it: %s %s" % (dep["app"], dep["version"]))
os.chdir(curdir)
gLogger.notice("Successfully installed %s %s in %s" % (dep["app"], dep["version"], area))
res = getTarBallLocation(app, config, area)
if not res['OK']:
gLogger.error("Could not install dependency %s %s: %s" % (appName, appVersion, res['Message']))
return S_ERROR('Failed to install software')
res_from_getTarBall = res['Value']
app_tar = res_from_getTarBall[0]
TarBallURL = res_from_getTarBall[1]
overwrite = res_from_getTarBall[2]
md5sum = res_from_getTarBall[3]
res = install(app, app_tar, TarBallURL, overwrite, md5sum, area)
os.chdir(curdir)
if not res['OK']:
gLogger.error("Could not install software %s %s: %s" % (appName, appVersion, res['Message']))
return S_ERROR('Failed to install software')
res_from_install = res['Value']
res = check(app, area, res_from_install)
os.chdir(curdir)
if not res['OK']:
gLogger.error("Failed to check software %s %s" % (appName, appVersion))
return S_ERROR('Failed to check integrity of software')
res_from_check = res['Value']
res = configure(app, area, res_from_check)
os.chdir(curdir)
if not res['OK']:
gLogger.error("Failed to configure software %s %s" % (appName, appVersion))
return S_ERROR('Failed to configure software')
res = clean(area, res_from_install)
os.chdir(curdir)
if not res['OK']:
gLogger.error("Failed to clean useless tar balls, deal with it")
return S_OK()
gLogger.notice("Successfully installed %s %s in %s" % (appName, appVersion, area))
return res