def test_getEnvironmentScript(self):
with patch('%s.checkCVMFS' % MODULE_NAME,
new=Mock(return_value=S_OK(('/otherfolder/otherfile',
'/cvmfsfolder/myenvscript')))):
result = getEnvironmentScript('a', 'b', 'c', None)
assertDiracSucceedsWith_equals(result, '/cvmfsfolder/myenvscript',
self)
def test_getEnvironmentScript_from_passed_func(self):
def return_my_value(platform, appname, appversion): #pylint: disable=missing-docstring
return (appname, platform, appversion)
with patch('%s.checkCVMFS' % MODULE_NAME,
new=Mock(return_value=S_ERROR('some_cvmfs_error'))):
result = getEnvironmentScript('a', 'b', 'c', return_my_value)
assertEqualsImproved(result, ('b', 'a', 'c'), self)
def test_getEnvironmentScript_cvmfs_empty(self):
def return_my_value(platform, appname, appversion): #pylint: disable=missing-docstring
return (platform, appname, appversion)
with patch('%s.checkCVMFS' % MODULE_NAME,
new=Mock(return_value=S_OK(('entry', '')))):
result = getEnvironmentScript('a', 'b', 'c', return_my_value)
assertEqualsImproved(result, ('a', 'b', 'c'), self)
def runIt(self):
"""
Called by Agent
Execute the following:
- resolve where the soft was installed
- prepare the list of file to feed Marlin with
- create the XML file on which Marlin has to run, done by :any:`prepareXMLFile`
- run Marlin and catch the exit code
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
res = getEnvironmentScript(self.platform, "marlin", self.applicationVersion, self.getEnvScript)
if not res['OK']:
self.log.error("Failed to get the env script")
return res
env_script_path = res["Value"]
res = self.GetInputFiles()
if not res['OK']:
self.log.error("Failed getting input files:", res['Message'])
return res
listofslcio = res['Value']
finalXML = "marlinxml_" + self.STEP_NUMBER + ".xml"
steeringfiledirname = ''
res = getSteeringFileDirName(self.platform, "marlin", self.applicationVersion)
if res['OK']:
steeringfiledirname = res['Value']
else:
self.log.warn('Could not find the steering file directory', res['Message'])
##Handle PandoraSettings.xml
pandorasettings = 'PandoraSettings.xml'
if not os.path.exists(pandorasettings):
if steeringfiledirname and os.path.exists(os.path.join(steeringfiledirname, pandorasettings)):
try:
shutil.copy(os.path.join(steeringfiledirname, pandorasettings),
os.path.join(os.getcwd(), pandorasettings))
except EnvironmentError, x:
self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
def getEnvironmentScript(self):
"""This function gets environment script path from 'dirac.cfg' file
according to the version, application name and platform.
:return: The success or failure of the environment script creation
:rtype: DIRAC.S_OK, DIRAC.S_ERROR
"""
environmentScript = getEnvironmentScript(self.platform, self.applicationName.lower(),
self.applicationVersion, self.generateScriptOnTheFly)
if 'OK' in environmentScript and not environmentScript['OK']:
self.log.error("Environment : 'dirac.cfg' file look up failed")
return False
self.environmentScript = environmentScript["Value"]
return os.path.exists(self.environmentScript)
def getEnvironmentScript(self):
"""This function gets environment script path from 'dirac.cfg' file
according to the version, application name and platform.
:return: The success or failure of the environment script creation
:func:`~DIRAC.Core.Utilities.ReturnValues.S_OK`, :func:`~DIRAC.Core.Utilities.ReturnValues.S_ERROR`
"""
environmentScript = getEnvironmentScript(self.platform, self.applicationName.lower(),
self.applicationVersion, self.generateScriptOnTheFly)
if 'OK' in environmentScript and not environmentScript['OK']:
LOG.error("Environment : 'dirac.cfg' file look up failed")
return False
self.environmentScript = environmentScript["Value"]
return os.path.exists(self.environmentScript)
def runIt(self):
"""
Called by JobAgent
Execute the following:
- get the environment variables that should have been set during installation
- find the detector model xml, using CS query to obtain the path
- prepare the steering file and command line parameters
- run DDSim on this steering file and catch the exit status
:rtype: S_OK, S_ERROR
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:",
self.result['Message'])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'DDSim should not proceed as previous step did not end properly'
)
##TODO: Setup LD_LIBRARY_PATH for extensions
res = getEnvironmentScript(self.platform, self.applicationName,
self.applicationVersion, self.getEnvScript)
if not res['OK']:
self.log.error("Could not obtain the environment script: ",
res["Message"])
return res
envScriptPath = res["Value"]
#get the path to the detector model, either local or from the software
resXML = self._getDetectorXML()
if not resXML['OK']:
self.log.error("Could not obtain the detector XML file: ",
resXML["Message"])
return resXML
compactFile = resXML['Value']
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error("Generator file not found")
return res
self.InputFile = res['Value']
## if steering file is set try to find it
if len(self.SteeringFile) > 0:
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
res = getSteeringFileDirName(self.platform,
self.applicationName,
self.applicationVersion)
if not res['OK']:
self.log.error(
"Could not find where the steering files are")
return res
steeringfiledirname = res['Value']
if os.path.exists(
os.path.join(steeringfiledirname, self.SteeringFile)):
self.SteeringFile = os.path.join(steeringfiledirname,
self.SteeringFile)
if not os.path.exists(self.SteeringFile):
self.log.error("Missing steering file")
return S_ERROR("Could not find steering file")
self.extraCLIarguments += " --steeringFile %s " % self.SteeringFile
if self.startFrom:
self.extraCLIarguments += " --skipNEvents %s " % self.startFrom
if self.debug:
self.extraCLIarguments += " --printLevel DEBUG "
##Same as for mokka: using ParticleGun does not imply InputFile
if self.InputFile:
self.InputFile = [self.InputFile] if isinstance(
self.InputFile, basestring) else self.InputFile
self.extraCLIarguments += " --inputFile %s " % self.InputFile[0]
if self.NumberOfEvents:
self.extraCLIarguments += " --numberOfEvents %s " % self.NumberOfEvents
self.extraCLIarguments += " --random.seed %s " % self.randomSeed
if self.OutputFile:
self.extraCLIarguments += " --outputFile %s " % self.OutputFile
scriptName = 'DDSim_%s_Run_%s.sh' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = []
script.append('#!/bin/bash')
script.append(
'#####################################################################'
)
script.append(
'# Dynamically generated script to run a production or analysis job. #'
)
script.append(
'#####################################################################'
)
script.append('source %s' % envScriptPath)
script.append('echo =========')
## for user provided libraries
if os.path.exists("lib"):
script.append("export LD_LIBRARY_PATH=$PWD/lib:$LD_LIBRARY_PATH")
script.append('env | sort >> localEnv.log')
script.append('echo ddsim:`which ddsim`')
script.append('echo =========')
comm = 'ddsim --compactFile %(compactFile)s %(extraArgs)s' % dict(
compactFile=compactFile, extraArgs=self.extraCLIarguments)
self.log.info("Command:", comm)
script.append(comm)
script.append('declare -x appstatus=$?')
script.append('exit $appstatus')
with open(scriptName, 'w') as scriptFile:
scriptFile.write("\n".join(script))
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'bash "./%s"' % scriptName
self.setApplicationStatus('DDSim %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
comm,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
self.log.error(
"Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed to produce log file' %
(self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log %s' %
(self.applicationName, self.applicationLog))
status = resultTuple[0]
self.log.info("Status after the application execution is %s" % status)
return self.finalStatusReport(status)
def runIt(self):
""" Called from ModuleBase
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('SLIC Pandora should not proceed as previous step did not end properly')
###Get the env script
res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript)
if not res['OK']:
self.log.error("Failed to get the environment script:", res["Message"])
return res
env_script_path = res["Value"]
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error('Could not find input files')
self.setApplicationStatus('SLICPandora: missing slcio file')
return S_ERROR('Missing slcio file!')
runonslcio = res['Value'][0]
if not self.detectorxml.count(".xml") or not os.path.exists(os.path.basename(self.detectorxml)):
detmodel = self.detectorxml.replace("_pandora.xml", "")
if os.path.exists(detmodel + ".zip"):
try:
unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd())
except (RuntimeError, OSError) as err:
self.log.error("Exception when unpacking detectormodel zip file:", str(err))
os.unlink(detmodel + ".zip")
if not os.path.exists(detmodel + ".zip"):
#retrieve detector model from web
detector_urls = self.ops.getValue('/SLICweb/SLICDetectorModels', [''])
if len(detector_urls[0]) < 1:
self.log.error('Could not find in CS the URL for detector model')
return S_ERROR('Could not find in CS the URL for detector model')
for detector_url in detector_urls:
try:
urllib.urlretrieve("%s%s"%(detector_url, detmodel + ".zip"), detmodel + ".zip")
except IOError:
self.log.error("Download of detector model failed")
continue
try:
unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd())
break
except (RuntimeError, OSError) as err:
self.log.error("Exception for zip file obtained from ", detector_url)
self.log.error("Exception:", str(err))
os.unlink(detmodel + ".zip")
continue
#if os.path.exists(detmodel): #and os.path.isdir(detmodel):
self.detectorxml = os.path.join(os.getcwd(), self.detectorxml)
self.detectorxml = self.detectorxml + "_pandora.xml"
if not os.path.exists(self.detectorxml):
self.log.error('Detector model xml %s was not found, exiting' % self.detectorxml)
return S_ERROR('Detector model xml was not found, exiting')
oldversion = False
if self.applicationVersion in ['CLIC_CDR', 'CDR1', 'CDR2', 'CDR0', 'V2', 'V3', 'V4']:
oldversion = True
scriptName = 'SLICPandora_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/bash \n')
script.write('#####################################################################\n')
script.write('# Dynamically generated script to run a production or analysis job. #\n')
script.write('#####################################################################\n')
script.write("source %s\n" % env_script_path)
script.write("declare -x file=./Settings/%s\n" % self.pandorasettings)
script.write("""
if [ -e "${file}" ]
then
declare -x PANDORASETTINGS=$file
else
if [ -d "${PANDORASETTINGSDIR}" ]
then
cp $PANDORASETTINGSDIR/*.xml .
declare -x PANDORASETTINGS=%s
fi
fi
if [ ! -e "${PANDORASETTINGS}" ]
then
echo "Missing PandoraSettings file"
exit 1
fi
""" % self.pandorasettings )
script.write('echo =============================\n')
script.write('echo PATH is \n')
script.write('echo $PATH | tr ":" "\n" \n')
script.write('echo ==============\n')
if os.path.exists("./lib"):
script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n')
script.write('echo =============================\n')
script.write('echo LD_LIBRARY_PATH is \n')
script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n')
script.write('echo ============================= \n')
script.write('env | sort >> localEnv.log\n')
#Now run it
if oldversion:
comm = 'PandoraFrontend %s $PANDORASETTINGS %s %s %s' % (self.detectorxml, runonslcio,
self.OutputFile, str(self.NumberOfEvents))
else:
comm = 'PandoraFrontend -g %s -c $PANDORASETTINGS -i %s -o %s -r %s' % (self.detectorxml,
runonslcio, self.OutputFile,
str(self.NumberOfEvents))
comm = "%s %s\n" % (comm, self.extraCLIarguments)
self.log.info("Will run %s" % comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
#script.write('where\n')
#script.write('quit\n')
#script.write('EOF\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('SLICPandora %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput,
bufferLimit = 20971520)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
if resultTuple[0]:
self.log.error("There was an error during the execution")
if not os.path.exists(self.applicationLog):
self.log.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' % (self.applicationName))
logf = open(self.applicationLog,'r')
if "Missing PandoraSettings file" in logf.readlines()[-1]:
self.log.error("Issue with the Pandora Settings file.")
logf.close()
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
self.log.info( "Status after the application execution is %s" % str( status ) )
return self.finalStatusReport(status)
def runIt(self):
"""
Called by Agent
Execute the following:
- define the platform
- check for presence of ROOTSYS variable
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
LOG.error("Failed to resolve input parameters:", self.result['Message'])
return self.result
res = getEnvironmentScript(self.platform, "root", self.applicationVersion, self.getRootEnvScript)
LOG.notice("Got the environment script: %s" % res)
if not res['OK']:
LOG.error("Error getting the env script: ", res['Message'])
return res
envScriptPath = res['Value']
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('ROOT should not proceed as previous step did not end properly')
if len(self.script) < 1:
LOG.error('Macro file not defined, should not happen!')
return S_ERROR("Macro file not defined")
self.script = os.path.basename(self.script)
scriptName = 'Root_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName,'w')
script.write('#!/bin/sh \n')
script.write('#####################################################################\n')
script.write('# Dynamically generated script to run a production or analysis job. #\n')
script.write('#####################################################################\n')
script.write('source %s\n' % envScriptPath )
if os.path.exists("./lib"):
script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n')
script.write('echo =============================\n')
script.write('echo LD_LIBRARY_PATH is\n')
script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n')
script.write('echo =============================\n')
script.write('echo PATH is\n')
script.write('echo $PATH | tr ":" "\n"\n')
script.write('echo =============================\n')
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
comm = "root -b -q %s" % self.script
if self.arguments:
## need rawstring for arguments so we don't lose escaped quotation marks for string arguments
comm = comm + r'\(%s\)' % self.arguments
comm = comm + "\n"
LOG.info("Will run %s" % (comm))
script.write(comm)
script.write('declare -x appstatus=$?\n')
#script.write('where\n')
#script.write('quit\n')
#script.write('EOF\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0o755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('ROOT %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
LOG.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('root failed terribly, you are doomed!')
return S_ERROR('root did not produce the expected log' )
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
LOG.info("Status after the application execution is %s" % str(status))
return self.finalStatusReport(status)
def runIt(self):
""" Called by ModuleBase
Executes the following:
- read the application parameters that where defined in ILCJob, and stored in the job definition
- setup the SQL server and run it in the background
- prepare the steering file using :any:`prepareSteeringFile`
- run Mokka and catch its return status
:return: S_OK(), S_ERROR()
"""
#if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
# LOG.info('Skip this module, failure detected in a previous step :')
# LOG.info('Workflow status : %s' %(self.workflowStatus))
# LOG.info('Step Status %s' %(self.stepStatus))
# return S_OK()
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
LOG.error("Failed to resolve the input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('Mokka should not proceed as previous step did not end properly')
LOG.info("Executing Mokka %s" % (self.applicationVersion))
if self.dbSlice:
if self.dbSlice.lower().startswith("lfn"):
self.dbSlice = os.path.basename(self.dbSlice)
self.db_dump_name = self.dbSlice
else:
self.db_dump_name = "CLICMokkaDB.sql"
res = getEnvironmentScript(self.platform, "mokka", self.applicationVersion, self.getEnvScript)
LOG.notice("Got the environment script: %s" % res)
if not res['OK']:
LOG.error("Error getting the env script: ", res['Message'])
return res
env_script_path = res['Value']
####Setup MySQL instance
mysqlBasePath = '%s/mysqltmp/MokkaDBRoot-%s' %(os.getcwd(), generateRandomString(8))
LOG.notice("Placing mysql files in %s" % mysqlBasePath)
###steering file that will be used to run
mokkasteer = "mokka.steer"
if os.path.exists("mokka.steer"):
try:
os.rename("mokka.steer", "mymokka.steer")
except EnvironmentError as err:
LOG.error("Failed renaming the steering file: ", str(err))
self.SteeringFile = "mymokka.steer"
###prepare steering file
#first, I need to take the stdhep file, find its path (possible LFN)
if len(self.InputFile) > 0:
#self.InputFile = os.path.basename(self.InputFile)
res = resolveIFpaths(self.InputFile)
if not res['OK']:
LOG.error("Generator file not found")
return res
self.InputFile = res['Value']
if len(self.macFile) > 0:
self.macFile = os.path.basename(self.macFile)
##idem for steering file
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
LOG.verbose("Steering file %s not found locally" % self.SteeringFile)
res = getSteeringFileDirName(self.platform, "mokka", self.applicationVersion)
if not res['OK']:
LOG.error("Missing Steering file directory:", res['Message'])
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
try:
shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile )
except EnvironmentError as err:
LOG.error("Failed copying file", self.SteeringFile)
return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(err)))
#self.steeringFile = os.path.join(mySoftwareRoot,"steeringfiles",self.steeringFile)
if not os.path.exists(self.SteeringFile):
LOG.error("Missing steering file, should not happen!")
return S_ERROR("Could not find steering file")
else:
LOG.verbose("Found local copy of %s" % self.SteeringFile)
### The following is because if someone uses particle gun, there is no InputFile
if not len(self.InputFile):
self.InputFile = ['']
steerok = prepareSteeringFile(self.SteeringFile, mokkasteer, self.detectorModel, self.InputFile[0],
self.macFile, self.NumberOfEvents, self.startFrom, self.RandomSeed,
self.mcRunNumber,
self.ProcessID,
self.debug,
self.OutputFile,
self.inputdataMeta)
if not steerok['OK']:
LOG.error('Failed to create MOKKA steering file')
return S_ERROR('Failed to create MOKKA steering file')
###Extra option depending on mokka version
mokkaextraoption = ""
if self.applicationVersion not in ["v07-02", "v07-02fw", "v07-02fwhp", "MokkaRevision42", "MokkaRevision43",
"MokkaRevision44", "Revision45"]:
mokkaextraoption = "-U"
scriptName = 'Mokka_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/bash \n')
script.write('#####################################################################\n')
script.write('# Dynamically generated script to run a production or analysis job. #\n')
script.write('#####################################################################\n')
script.write("source %s\n" % env_script_path)
if os.path.exists("./lib"):
script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n' )
script.write('echo =============================\n')
script.write('echo Content of mokka.steer:\n')
script.write('cat mokka.steer\n')
if self.macFile:
script.write('echo =============================\n')
script.write('echo Content of %s\n' % os.path.basename(self.macFile))
script.write("cat %s\n" % os.path.basename(self.macFile))
elif os.path.exists("./mokkamac.mac"):
script.write("""
echo =============================
echo Content of mokkamac.mac:
cat mokkamac.mac
""")
script.write('echo =============================\n')
script.write('echo LD_LIBRARY_PATH is\n')
script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n')
script.write('echo =============================\n')
script.write('echo PATH is\n')
script.write('echo $PATH | tr ":" "\n"\n')
if self.debug:
script.write('echo ldd of Mokka is\n')
script.write('ldd `which Mokka` \n' )
script.write('echo =============================\n')
script.write('echo ldd of mysql is\n')
script.write('ldd `which mysql` \n' )
script.write('echo =============================\n')
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write("""if [ ! -e ./%(DBDUMP)s ]; then
echo "Missing DB dump!" >&2
exit 1
fi
""" % {'DBDUMP': self.db_dump_name} )
##Now start the MySQL server and configure.
script.write("declare -x MOKKADBROOT=%s\n" % mysqlBasePath)
script.write("declare -x WORKDIR=%s\n" % os.getcwd())
script.write("declare -x LOGDIR=$WORKDIR/mysqllogs/\n")
script.write("mkdir -p $LOGDIR\n")
#mysql socket must not be longer than 107 characters!
script.write("declare -x SOCKETBASE=/tmp/mokka-%s\n" % generateRandomString(8) )
script.write("mkdir -p $SOCKETBASE\n")
script.write("declare -x SOCKETPATH=$SOCKETBASE/mysql.sock\n" )
script.write("mkdir -p $MOKKADBROOT\n")
script.write("declare -x MYSQLDATA=$MOKKADBROOT/data\n")
script.write("rm -rf $MYSQLDATA\n")
script.write("mkdir -p $MYSQLDATA\n")
script.write("date\n")
script.write("""echo "*** Installing MySQL"
echo "mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log"
mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log
install_st=$?
if [ $install_st -ne 0 ]
then
exit $install_st
fi
date
echo "*** Running mysqld-safe"
cd $MYSQL
bin/mysqld_safe --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log &
COUNT=0
while [ -z "$socket_grep" ] ; do
socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH")
echo -n .
sleep 1
if [ $COUNT -eq 100 ]; then
echo "Failed to find socket"
break
fi
COUNT=$(( $COUNT + 1 ))
done
cd -
echo "*** Configuring MySQL"
mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot password 'rootpass'
declare -x admin_st=$?
if [ $admin_st -ne 0 ]
then
echo "*** mysqladmin failed, cannot proceed" >&2
mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown
exit $admin_st
fi
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO root;'
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO consult IDENTIFIED BY \"consult\";'
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'DELETE FROM mysql.user WHERE User = \"\"; FLUSH PRIVILEGES;'
echo "*** Installing Mokka DB"
date
echo "mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s"
time mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s
sleep 5
date\n""" % {'DBDUMP': self.db_dump_name})
#Now take care of the particle tables.
script.write("""
if [ -e "./particle.tbl" ]
then
declare -x PARTICLETBL=./particle.tbl
fi
if [ -d "${MOKKA}" ]
#As the local env does not set MOKKA, it must come from CVMFS
then
declare -x PARTICLETBL=$MOKKA/particle.tbl
fi
if [ -d "${MOKKATARBALL}" ]
#As the local env does set MOKKATARBALL, it must be there
then
declare -x PARTICLETBL=$MOKKATARBALL/ConfigFiles/particle.tbl
fi
echo "/Mokka/init/PDGFile $PARTICLETBL" >> %s
\n"""% mokkasteer)
##Now run Mokka
comm = 'Mokka %s -h localhost:$SOCKETPATH %s %s\n' % (mokkaextraoption, mokkasteer, self.extraCLIarguments)
LOG.info("Command : %s" % (comm))
script.write(comm)
script.write('declare -x appstatus=$?\n')
#Now shutdown the MySQL server
script.write("mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown\n")
script.write("""
while [ -n "$socket_grep" ] ; do
socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH")
echo -n .
sleep 1
done
""")
script.write("rm -f %s\n" % self.db_dump_name)#remove db file
script.write('rm -rf $MYSQLDATA\n')#cleanup
script.write('rm -rf $MOKKADBROOT\n')#cleanup
#script.write('quit\n')
#script.write('EOF\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0o755)
comm = 'sh -c "./%s"' % scriptName
self.setApplicationStatus('Mokka %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
LOG.info("Status after Mokka execution is %s" % str(status))
if not os.path.exists(self.applicationLog):
LOG.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
if not self.ignoreapperrors:
LOG.error("Missing log file")
return S_ERROR('%s did not produce the expected log' % (self.applicationName))
###Now change the name of Mokka output to the specified filename
if os.path.exists("out.slcio"):
if len(self.OutputFile) > 0:
os.rename("out.slcio", self.OutputFile)
failed = False
if status not in [0, 106, 9]:
LOG.error("Mokka execution completed with errors:")
failed = True
elif status in [106, 9]:
LOG.info("Mokka execution reached end of input generator file")
else:
LOG.info("Mokka execution finished successfully")
message = 'Mokka %s Successful' % (self.applicationVersion)
if failed is True:
LOG.error("==================================\n StdError:\n")
LOG.error(self.stdError)
#self.setApplicationStatus('%s Exited With Status %s' %(self.applicationName,status))
LOG.error('Mokka Exited With Status %s' % (status))
message = 'Mokka Exited With Status %s' % (status)
self.setApplicationStatus(message)
if not self.ignoreapperrors:
LOG.error("Application exists with error:", message)
return S_ERROR(message)
else:
if status in [106, 9]:
message = 'Mokka %s reached end of input generator file' % (self.applicationVersion)
self.setApplicationStatus(message)
return S_OK( { 'OutputFile' : self.OutputFile } )
def test_getEnvironmentScript_from_passed_func( self ):
def return_my_value( platform, appname, appversion ): #pylint: disable=C0111
return ( appname, platform, appversion )
with patch('%s.checkCVMFS' % MODULE_NAME, new=Mock(return_value=S_ERROR('some_cvmfs_error'))):
result = getEnvironmentScript( 'a', 'b', 'c', return_my_value )
assertEqualsImproved( result, ('b', 'a', 'c'), self )
def test_getEnvironmentScript_cvmfs_empty( self ):
def return_my_value( platform, appname, appversion ): #pylint: disable=C0111
return ( platform, appname, appversion )
with patch('%s.checkCVMFS' % MODULE_NAME, new=Mock(return_value=S_OK(('entry', '')))):
result = getEnvironmentScript( 'a', 'b', 'c', return_my_value )
assertEqualsImproved( result, ('a', 'b', 'c'), self )
def test_getEnvironmentScript( self ):
with patch('%s.checkCVMFS' % MODULE_NAME, new=Mock(return_value=S_OK(('/otherfolder/otherfile', '/cvmfsfolder/myenvscript')))):
result = getEnvironmentScript( 'a', 'b', 'c', None )
assertDiracSucceedsWith_equals( result, '/cvmfsfolder/myenvscript', self )
def runIt(self):
""" Run the module
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error('Failed to resolve input parameters:',
self.result['Message'])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'%s should not proceed as previous step did not end properly' %
self.applicationName)
res = getEnvironmentScript(self.platform, "tomato",
self.applicationVersion, self.getEnvScript)
if not res["OK"]:
self.log.error("Failed to get the env for Tomato:", res["Message"])
return res
env_script_path = res["Value"]
res = self.prepareMARLIN_DLL(env_script_path)
if not res['OK']:
self.log.error('Failed building MARLIN_DLL: %s' % res['Message'])
self.setApplicationStatus('Failed to setup MARLIN_DLL')
return S_ERROR('Something wrong with software installation')
marlin_dll = res['Value']
res = self._getInputFiles()
if not res['OK']:
self.log.error(res['Message'])
return res
listofslcio = res['Value']
finalXML = 'tomato.xml'
res = prepareTomatoSalad(None, finalXML, self.OutputFile, listofslcio,
self.collection)
if not res['OK']:
self.log.error('Could not prepare the Tomato XML: %s' %
res['Message'])
self.setApplicationStatus('Failed to setup Tomato')
return S_ERROR('Failed to setup Tomato')
self.result = self.runMarlin(finalXML, env_script_path, marlin_dll)
if not self.result['OK']:
self.log.error('Something wrong during running: %s' %
self.result['Message'])
self.setApplicationStatus('Error during running %s' %
self.applicationName)
return S_ERROR('Failed to run %s' % self.applicationName)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
self.log.error(
"Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' %
(self.applicationName))
if not self.ignoreapperrors:
self.log.error('Missing log file')
return S_ERROR('%s did not produce the expected log' %
(self.applicationName))
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
self.log.info("Status after the application execution is %s" %
str(status))
return self.finalStatusReport(status)
def runIt(self):
""" Called by ModuleBase
Executes the following:
- read the application parameters that where defined in ILCJob, and stored in the job definition
- setup the SQL server and run it in the background
- prepare the steering file using :any:`prepareSteeringFile`
- run Mokka and catch its return status
:return: S_OK(), S_ERROR()
"""
#if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
# self.log.info('Skip this module, failure detected in a previous step :')
# self.log.info('Workflow status : %s' %(self.workflowStatus))
# self.log.info('Step Status %s' %(self.stepStatus))
# return S_OK()
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error("Failed to resolve the input parameters:",
self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'Mokka should not proceed as previous step did not end properly'
)
self.log.info("Executing Mokka %s" % (self.applicationVersion))
if self.dbSlice:
if self.dbSlice.lower().startswith("lfn"):
self.dbSlice = os.path.basename(self.dbSlice)
self.db_dump_name = self.dbSlice
else:
self.db_dump_name = "CLICMokkaDB.sql"
res = getEnvironmentScript(self.platform, "mokka",
self.applicationVersion, self.getEnvScript)
self.log.notice("Got the environment script: %s" % res)
if not res['OK']:
self.log.error("Error getting the env script: ", res['Message'])
return res
env_script_path = res['Value']
####Setup MySQL instance
mysqlBasePath = '%s/mysqltmp/MokkaDBRoot-%s' % (
os.getcwd(), generateRandomString(8))
self.log.notice("Placing mysql files in %s" % mysqlBasePath)
###steering file that will be used to run
mokkasteer = "mokka.steer"
if os.path.exists("mokka.steer"):
try:
os.rename("mokka.steer", "mymokka.steer")
except EnvironmentError as err:
self.log.error("Failed renaming the steering file: ", str(err))
self.SteeringFile = "mymokka.steer"
###prepare steering file
#first, I need to take the stdhep file, find its path (possible LFN)
if len(self.InputFile) > 0:
#self.InputFile = os.path.basename(self.InputFile)
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error("Generator file not found")
return res
self.InputFile = res['Value']
if len(self.macFile) > 0:
self.macFile = os.path.basename(self.macFile)
##idem for steering file
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
self.log.verbose("Steering file %s not found locally" %
self.SteeringFile)
res = getSteeringFileDirName(self.platform, "mokka",
self.applicationVersion)
if not res['OK']:
self.log.error("Missing Steering file directory:",
res['Message'])
return res
steeringfiledirname = res['Value']
if os.path.exists(
os.path.join(steeringfiledirname, self.SteeringFile)):
try:
shutil.copy(
os.path.join(steeringfiledirname, self.SteeringFile),
"./" + self.SteeringFile)
except EnvironmentError as err:
self.log.error("Failed copying file", self.SteeringFile)
return S_ERROR('Failed to access file %s: %s' %
(self.SteeringFile, str(err)))
#self.steeringFile = os.path.join(mySoftwareRoot,"steeringfiles",self.steeringFile)
if not os.path.exists(self.SteeringFile):
self.log.error("Missing steering file, should not happen!")
return S_ERROR("Could not find steering file")
else:
self.log.verbose("Found local copy of %s" % self.SteeringFile)
### The following is because if someone uses particle gun, there is no InputFile
if not len(self.InputFile):
self.InputFile = ['']
steerok = prepareSteeringFile(
self.SteeringFile, mokkasteer, self.detectorModel,
self.InputFile[0], self.macFile, self.NumberOfEvents,
self.startFrom, self.RandomSeed, self.mcRunNumber, self.ProcessID,
self.debug, self.OutputFile, self.inputdataMeta)
if not steerok['OK']:
self.log.error('Failed to create MOKKA steering file')
return S_ERROR('Failed to create MOKKA steering file')
###Extra option depending on mokka version
mokkaextraoption = ""
if self.applicationVersion not in [
"v07-02", "v07-02fw", "v07-02fwhp", "MokkaRevision42",
"MokkaRevision43", "MokkaRevision44", "Revision45"
]:
mokkaextraoption = "-U"
scriptName = 'Mokka_%s_Run_%s.sh' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/bash \n')
script.write(
'#####################################################################\n'
)
script.write(
'# Dynamically generated script to run a production or analysis job. #\n'
)
script.write(
'#####################################################################\n'
)
script.write("source %s\n" % env_script_path)
if os.path.exists("./lib"):
script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n')
script.write('echo =============================\n')
script.write('echo Content of mokka.steer:\n')
script.write('cat mokka.steer\n')
if self.macFile:
script.write('echo =============================\n')
script.write('echo Content of %s\n' %
os.path.basename(self.macFile))
script.write("cat %s\n" % os.path.basename(self.macFile))
elif os.path.exists("./mokkamac.mac"):
script.write("""
echo =============================
echo Content of mokkamac.mac:
cat mokkamac.mac
""")
script.write('echo =============================\n')
script.write('echo LD_LIBRARY_PATH is\n')
script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n')
script.write('echo =============================\n')
script.write('echo PATH is\n')
script.write('echo $PATH | tr ":" "\n"\n')
if self.debug:
script.write('echo ldd of Mokka is\n')
script.write('ldd `which Mokka` \n')
script.write('echo =============================\n')
script.write('echo ldd of mysql is\n')
script.write('ldd `which mysql` \n')
script.write('echo =============================\n')
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write("""if [ ! -e ./%(DBDUMP)s ]; then
echo "Missing DB dump!" >&2
exit 1
fi
""" % {'DBDUMP': self.db_dump_name})
##Now start the MySQL server and configure.
script.write("declare -x MOKKADBROOT=%s\n" % mysqlBasePath)
script.write("declare -x WORKDIR=%s\n" % os.getcwd())
script.write("declare -x LOGDIR=$WORKDIR/mysqllogs/\n")
script.write("mkdir -p $LOGDIR\n")
#mysql socket must not be longer than 107 characters!
script.write("declare -x SOCKETBASE=/tmp/mokka-%s\n" %
generateRandomString(8))
script.write("mkdir -p $SOCKETBASE\n")
script.write("declare -x SOCKETPATH=$SOCKETBASE/mysql.sock\n")
script.write("mkdir -p $MOKKADBROOT\n")
script.write("declare -x MYSQLDATA=$MOKKADBROOT/data\n")
script.write("rm -rf $MYSQLDATA\n")
script.write("mkdir -p $MYSQLDATA\n")
script.write("date\n")
script.write("""echo "*** Installing MySQL"
echo "mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log"
mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log
install_st=$?
if [ $install_st -ne 0 ]
then
exit $install_st
fi
date
echo "*** Running mysqld-safe"
cd $MYSQL
bin/mysqld_safe --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log &
COUNT=0
while [ -z "$socket_grep" ] ; do
socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH")
echo -n .
sleep 1
if [ $COUNT -eq 100 ]; then
echo "Failed to find socket"
break
fi
COUNT=$(( $COUNT + 1 ))
done
cd -
echo "*** Configuring MySQL"
mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot password 'rootpass'
declare -x admin_st=$?
if [ $admin_st -ne 0 ]
then
echo "*** mysqladmin failed, cannot proceed" >&2
mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown
exit $admin_st
fi
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO root;'
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO consult IDENTIFIED BY \"consult\";'
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'DELETE FROM mysql.user WHERE User = \"\"; FLUSH PRIVILEGES;'
echo "*** Installing Mokka DB"
date
echo "mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s"
time mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s
sleep 5
date\n""" % {'DBDUMP': self.db_dump_name})
#Now take care of the particle tables.
script.write("""
if [ -e "./particle.tbl" ]
then
declare -x PARTICLETBL=./particle.tbl
fi
if [ -d "${MOKKA}" ]
#As the local env does not set MOKKA, it must come from CVMFS
then
declare -x PARTICLETBL=$MOKKA/particle.tbl
fi
if [ -d "${MOKKATARBALL}" ]
#As the local env does set MOKKATARBALL, it must be there
then
declare -x PARTICLETBL=$MOKKATARBALL/ConfigFiles/particle.tbl
fi
echo "/Mokka/init/PDGFile $PARTICLETBL" >> %s
\n""" % mokkasteer)
##Now run Mokka
comm = 'Mokka %s -h localhost:$SOCKETPATH %s %s\n' % (
mokkaextraoption, mokkasteer, self.extraCLIarguments)
self.log.info("Command : %s" % (comm))
script.write(comm)
script.write('declare -x appstatus=$?\n')
#Now shutdown the MySQL server
script.write(
"mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown\n"
)
script.write("""
while [ -n "$socket_grep" ] ; do
socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH")
echo -n .
sleep 1
done
""")
script.write("rm -f %s\n" % self.db_dump_name) #remove db file
script.write('rm -rf $MYSQLDATA\n') #cleanup
script.write('rm -rf $MOKKADBROOT\n') #cleanup
#script.write('quit\n')
#script.write('EOF\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'sh -c "./%s"' % scriptName
self.setApplicationStatus('Mokka %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
comm,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
self.log.info("Status after Mokka execution is %s" % str(status))
if not os.path.exists(self.applicationLog):
self.log.error(
"Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' %
(self.applicationName))
if not self.ignoreapperrors:
self.log.error("Missing log file")
return S_ERROR('%s did not produce the expected log' %
(self.applicationName))
###Now change the name of Mokka output to the specified filename
if os.path.exists("out.slcio"):
if len(self.OutputFile) > 0:
os.rename("out.slcio", self.OutputFile)
failed = False
if status not in [0, 106, 9]:
self.log.error("Mokka execution completed with errors:")
failed = True
elif status in [106, 9]:
self.log.info(
"Mokka execution reached end of input generator file")
else:
self.log.info("Mokka execution finished successfully")
message = 'Mokka %s Successful' % (self.applicationVersion)
if failed is True:
self.log.error("==================================\n StdError:\n")
self.log.error(self.stdError)
#self.setApplicationStatus('%s Exited With Status %s' %(self.applicationName,status))
self.log.error('Mokka Exited With Status %s' % (status))
message = 'Mokka Exited With Status %s' % (status)
self.setApplicationStatus(message)
if not self.ignoreapperrors:
self.log.error("Application exists with error:", message)
return S_ERROR(message)
else:
if status in [106, 9]:
message = 'Mokka %s reached end of input generator file' % (
self.applicationVersion)
self.setApplicationStatus(message)
return S_OK({'OutputFile': self.OutputFile})
def runIt(self):
""" Run the module
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
LOG.error('Failed to resolve input parameters:', self.result['Message'])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
res = getEnvironmentScript(self.platform, "tomato", self.applicationVersion, self.getEnvScript)
if not res["OK"]:
LOG.error("Failed to get the env for Tomato:", res["Message"])
return res
env_script_path = res["Value"]
res = self.prepareMARLIN_DLL(env_script_path)
if not res['OK']:
LOG.error('Failed building MARLIN_DLL: %s' % res['Message'])
self.setApplicationStatus('Failed to setup MARLIN_DLL')
return S_ERROR('Something wrong with software installation')
marlin_dll = res['Value']
res = self._getInputFiles()
if not res['OK']:
LOG.error(res['Message'])
return res
listofslcio = res['Value']
finalXML = 'tomato.xml'
res = prepareTomatoSalad(None, finalXML, self.OutputFile, listofslcio, self.collection)
if not res['OK']:
LOG.error('Could not prepare the Tomato XML: %s' % res['Message'])
self.setApplicationStatus('Failed to setup Tomato')
return S_ERROR('Failed to setup Tomato')
self.result = self.runMarlin(finalXML, env_script_path, marlin_dll)
if not self.result['OK']:
LOG.error('Something wrong during running: %s' % self.result['Message'])
self.setApplicationStatus('Error during running %s' % self.applicationName)
return S_ERROR('Failed to run %s' % self.applicationName)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
LOG.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
if not self.ignoreapperrors:
LOG.error('Missing log file')
return S_ERROR('%s did not produce the expected log' % (self.applicationName))
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
LOG.info("Status after the application execution is %s" % str(status))
return self.finalStatusReport(status)
def runIt(self):
"""
Called by JobAgent
Execute the following:
- get the environment variables that should have been set during installation
- download the detector model, using CS query to fetch the address
- prepare the mac file using :any:`prepareMacFile`
- run SLIC on this mac File and catch the exit status
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:",
self.result['Message'])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'SLIC should not proceed as previous step did not end properly'
)
res = getEnvironmentScript(self.platform, self.applicationName,
self.applicationVersion, self.getEnvScript)
if not res['OK']:
self.log.error("Could not obtain the environment script: ",
res["Message"])
return res
env_script_path = res["Value"]
retMod = self.getDetectorModel()
if not retMod:
return retMod
slicmac = 'slicmac.mac'
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error("Generator file not found")
return res
self.InputFile = res['Value']
if len(self.SteeringFile) > 0:
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
res = getSteeringFileDirName(self.platform,
self.applicationName,
self.applicationVersion)
if not res['OK']:
self.log.error(
"Could not find where the steering files are")
return res
steeringfiledirname = res['Value']
if os.path.exists(
os.path.join(steeringfiledirname, self.SteeringFile)):
self.SteeringFile = os.path.join(steeringfiledirname,
self.SteeringFile)
if not os.path.exists(self.SteeringFile):
self.log.error("Missing steering file")
return S_ERROR("Could not find mac file")
##Same as for mokka: using ParticleGun does not imply InputFile
if not len(self.InputFile):
self.InputFile = ['']
macok = prepareMacFile(self.SteeringFile, slicmac, self.InputFile[0],
self.NumberOfEvents, self.startFrom,
self.detectorModel, self.RandomSeed,
self.OutputFile, self.debug)
if not macok['OK']:
self.log.error('Failed to create SLIC mac file')
return S_ERROR('Error when creating SLIC mac file')
scriptName = 'SLIC_%s_Run_%s.sh' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/bash \n')
script.write(
'#####################################################################\n'
)
script.write(
'# Dynamically generated script to run a production or analysis job. #\n'
)
script.write(
'#####################################################################\n'
)
script.write("source %s\n" % (env_script_path))
preloadDir = "%s/preload" % os.getcwd()
if os.path.exists(preloadDir):
preadloadLibs = set(os.listdir(preloadDir))
preloadString = ":".join(lib for lib in preadloadLibs
if ".so" in lib)
script.write('declare -x LD_LIBRARY_PATH=%s:${LD_LIBRARY_PATH}\n' %
preloadDir)
script.write('declare -x LD_PRELOAD=%s\n' % preloadString)
script.write('echo =========\n')
script.write('env | sort >> localEnv.log\n')
script.write('echo SLIC:\n')
script.write("which slic\n")
script.write('echo =========\n')
comm = 'slic -P $PARTICLE_TBL -m %s %s\n' % (slicmac,
self.extraCLIarguments)
self.log.info("Command:", comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'sh -c "./%s"' % scriptName
self.setApplicationStatus('SLIC %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
comm,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
self.log.error(
"Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' %
(self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' %
(self.applicationName))
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
self.log.info("Status after the application execution is %s" %
str(status))
return self.finalStatusReport(status)
def runIt(self):
"""
Called by Agent
Execute the following:
- define the platform
- check for presence of ROOTSYS variable
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:",
self.result['Message'])
return self.result
res = getEnvironmentScript(self.platform, "root",
self.applicationVersion,
self.getRootEnvScript)
self.log.notice("Got the environment script: %s" % res)
if not res['OK']:
self.log.error("Error getting the env script: ", res['Message'])
return res
envScriptPath = res['Value']
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'ROOT should not proceed as previous step did not end properly'
)
if len(self.script) < 1:
self.log.error('Macro file not defined, should not happen!')
return S_ERROR("Macro file not defined")
self.script = os.path.basename(self.script)
scriptName = 'Root_%s_Run_%s.sh' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/sh \n')
script.write(
'#####################################################################\n'
)
script.write(
'# Dynamically generated script to run a production or analysis job. #\n'
)
script.write(
'#####################################################################\n'
)
script.write('source %s\n' % envScriptPath)
if os.path.exists("./lib"):
script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n')
script.write('echo =============================\n')
script.write('echo LD_LIBRARY_PATH is\n')
script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n')
script.write('echo =============================\n')
script.write('echo PATH is\n')
script.write('echo $PATH | tr ":" "\n"\n')
script.write('echo =============================\n')
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
comm = "root -b -q %s" % self.script
if self.arguments:
## need rawstring for arguments so we don't lose escaped quotation marks for string arguments
comm = comm + r'\(%s\)' % self.arguments
comm = comm + "\n"
self.log.info("Will run %s" % (comm))
script.write(comm)
script.write('declare -x appstatus=$?\n')
#script.write('where\n')
#script.write('quit\n')
#script.write('EOF\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('ROOT %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
comm,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
self.log.error(
"Something went terribly wrong, the log file is not present")
self.setApplicationStatus('root failed terribly, you are doomed!')
return S_ERROR('root did not produce the expected log')
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
self.log.info("Status after the application execution is %s" %
str(status))
return self.finalStatusReport(status)
def runIt(self):
""" Called from ModuleBase
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:",
self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'SLIC Pandora should not proceed as previous step did not end properly'
)
###Get the env script
res = getEnvironmentScript(self.platform, self.applicationName,
self.applicationVersion, self.getEnvScript)
if not res['OK']:
self.log.error("Failed to get the environment script:",
res["Message"])
return res
env_script_path = res["Value"]
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error('Could not find input files')
self.setApplicationStatus('SLICPandora: missing slcio file')
return S_ERROR('Missing slcio file!')
runonslcio = res['Value'][0]
if not self.detectorxml.count(".xml") or not os.path.exists(
os.path.basename(self.detectorxml)):
detmodel = self.detectorxml.replace("_pandora.xml", "")
if os.path.exists(detmodel + ".zip"):
try:
unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd())
except (RuntimeError, OSError) as err:
self.log.error(
"Exception when unpacking detectormodel zip file:",
str(err))
os.unlink(detmodel + ".zip")
if not os.path.exists(detmodel + ".zip"):
#retrieve detector model from web
detector_urls = self.ops.getValue(
'/SLICweb/SLICDetectorModels', [''])
if len(detector_urls[0]) < 1:
self.log.error(
'Could not find in CS the URL for detector model')
return S_ERROR(
'Could not find in CS the URL for detector model')
for detector_url in detector_urls:
try:
urllib.urlretrieve(
"%s%s" % (detector_url, detmodel + ".zip"),
detmodel + ".zip")
except IOError:
self.log.error("Download of detector model failed")
continue
try:
unzip_file_into_dir(open(detmodel + ".zip"),
os.getcwd())
break
except (RuntimeError, OSError) as err:
self.log.error("Exception for zip file obtained from ",
detector_url)
self.log.error("Exception:", str(err))
os.unlink(detmodel + ".zip")
continue
#if os.path.exists(detmodel): #and os.path.isdir(detmodel):
self.detectorxml = os.path.join(os.getcwd(), self.detectorxml)
if not self.detectorxml.endswith('xml'):
self.detectorxml = self.detectorxml + "_pandora.xml"
if not os.path.exists(self.detectorxml):
self.log.error('Detector model xml %s was not found, exiting' %
self.detectorxml)
return S_ERROR('Detector model xml was not found, exiting')
oldversion = False
if self.applicationVersion in [
'CLIC_CDR', 'CDR1', 'CDR2', 'CDR0', 'V2', 'V3', 'V4'
]:
oldversion = True
scriptName = 'SLICPandora_%s_Run_%s.sh' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/bash \n')
script.write(
'#####################################################################\n'
)
script.write(
'# Dynamically generated script to run a production or analysis job. #\n'
)
script.write(
'#####################################################################\n'
)
script.write("source %s\n" % env_script_path)
script.write("declare -x file=./Settings/%s\n" % self.pandorasettings)
script.write("""
if [ -e "${file}" ]
then
declare -x PANDORASETTINGS=$file
else
if [ -d "${PANDORASETTINGSDIR}" ]
then
cp $PANDORASETTINGSDIR/*.xml .
declare -x PANDORASETTINGS=%s
fi
fi
if [ ! -e "${PANDORASETTINGS}" ]
then
echo "Missing PandoraSettings file"
exit 1
fi
""" % self.pandorasettings)
script.write('echo =============================\n')
script.write('echo PATH is \n')
script.write('echo $PATH | tr ":" "\n" \n')
script.write('echo ==============\n')
if os.path.exists("./lib"):
script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n')
script.write('echo =============================\n')
script.write('echo LD_LIBRARY_PATH is \n')
script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n')
script.write('echo ============================= \n')
script.write('env | sort >> localEnv.log\n')
#Now run it
if oldversion:
comm = 'PandoraFrontend %s $PANDORASETTINGS %s %s %s' % (
self.detectorxml, runonslcio, self.OutputFile,
str(self.NumberOfEvents))
else:
comm = 'PandoraFrontend -g %s -c $PANDORASETTINGS -i %s -o %s -r %s' % (
self.detectorxml, runonslcio, self.OutputFile,
str(self.NumberOfEvents))
comm = "%s %s\n" % (comm, self.extraCLIarguments)
self.log.info("Will run %s" % comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
#script.write('where\n')
#script.write('quit\n')
#script.write('EOF\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'sh -c "./%s"' % (scriptName)
self.setApplicationStatus('SLICPandora %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
comm,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
if resultTuple[0]:
self.log.error("There was an error during the execution")
if not os.path.exists(self.applicationLog):
self.log.error(
"Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' %
(self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' %
(self.applicationName))
logf = open(self.applicationLog, 'r')
if "Missing PandoraSettings file" in logf.readlines()[-1]:
self.log.error("Issue with the Pandora Settings file.")
logf.close()
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
self.log.info("Status after the application execution is %s" %
str(status))
return self.finalStatusReport(status)
def runIt(self):
"""
Called by Agent
Execute the following:
- resolve where the soft was installed
- prepare the list of file to feed Marlin with
- create the XML file on which Marlin has to run, done by :any:`prepareXMLFile`
- run Marlin and catch the exit code
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
#get the path to the detector model, either local or from the software
compactFile = None
if self.detectorModel:
resXML = self._getDetectorXML()
if not resXML['OK']:
self.log.error("Could not obtain the detector XML file: ", resXML["Message"])
return resXML
compactFile = resXML['Value']
res = getEnvironmentScript(self.platform, "marlin", self.applicationVersion, self.getEnvScript)
if not res['OK']:
self.log.error("Failed to get the env script")
return res
env_script_path = res["Value"]
res = self._getInputFiles()
if not res['OK']:
self.log.error("Failed getting input files:", res['Message'])
return res
listofslcio = res['Value']
finalXML = "marlinxml_" + self.STEP_NUMBER + ".xml"
steeringfiledirname = ''
res = getSteeringFileDirName(self.platform, "marlin", self.applicationVersion)
if res['OK']:
steeringfiledirname = res['Value']
else:
self.log.warn('Could not find the steering file directory', res['Message'])
##Handle PandoraSettings.xml
pandorasettings = 'PandoraSettings.xml'
if not os.path.exists(pandorasettings):
if steeringfiledirname and os.path.exists(os.path.join(steeringfiledirname, pandorasettings)):
try:
shutil.copy(os.path.join(steeringfiledirname, pandorasettings),
os.path.join(os.getcwd(), pandorasettings))
except EnvironmentError, x:
self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
def runIt(self):
"""
Called by JobAgent
Execute the following:
- get the environment variables that should have been set during installation
- prepare the steering file and command line parameters
- run Whizard2 on this steering file and catch the exit status
:rtype: :func:`~DIRAC.Core.Utilities.ReturnValues.S_OK`, :func:`~DIRAC.Core.Utilities.ReturnValues.S_ERROR`
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
LOG.error("Failed to resolve input parameters:", self.result['Message'])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('Whizard2 should not proceed as previous step did not end properly')
resIntProc = self.resolveIntegratedProcess()
if not resIntProc['OK']:
return resIntProc
# get the enviroment script
res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, S_ERROR("No init script provided in CVMFS!"))
if not res['OK']:
LOG.error("Could not obtain the environment script: ", res["Message"])
return res
envScriptPath = res["Value"]
whizard2SteerName = 'Whizard2_%s_Steer_%s.sin' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(whizard2SteerName):
os.remove(whizard2SteerName)
whizard2Steer = []
whizard2Steer.append('!Seed set via API')
whizard2Steer.append('seed = %s' % self.randomSeed)
whizard2Steer.append('')
whizard2Steer.append('!Parameters set via whizard2SinFile')
whizard2Steer.append('')
whizard2Steer.append(self.whizard2SinFile)
whizard2Steer.append('')
whizard2Steer.append('!Number of events set via API')
whizard2Steer.append('')
whizard2Steer.append('n_events = %s' % self.NumberOfEvents)
whizard2Steer.append('')
whizard2Steer.append('simulate (%s) {' % ",".join(self.decayProc))
whizard2Steer.append(' $sample = "%s"' % self.OutputFile.rsplit('.',1)[0] )
if self.OutputFile.rsplit('.',1)[-1] == 'slcio':
whizard2Steer.append(' sample_format = lcio')
whizard2Steer.append(' $extension_lcio = "slcio"')
else:
whizard2Steer.append(' sample_format = %s' % self.OutputFile.rsplit('.',1)[-1] )
whizard2Steer.append(' $extension_{st} = "{st}"'.format(st=self.OutputFile.rsplit('.',1)[-1]))
whizard2Steer.append('}')
with open(whizard2SteerName, 'w') as steerFile:
steerFile.write( "\n".join(whizard2Steer) )
scriptName = 'Whizard2_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = []
script.append('#!/bin/bash')
script.append('#####################################################################')
script.append('# Dynamically generated script to run a production or analysis job. #')
script.append('#####################################################################')
script.append('source %s' % envScriptPath)
script.append('echo =========')
script.append('env | sort >> localEnv.log')
script.append('echo whizard:`which whizard`')
script.append('echo =========')
script.append('whizard %s' % whizard2SteerName )
script.append('declare -x appstatus=$?')
script.append('exit $appstatus')
with open(scriptName, 'w') as scriptFile:
scriptFile.write( "\n".join(script) )
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0o755)
comm = 'bash "./%s"' % scriptName
self.setApplicationStatus('Whizard2 %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520)
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
LOG.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed to produce log file' % (self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log %s' % (self.applicationName, self.applicationLog))
status = resultTuple[0]
LOG.info("Status after the application execution is %s" % status)
return self.finalStatusReport(status)
def runIt(self):
"""
Called by JobAgent
Execute the following:
- get the environment variables that should have been set during installation
- download the detector model, using CS query to fetch the address
- prepare the mac file using :any:`prepareMacFile`
- run SLIC on this mac File and catch the exit status
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
LOG.error("Failed to resolve input parameters:", self.result['Message'])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('SLIC should not proceed as previous step did not end properly')
res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript)
if not res['OK']:
LOG.error("Could not obtain the environment script: ", res["Message"])
return res
env_script_path = res["Value"]
retMod = self.getDetectorModel()
if not retMod:
return retMod
slicmac = 'slicmac.mac'
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
LOG.error("Generator file not found")
return res
self.InputFile = res['Value']
if len(self.SteeringFile) > 0:
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
LOG.error("Could not find where the steering files are")
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile)
if not os.path.exists(self.SteeringFile):
LOG.error("Missing steering file")
return S_ERROR("Could not find mac file")
##Same as for mokka: using ParticleGun does not imply InputFile
if not len(self.InputFile):
self.InputFile = ['']
macok = prepareMacFile(self.SteeringFile, slicmac, self.InputFile[0],
self.NumberOfEvents, self.startFrom, self.detectorModel,
self.RandomSeed, self.OutputFile, self.debug)
if not macok['OK']:
LOG.error('Failed to create SLIC mac file')
return S_ERROR('Error when creating SLIC mac file')
scriptName = 'SLIC_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/bash \n')
script.write('#####################################################################\n')
script.write('# Dynamically generated script to run a production or analysis job. #\n')
script.write('#####################################################################\n')
script.write("source %s\n" % (env_script_path))
preloadDir = "%s/preload" % os.getcwd()
if os.path.exists( preloadDir ):
preadloadLibs = set( os.listdir( preloadDir ) )
preloadString =":".join( lib for lib in preadloadLibs if ".so" in lib )
script.write( 'declare -x LD_LIBRARY_PATH=%s:${LD_LIBRARY_PATH}\n' % preloadDir )
script.write( 'declare -x LD_PRELOAD=%s\n' % preloadString )
script.write('echo =========\n')
script.write('env | sort >> localEnv.log\n')
script.write('echo SLIC:\n')
script.write("which slic\n")
script.write('echo =========\n')
comm = 'slic -P $PARTICLE_TBL -m %s %s\n' % (slicmac, self.extraCLIarguments)
LOG.info("Command:", comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0o755)
comm = 'sh -c "./%s"' % scriptName
self.setApplicationStatus('SLIC %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
LOG.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' % (self.applicationName))
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
LOG.info("Status after the application execution is %s" % str(status))
return self.finalStatusReport(status)
def runIt(self):
"""
Called by JobAgent
Execute the following:
- get the environment variables that should have been set during installation
- find the detector model xml, using CS query to obtain the path
- prepare the steering file and command line parameters
- run DDSim on this steering file and catch the exit status
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:", self.result['Message'])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('DDSim should not proceed as previous step did not end properly')
##TODO: Setup LD_LIBRARY_PATH for extensions
res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript)
if not res['OK']:
self.log.error("Could not obtain the environment script: ", res["Message"])
return res
envScriptPath = res["Value"]
#get the path to the detector model, either local or from the software
resXML = self._getDetectorXML()
if not resXML['OK']:
self.log.error("Could not obtain the detector XML file: ", resXML["Message"])
return resXML
compactFile = resXML['Value']
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error("Generator file not found")
return res
self.InputFile = res['Value']
## if steering file is set try to find it
if len(self.SteeringFile) > 0:
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
self.log.error("Could not find where the steering files are")
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile)
if not os.path.exists(self.SteeringFile):
self.log.error("Missing steering file")
return S_ERROR("Could not find steering file")
self.extraCLIarguments += " --steeringFile %s " % self.SteeringFile
if self.startFrom:
self.extraCLIarguments += " --skipNEvents %s " % self.startFrom
if self.debug:
self.extraCLIarguments += " --printLevel DEBUG "
##Same as for mokka: using ParticleGun does not imply InputFile
if self.InputFile:
self.InputFile = [self.InputFile] if isinstance(self.InputFile, basestring) else self.InputFile
self.extraCLIarguments += " --inputFile %s " % self.InputFile[0]
if self.NumberOfEvents:
self.extraCLIarguments += " --numberOfEvents %s " % self.NumberOfEvents
self.extraCLIarguments += " --random.seed %s " % self.randomSeed
if self.OutputFile:
self.extraCLIarguments += " --outputFile %s " % self.OutputFile
scriptName = 'DDSim_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = []
script.append('#!/bin/bash')
script.append('#####################################################################')
script.append('# Dynamically generated script to run a production or analysis job. #')
script.append('#####################################################################')
script.append('source %s' % envScriptPath)
script.append('echo =========')
script.append('env | sort >> localEnv.log')
script.append('echo ddsim:`which ddsim`')
script.append('echo =========')
comm = 'ddsim --compactFile %(compactFile)s %(extraArgs)s' % dict(compactFile=compactFile,
extraArgs=self.extraCLIarguments)
self.log.info("Command:", comm)
script.append(comm)
script.append('declare -x appstatus=$?')
script.append('exit $appstatus')
with open(scriptName, 'w') as scriptFile:
scriptFile.write( "\n".join(script) )
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'bash "./%s"' % scriptName
self.setApplicationStatus('DDSim %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520)
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
self.log.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed to produce log file' % (self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log %s' % (self.applicationName, self.applicationLog))
status = resultTuple[0]
self.log.info( "Status after the application execution is %s" % status )
return self.finalStatusReport(status)
def runIt(self):
"""
Called by JobAgent
Execute the following:
- get the environment variables that should have been set during installation
- prepare the steering file and command line parameters
- run Whizard2 on this steering file and catch the exit status
:rtype: S_OK, S_ERROR
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:",
self.result['Message'])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'Whizard2 should not proceed as previous step did not end properly'
)
# get the enviroment script
res = getEnvironmentScript(
self.platform, self.applicationName, self.applicationVersion,
S_ERROR("No init script provided in CVMFS!"))
if not res['OK']:
self.log.error("Could not obtain the environment script: ",
res["Message"])
return res
envScriptPath = res["Value"]
whizard2SteerName = 'Whizard2_%s_Steer_%s.sin' % (
self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(whizard2SteerName):
os.remove(whizard2SteerName)
whizard2Steer = []
whizard2Steer.append('!Seed set via API')
whizard2Steer.append('seed = %s' % self.randomSeed)
whizard2Steer.append('')
whizard2Steer.append('!Parameters set via whizard2SinFile')
whizard2Steer.append('')
whizard2Steer.append(self.whizard2SinFile)
whizard2Steer.append('')
whizard2Steer.append('!Number of events set via API')
whizard2Steer.append('')
whizard2Steer.append('n_events = %s' % self.NumberOfEvents)
whizard2Steer.append('')
whizard2Steer.append('simulate (decay_proc) {')
whizard2Steer.append(' $sample = "%s"' %
self.OutputFile.rsplit('.', 1)[0])
if self.OutputFile.rsplit('.', 1)[-1] == 'slcio':
whizard2Steer.append(' sample_format = lcio')
whizard2Steer.append(' $extension_lcio = "slcio"')
else:
whizard2Steer.append(' sample_format = %s' %
self.OutputFile.rsplit('.', 1)[-1])
whizard2Steer.append(' $extension_{st} = "{st}"'.format(
st=self.OutputFile.rsplit('.', 1)[-1]))
whizard2Steer.append('}')
with open(whizard2SteerName, 'w') as steerFile:
steerFile.write("\n".join(whizard2Steer))
scriptName = 'Whizard2_%s_Run_%s.sh' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = []
script.append('#!/bin/bash')
script.append(
'#####################################################################'
)
script.append(
'# Dynamically generated script to run a production or analysis job. #'
)
script.append(
'#####################################################################'
)
script.append('source %s' % envScriptPath)
script.append('echo =========')
script.append('env | sort >> localEnv.log')
script.append('echo whizard:`which whizard`')
script.append('echo =========')
script.append('whizard %s' % whizard2SteerName)
script.append('declare -x appstatus=$?')
script.append('exit $appstatus')
with open(scriptName, 'w') as scriptFile:
scriptFile.write("\n".join(script))
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'bash "./%s"' % scriptName
self.setApplicationStatus('Whizard2 %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
comm,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
self.log.error(
"Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed to produce log file' %
(self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log %s' %
(self.applicationName, self.applicationLog))
status = resultTuple[0]
self.log.info("Status after the application execution is %s" % status)
return self.finalStatusReport(status)