def runIt(self):
""" Called from Workflow
"""
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
if not self.OutputFile:
self.log.error("OutputFile not specified")
return S_ERROR("OutputFile not specified")
res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
self.log.error('Application %s was not found in either the local area or shared area' % self.applicationName)
self.setApplicationStatus('%s: Could not find neither local area not shared area install' % self.applicationName)
return res
mySoftDir = res['Value']
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
self.log.verbose('Getting the steering files directory')
res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
self.log.error("Could not locate the steering file directory")
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
try:
shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile )
except EnvironmentError, x:
self.log.error("Failed to get the cuts file")
return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(x)))
def test_getsteerfiledirname_fails( self ):
getval_mock = Mock()
getval_mock.getValue.return_value = ''
ops_mock = Mock()
ops_mock.return_value = getval_mock
with patch("%s.Operations" % MODULE_NAME, new=ops_mock):
assertDiracFailsWith( getSteeringFileDirName( 'mytest_plat', 'myappTestme', 'vTest1.0' ),
'could not find attached steeringfile version', self )
def runIt(self):
"""
Called by Agent
Execute the following:
- resolve where the soft was installed
- prepare the list of file to feed Marlin with
- create the XML file on which Marlin has to run, done by :any:`prepareXMLFile`
- run Marlin and catch the exit code
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
res = getEnvironmentScript(self.platform, "marlin", self.applicationVersion, self.getEnvScript)
if not res['OK']:
self.log.error("Failed to get the env script")
return res
env_script_path = res["Value"]
res = self.GetInputFiles()
if not res['OK']:
self.log.error("Failed getting input files:", res['Message'])
return res
listofslcio = res['Value']
finalXML = "marlinxml_" + self.STEP_NUMBER + ".xml"
steeringfiledirname = ''
res = getSteeringFileDirName(self.platform, "marlin", self.applicationVersion)
if res['OK']:
steeringfiledirname = res['Value']
else:
self.log.warn('Could not find the steering file directory', res['Message'])
##Handle PandoraSettings.xml
pandorasettings = 'PandoraSettings.xml'
if not os.path.exists(pandorasettings):
if steeringfiledirname and os.path.exists(os.path.join(steeringfiledirname, pandorasettings)):
try:
shutil.copy(os.path.join(steeringfiledirname, pandorasettings),
os.path.join(os.getcwd(), pandorasettings))
except EnvironmentError, x:
self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
def test_getsteerfiledirname_fails(self):
getval_mock = Mock()
getval_mock.getValue.return_value = ''
ops_mock = Mock()
ops_mock.return_value = getval_mock
with patch("%s.Operations" % MODULE_NAME, new=ops_mock):
assertDiracFailsWith(
getSteeringFileDirName('mytest_plat', 'myappTestme',
'vTest1.0'),
'could not find attached steeringfile version', self)
def test_getsteerfiledirname( self ):
getval_mock = Mock()
getval_mock.getValue.return_value = 'my_test_getval'
ops_mock = Mock()
ops_mock.return_value = getval_mock
cvmfs_mock = Mock(return_value=S_ERROR('some_err'))
soft_mock = Mock(return_value=S_OK('my_test_dir_retval'))
check_mock = Mock(return_value=S_OK('ignoreme'))
with patch("%s.Operations" % MODULE_NAME, new=ops_mock), \
patch('%s.checkCVMFS' % MODULE_NAME, new=cvmfs_mock), \
patch('%s.getSoftwareFolder' % MODULE_NAME, new=soft_mock), \
patch('%s.check' % MODULE_NAME, new=check_mock):
assertDiracSucceedsWith_equals( getSteeringFileDirName( 'mytest_plat', 'myappTestme', 'vTest1.0' ),
'my_test_dir_retval', self )
getval_mock.getValue.assert_called_once_with(
'/AvailableTarBalls/mytest_plat/myappTestme/vTest1.0/Dependencies/steeringfiles/version', '' )
def runIt(self):
""" Called from Workflow
"""
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'%s should not proceed as previous step did not end properly' %
self.applicationName)
if not self.OutputFile:
self.log.error("OutputFile not specified")
return S_ERROR("OutputFile not specified")
res = getSoftwareFolder(self.platform, self.applicationName,
self.applicationVersion)
if not res['OK']:
self.log.error(
'Application %s was not found in either the local area or shared area'
% self.applicationName)
self.setApplicationStatus(
'%s: Could not find neither local area not shared area install'
% self.applicationName)
return res
mySoftDir = res['Value']
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
self.log.verbose('Getting the steering files directory')
res = getSteeringFileDirName(self.platform, self.applicationName,
self.applicationVersion)
if not res['OK']:
self.log.error("Could not locate the steering file directory")
return res
steeringfiledirname = res['Value']
if os.path.exists(
os.path.join(steeringfiledirname, self.SteeringFile)):
try:
shutil.copy(
os.path.join(steeringfiledirname, self.SteeringFile),
"./" + self.SteeringFile)
except EnvironmentError, x:
self.log.error("Failed to get the cuts file")
return S_ERROR('Failed to access file %s: %s' %
(self.SteeringFile, str(x)))
def test_getsteerfiledirname(self):
getval_mock = Mock()
getval_mock.getValue.return_value = 'my_test_getval'
ops_mock = Mock()
ops_mock.return_value = getval_mock
cvmfs_mock = Mock(return_value=S_ERROR('some_err'))
soft_mock = Mock(return_value=S_OK('my_test_dir_retval'))
check_mock = Mock(return_value=S_OK('ignoreme'))
with patch("%s.Operations" % MODULE_NAME, new=ops_mock), \
patch('%s.checkCVMFS' % MODULE_NAME, new=cvmfs_mock), \
patch('%s.getSoftwareFolder' % MODULE_NAME, new=soft_mock), \
patch('%s.check' % MODULE_NAME, new=check_mock):
assertDiracSucceedsWith_equals(
getSteeringFileDirName('mytest_plat', 'myappTestme',
'vTest1.0'), 'my_test_dir_retval', self)
getval_mock.getValue.assert_called_once_with(
'/AvailableTarBalls/mytest_plat/myappTestme/vTest1.0/Dependencies/steeringfiles/version',
'')
def execute(self):
""" Called from Workflow
"""
self.result = self.resolveInputVariables()
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('StdHepCut should not proceed as previous step did not end properly')
appDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall'% (self.systemConfig, "stdhepcut",
self.applicationVersion), '')
if not appDir:
self.log.error('Could not get info from CS')
self.setApplicationStatus('Failed finding info from CS')
return S_ERROR('Failed finding info from CS')
appDir = appDir.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(appDir)
if not res['OK']:
self.setApplicationStatus('%s: Could not find neither local area not shared area install' % self.applicationName)
return res
mySoftDir = res['Value']
new_ld_lib_path = GetNewLDLibs(self.systemConfig, self.applicationName, self.applicationVersion)
new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path
if os.path.exists("./lib"):
new_ld_lib_path = "./lib:" + new_ld_lib_path
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
res = getSteeringFileDirName(self.systemConfig, self.applicationName, self.applicationVersion)
if not res['OK']:
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
try:
shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile )
except Exception, x:
return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(x)))
def runIt(self):
"""
Called by JobAgent
Execute the following:
- get the environment variables that should have been set during installation
- find the detector model xml, using CS query to obtain the path
- prepare the steering file and command line parameters
- run DDSim on this steering file and catch the exit status
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:", self.result['Message'])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('DDSim should not proceed as previous step did not end properly')
##TODO: Setup LD_LIBRARY_PATH for extensions
res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript)
if not res['OK']:
self.log.error("Could not obtain the environment script: ", res["Message"])
return res
envScriptPath = res["Value"]
#get the path to the detector model, either local or from the software
resXML = self._getDetectorXML()
if not resXML['OK']:
self.log.error("Could not obtain the detector XML file: ", resXML["Message"])
return resXML
compactFile = resXML['Value']
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error("Generator file not found")
return res
self.InputFile = res['Value']
## if steering file is set try to find it
if len(self.SteeringFile) > 0:
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
self.log.error("Could not find where the steering files are")
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile)
if not os.path.exists(self.SteeringFile):
self.log.error("Missing steering file")
return S_ERROR("Could not find steering file")
self.extraCLIarguments += " --steeringFile %s " % self.SteeringFile
if self.startFrom:
self.extraCLIarguments += " --skipNEvents %s " % self.startFrom
if self.debug:
self.extraCLIarguments += " --printLevel DEBUG "
##Same as for mokka: using ParticleGun does not imply InputFile
if self.InputFile:
self.InputFile = [self.InputFile] if isinstance(self.InputFile, basestring) else self.InputFile
self.extraCLIarguments += " --inputFile %s " % self.InputFile[0]
if self.NumberOfEvents:
self.extraCLIarguments += " --numberOfEvents %s " % self.NumberOfEvents
self.extraCLIarguments += " --random.seed %s " % self.randomSeed
if self.OutputFile:
self.extraCLIarguments += " --outputFile %s " % self.OutputFile
scriptName = 'DDSim_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = []
script.append('#!/bin/bash')
script.append('#####################################################################')
script.append('# Dynamically generated script to run a production or analysis job. #')
script.append('#####################################################################')
script.append('source %s' % envScriptPath)
script.append('echo =========')
script.append('env | sort >> localEnv.log')
script.append('echo ddsim:`which ddsim`')
script.append('echo =========')
comm = 'ddsim --compactFile %(compactFile)s %(extraArgs)s' % dict(compactFile=compactFile,
extraArgs=self.extraCLIarguments)
self.log.info("Command:", comm)
script.append(comm)
script.append('declare -x appstatus=$?')
script.append('exit $appstatus')
with open(scriptName, 'w') as scriptFile:
scriptFile.write( "\n".join(script) )
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'bash "./%s"' % scriptName
self.setApplicationStatus('DDSim %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520)
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
self.log.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed to produce log file' % (self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log %s' % (self.applicationName, self.applicationLog))
status = resultTuple[0]
self.log.info( "Status after the application execution is %s" % status )
return self.finalStatusReport(status)
def runIt(self):
""" Called by ModuleBase
Executes the following:
- read the application parameters that where defined in ILCJob, and stored in the job definition
- setup the SQL server and run it in the background
- prepare the steering file using :any:`prepareSteeringFile`
- run Mokka and catch its return status
:return: S_OK(), S_ERROR()
"""
#if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
# self.log.info('Skip this module, failure detected in a previous step :')
# self.log.info('Workflow status : %s' %(self.workflowStatus))
# self.log.info('Step Status %s' %(self.stepStatus))
# return S_OK()
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error("Failed to resolve the input parameters:",
self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'Mokka should not proceed as previous step did not end properly'
)
self.log.info("Executing Mokka %s" % (self.applicationVersion))
if self.dbSlice:
if self.dbSlice.lower().startswith("lfn"):
self.dbSlice = os.path.basename(self.dbSlice)
self.db_dump_name = self.dbSlice
else:
self.db_dump_name = "CLICMokkaDB.sql"
res = getEnvironmentScript(self.platform, "mokka",
self.applicationVersion, self.getEnvScript)
self.log.notice("Got the environment script: %s" % res)
if not res['OK']:
self.log.error("Error getting the env script: ", res['Message'])
return res
env_script_path = res['Value']
####Setup MySQL instance
mysqlBasePath = '%s/mysqltmp/MokkaDBRoot-%s' % (
os.getcwd(), generateRandomString(8))
self.log.notice("Placing mysql files in %s" % mysqlBasePath)
###steering file that will be used to run
mokkasteer = "mokka.steer"
if os.path.exists("mokka.steer"):
try:
os.rename("mokka.steer", "mymokka.steer")
except EnvironmentError as err:
self.log.error("Failed renaming the steering file: ", str(err))
self.SteeringFile = "mymokka.steer"
###prepare steering file
#first, I need to take the stdhep file, find its path (possible LFN)
if len(self.InputFile) > 0:
#self.InputFile = os.path.basename(self.InputFile)
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error("Generator file not found")
return res
self.InputFile = res['Value']
if len(self.macFile) > 0:
self.macFile = os.path.basename(self.macFile)
##idem for steering file
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
self.log.verbose("Steering file %s not found locally" %
self.SteeringFile)
res = getSteeringFileDirName(self.platform, "mokka",
self.applicationVersion)
if not res['OK']:
self.log.error("Missing Steering file directory:",
res['Message'])
return res
steeringfiledirname = res['Value']
if os.path.exists(
os.path.join(steeringfiledirname, self.SteeringFile)):
try:
shutil.copy(
os.path.join(steeringfiledirname, self.SteeringFile),
"./" + self.SteeringFile)
except EnvironmentError as err:
self.log.error("Failed copying file", self.SteeringFile)
return S_ERROR('Failed to access file %s: %s' %
(self.SteeringFile, str(err)))
#self.steeringFile = os.path.join(mySoftwareRoot,"steeringfiles",self.steeringFile)
if not os.path.exists(self.SteeringFile):
self.log.error("Missing steering file, should not happen!")
return S_ERROR("Could not find steering file")
else:
self.log.verbose("Found local copy of %s" % self.SteeringFile)
### The following is because if someone uses particle gun, there is no InputFile
if not len(self.InputFile):
self.InputFile = ['']
steerok = prepareSteeringFile(
self.SteeringFile, mokkasteer, self.detectorModel,
self.InputFile[0], self.macFile, self.NumberOfEvents,
self.startFrom, self.RandomSeed, self.mcRunNumber, self.ProcessID,
self.debug, self.OutputFile, self.inputdataMeta)
if not steerok['OK']:
self.log.error('Failed to create MOKKA steering file')
return S_ERROR('Failed to create MOKKA steering file')
###Extra option depending on mokka version
mokkaextraoption = ""
if self.applicationVersion not in [
"v07-02", "v07-02fw", "v07-02fwhp", "MokkaRevision42",
"MokkaRevision43", "MokkaRevision44", "Revision45"
]:
mokkaextraoption = "-U"
scriptName = 'Mokka_%s_Run_%s.sh' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/bash \n')
script.write(
'#####################################################################\n'
)
script.write(
'# Dynamically generated script to run a production or analysis job. #\n'
)
script.write(
'#####################################################################\n'
)
script.write("source %s\n" % env_script_path)
if os.path.exists("./lib"):
script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n')
script.write('echo =============================\n')
script.write('echo Content of mokka.steer:\n')
script.write('cat mokka.steer\n')
if self.macFile:
script.write('echo =============================\n')
script.write('echo Content of %s\n' %
os.path.basename(self.macFile))
script.write("cat %s\n" % os.path.basename(self.macFile))
elif os.path.exists("./mokkamac.mac"):
script.write("""
echo =============================
echo Content of mokkamac.mac:
cat mokkamac.mac
""")
script.write('echo =============================\n')
script.write('echo LD_LIBRARY_PATH is\n')
script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n')
script.write('echo =============================\n')
script.write('echo PATH is\n')
script.write('echo $PATH | tr ":" "\n"\n')
if self.debug:
script.write('echo ldd of Mokka is\n')
script.write('ldd `which Mokka` \n')
script.write('echo =============================\n')
script.write('echo ldd of mysql is\n')
script.write('ldd `which mysql` \n')
script.write('echo =============================\n')
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write("""if [ ! -e ./%(DBDUMP)s ]; then
echo "Missing DB dump!" >&2
exit 1
fi
""" % {'DBDUMP': self.db_dump_name})
##Now start the MySQL server and configure.
script.write("declare -x MOKKADBROOT=%s\n" % mysqlBasePath)
script.write("declare -x WORKDIR=%s\n" % os.getcwd())
script.write("declare -x LOGDIR=$WORKDIR/mysqllogs/\n")
script.write("mkdir -p $LOGDIR\n")
#mysql socket must not be longer than 107 characters!
script.write("declare -x SOCKETBASE=/tmp/mokka-%s\n" %
generateRandomString(8))
script.write("mkdir -p $SOCKETBASE\n")
script.write("declare -x SOCKETPATH=$SOCKETBASE/mysql.sock\n")
script.write("mkdir -p $MOKKADBROOT\n")
script.write("declare -x MYSQLDATA=$MOKKADBROOT/data\n")
script.write("rm -rf $MYSQLDATA\n")
script.write("mkdir -p $MYSQLDATA\n")
script.write("date\n")
script.write("""echo "*** Installing MySQL"
echo "mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log"
mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log
install_st=$?
if [ $install_st -ne 0 ]
then
exit $install_st
fi
date
echo "*** Running mysqld-safe"
cd $MYSQL
bin/mysqld_safe --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log &
COUNT=0
while [ -z "$socket_grep" ] ; do
socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH")
echo -n .
sleep 1
if [ $COUNT -eq 100 ]; then
echo "Failed to find socket"
break
fi
COUNT=$(( $COUNT + 1 ))
done
cd -
echo "*** Configuring MySQL"
mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot password 'rootpass'
declare -x admin_st=$?
if [ $admin_st -ne 0 ]
then
echo "*** mysqladmin failed, cannot proceed" >&2
mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown
exit $admin_st
fi
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO root;'
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO consult IDENTIFIED BY \"consult\";'
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'DELETE FROM mysql.user WHERE User = \"\"; FLUSH PRIVILEGES;'
echo "*** Installing Mokka DB"
date
echo "mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s"
time mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s
sleep 5
date\n""" % {'DBDUMP': self.db_dump_name})
#Now take care of the particle tables.
script.write("""
if [ -e "./particle.tbl" ]
then
declare -x PARTICLETBL=./particle.tbl
fi
if [ -d "${MOKKA}" ]
#As the local env does not set MOKKA, it must come from CVMFS
then
declare -x PARTICLETBL=$MOKKA/particle.tbl
fi
if [ -d "${MOKKATARBALL}" ]
#As the local env does set MOKKATARBALL, it must be there
then
declare -x PARTICLETBL=$MOKKATARBALL/ConfigFiles/particle.tbl
fi
echo "/Mokka/init/PDGFile $PARTICLETBL" >> %s
\n""" % mokkasteer)
##Now run Mokka
comm = 'Mokka %s -h localhost:$SOCKETPATH %s %s\n' % (
mokkaextraoption, mokkasteer, self.extraCLIarguments)
self.log.info("Command : %s" % (comm))
script.write(comm)
script.write('declare -x appstatus=$?\n')
#Now shutdown the MySQL server
script.write(
"mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown\n"
)
script.write("""
while [ -n "$socket_grep" ] ; do
socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH")
echo -n .
sleep 1
done
""")
script.write("rm -f %s\n" % self.db_dump_name) #remove db file
script.write('rm -rf $MYSQLDATA\n') #cleanup
script.write('rm -rf $MOKKADBROOT\n') #cleanup
#script.write('quit\n')
#script.write('EOF\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'sh -c "./%s"' % scriptName
self.setApplicationStatus('Mokka %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
comm,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
self.log.info("Status after Mokka execution is %s" % str(status))
if not os.path.exists(self.applicationLog):
self.log.error(
"Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' %
(self.applicationName))
if not self.ignoreapperrors:
self.log.error("Missing log file")
return S_ERROR('%s did not produce the expected log' %
(self.applicationName))
###Now change the name of Mokka output to the specified filename
if os.path.exists("out.slcio"):
if len(self.OutputFile) > 0:
os.rename("out.slcio", self.OutputFile)
failed = False
if status not in [0, 106, 9]:
self.log.error("Mokka execution completed with errors:")
failed = True
elif status in [106, 9]:
self.log.info(
"Mokka execution reached end of input generator file")
else:
self.log.info("Mokka execution finished successfully")
message = 'Mokka %s Successful' % (self.applicationVersion)
if failed is True:
self.log.error("==================================\n StdError:\n")
self.log.error(self.stdError)
#self.setApplicationStatus('%s Exited With Status %s' %(self.applicationName,status))
self.log.error('Mokka Exited With Status %s' % (status))
message = 'Mokka Exited With Status %s' % (status)
self.setApplicationStatus(message)
if not self.ignoreapperrors:
self.log.error("Application exists with error:", message)
return S_ERROR(message)
else:
if status in [106, 9]:
message = 'Mokka %s reached end of input generator file' % (
self.applicationVersion)
self.setApplicationStatus(message)
return S_OK({'OutputFile': self.OutputFile})
os.mkdir(os.path.join(lcsimfolder, "detectors"))
except OSError, x:
self.log.error("Could not create detectors folder !", str(x))
if os.path.exists(os.path.join(lcsimfolder, "detectors")):
self.log.verbose("Copy detector model.zip into the .lcsim/detectors folder")
shutil.copy(os.path.basename(self.detectorModel),
os.path.join(lcsimfolder, "detectors", os.path.basename(self.detectorModel)))
paths = {}
paths[os.path.basename(self.SteeringFile)] = os.path.basename(self.SteeringFile)
paths[os.path.basename(self.trackingstrategy)] = os.path.basename(self.trackingstrategy)
for myfile in paths.keys():
if len(myfile):
#file = os.path.basename(file)
if not os.path.exists(myfile):
res = getSteeringFileDirName(self.systemConfig, "lcsim", self.applicationVersion)
if not res['OK']:
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, myfile)):
paths[myfile] = os.path.join(steeringfiledirname, myfile)
if not os.path.exists(paths[myfile]):
return S_ERROR("Could not find file %s" % paths[myfile])
self.SteeringFile = paths[os.path.basename(self.SteeringFile)]
self.trackingstrategy = paths[os.path.basename(self.trackingstrategy)]
lcsimfile = "job_%s.lcsim" % self.STEP_NUMBER
res = PrepareLCSIMFile(self.SteeringFile, lcsimfile, self.NumberOfEvents,
self.trackingstrategy, runonslcio, jars, cachedir,
self.OutputFile, self.outputREC, self.outputDST, self.debug)
if not res['OK']:
def execute(self):
"""
Called by Agent
Execute the following:
- resolve where the soft was installed
- prepare the list of file to feed Marlin with
- create the XML file on which Marlin has to run, done by L{PrepareXMLFile}
- run Marlin and catch the exit code
@return: S_OK(), S_ERROR()
"""
self.result = self.resolveInputVariables()
if not self.systemConfig:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
marlinDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig,
"marlin",
self.applicationVersion), '')
marlinDir = marlinDir.replace(".tgz", "").replace(".tar.gz", "")
res = getSoftwareFolder(marlinDir)
if not res['OK']:
self.setApplicationStatus('Marlin: Could not find neither local area not shared area install')
return res
myMarlinDir = res['Value']
##Remove libc
removeLibc(myMarlinDir + "/LDLibs")
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = GetNewLDLibs(self.systemConfig, "marlin", self.applicationVersion)
res = self.GetInputFiles()
if not res['OK']:
self.log.error(res['Message'])
return res
listofslcio = res['Value']
finalXML = "marlinxml_" + self.STEP_NUMBER + ".xml"
steeringfiledirname = ''
res = getSteeringFileDirName(self.systemConfig, "marlin", self.applicationVersion)
if res['OK']:
steeringfiledirname = res['Value']
else:
self.log.warn('Could not find the steering file directory')
##Handle PandoraSettings.xml
pandorasettings = 'PandoraSettings.xml'
if not os.path.exists(pandorasettings):
if os.path.exists(os.path.join(myMarlinDir, 'Settings', pandorasettings)):
try:
shutil.copy(os.path.join(myMarlinDir, 'Settings', pandorasettings),
os.path.join(os.getcwd(), pandorasettings))
except Exception, x:
self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
elif steeringfiledirname and os.path.exists(os.path.join(steeringfiledirname,pandorasettings)):
try:
shutil.copy(os.path.join(steeringfiledirname, pandorasettings),
os.path.join(os.getcwd(), pandorasettings))
except Exception, x:
self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
os.mkdir(os.path.join(lcsimfolder, "detectors"))
except OSError, x:
LOG.error("Could not create detectors folder !", str(x))
if os.path.exists(os.path.join(lcsimfolder, "detectors")):
LOG.verbose("Copy detector model.zip into the .lcsim/detectors folder")
shutil.copy(os.path.basename(self.detectorModel),
os.path.join(lcsimfolder, "detectors", os.path.basename(self.detectorModel)))
paths = {}
paths[os.path.basename(self.SteeringFile)] = os.path.basename(self.SteeringFile)
paths[os.path.basename(self.trackingstrategy)] = os.path.basename(self.trackingstrategy)
for myfile in paths:
if len(myfile):
#file = os.path.basename(file)
if not os.path.exists(myfile):
res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
LOG.error('Failed finding the steering file folder:', res["Message"])
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, myfile)):
paths[myfile] = os.path.join(steeringfiledirname, myfile)
if not os.path.exists(paths[myfile]):
LOG.error('Failed finding file', paths[myfile])
return S_ERROR("Could not find file %s" % paths[myfile])
self.SteeringFile = paths[os.path.basename(self.SteeringFile)]
self.trackingstrategy = paths[os.path.basename(self.trackingstrategy)]
lcsimfile = "job_%s.lcsim" % self.STEP_NUMBER
res = prepareLCSIMFile(self.SteeringFile, lcsimfile, self.NumberOfEvents,
self.trackingstrategy, runonslcio, jars, cachedir,
def runIt(self):
""" Called by ModuleBase
Executes the following:
- read the application parameters that where defined in ILCJob, and stored in the job definition
- setup the SQL server and run it in the background
- prepare the steering file using :any:`prepareSteeringFile`
- run Mokka and catch its return status
:return: S_OK(), S_ERROR()
"""
#if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
# LOG.info('Skip this module, failure detected in a previous step :')
# LOG.info('Workflow status : %s' %(self.workflowStatus))
# LOG.info('Step Status %s' %(self.stepStatus))
# return S_OK()
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
LOG.error("Failed to resolve the input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('Mokka should not proceed as previous step did not end properly')
LOG.info("Executing Mokka %s" % (self.applicationVersion))
if self.dbSlice:
if self.dbSlice.lower().startswith("lfn"):
self.dbSlice = os.path.basename(self.dbSlice)
self.db_dump_name = self.dbSlice
else:
self.db_dump_name = "CLICMokkaDB.sql"
res = getEnvironmentScript(self.platform, "mokka", self.applicationVersion, self.getEnvScript)
LOG.notice("Got the environment script: %s" % res)
if not res['OK']:
LOG.error("Error getting the env script: ", res['Message'])
return res
env_script_path = res['Value']
####Setup MySQL instance
mysqlBasePath = '%s/mysqltmp/MokkaDBRoot-%s' %(os.getcwd(), generateRandomString(8))
LOG.notice("Placing mysql files in %s" % mysqlBasePath)
###steering file that will be used to run
mokkasteer = "mokka.steer"
if os.path.exists("mokka.steer"):
try:
os.rename("mokka.steer", "mymokka.steer")
except EnvironmentError as err:
LOG.error("Failed renaming the steering file: ", str(err))
self.SteeringFile = "mymokka.steer"
###prepare steering file
#first, I need to take the stdhep file, find its path (possible LFN)
if len(self.InputFile) > 0:
#self.InputFile = os.path.basename(self.InputFile)
res = resolveIFpaths(self.InputFile)
if not res['OK']:
LOG.error("Generator file not found")
return res
self.InputFile = res['Value']
if len(self.macFile) > 0:
self.macFile = os.path.basename(self.macFile)
##idem for steering file
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
LOG.verbose("Steering file %s not found locally" % self.SteeringFile)
res = getSteeringFileDirName(self.platform, "mokka", self.applicationVersion)
if not res['OK']:
LOG.error("Missing Steering file directory:", res['Message'])
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
try:
shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile )
except EnvironmentError as err:
LOG.error("Failed copying file", self.SteeringFile)
return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(err)))
#self.steeringFile = os.path.join(mySoftwareRoot,"steeringfiles",self.steeringFile)
if not os.path.exists(self.SteeringFile):
LOG.error("Missing steering file, should not happen!")
return S_ERROR("Could not find steering file")
else:
LOG.verbose("Found local copy of %s" % self.SteeringFile)
### The following is because if someone uses particle gun, there is no InputFile
if not len(self.InputFile):
self.InputFile = ['']
steerok = prepareSteeringFile(self.SteeringFile, mokkasteer, self.detectorModel, self.InputFile[0],
self.macFile, self.NumberOfEvents, self.startFrom, self.RandomSeed,
self.mcRunNumber,
self.ProcessID,
self.debug,
self.OutputFile,
self.inputdataMeta)
if not steerok['OK']:
LOG.error('Failed to create MOKKA steering file')
return S_ERROR('Failed to create MOKKA steering file')
###Extra option depending on mokka version
mokkaextraoption = ""
if self.applicationVersion not in ["v07-02", "v07-02fw", "v07-02fwhp", "MokkaRevision42", "MokkaRevision43",
"MokkaRevision44", "Revision45"]:
mokkaextraoption = "-U"
scriptName = 'Mokka_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/bash \n')
script.write('#####################################################################\n')
script.write('# Dynamically generated script to run a production or analysis job. #\n')
script.write('#####################################################################\n')
script.write("source %s\n" % env_script_path)
if os.path.exists("./lib"):
script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n' )
script.write('echo =============================\n')
script.write('echo Content of mokka.steer:\n')
script.write('cat mokka.steer\n')
if self.macFile:
script.write('echo =============================\n')
script.write('echo Content of %s\n' % os.path.basename(self.macFile))
script.write("cat %s\n" % os.path.basename(self.macFile))
elif os.path.exists("./mokkamac.mac"):
script.write("""
echo =============================
echo Content of mokkamac.mac:
cat mokkamac.mac
""")
script.write('echo =============================\n')
script.write('echo LD_LIBRARY_PATH is\n')
script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n')
script.write('echo =============================\n')
script.write('echo PATH is\n')
script.write('echo $PATH | tr ":" "\n"\n')
if self.debug:
script.write('echo ldd of Mokka is\n')
script.write('ldd `which Mokka` \n' )
script.write('echo =============================\n')
script.write('echo ldd of mysql is\n')
script.write('ldd `which mysql` \n' )
script.write('echo =============================\n')
script.write('env | sort >> localEnv.log\n')
script.write('echo =============================\n')
script.write("""if [ ! -e ./%(DBDUMP)s ]; then
echo "Missing DB dump!" >&2
exit 1
fi
""" % {'DBDUMP': self.db_dump_name} )
##Now start the MySQL server and configure.
script.write("declare -x MOKKADBROOT=%s\n" % mysqlBasePath)
script.write("declare -x WORKDIR=%s\n" % os.getcwd())
script.write("declare -x LOGDIR=$WORKDIR/mysqllogs/\n")
script.write("mkdir -p $LOGDIR\n")
#mysql socket must not be longer than 107 characters!
script.write("declare -x SOCKETBASE=/tmp/mokka-%s\n" % generateRandomString(8) )
script.write("mkdir -p $SOCKETBASE\n")
script.write("declare -x SOCKETPATH=$SOCKETBASE/mysql.sock\n" )
script.write("mkdir -p $MOKKADBROOT\n")
script.write("declare -x MYSQLDATA=$MOKKADBROOT/data\n")
script.write("rm -rf $MYSQLDATA\n")
script.write("mkdir -p $MYSQLDATA\n")
script.write("date\n")
script.write("""echo "*** Installing MySQL"
echo "mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log"
mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log
install_st=$?
if [ $install_st -ne 0 ]
then
exit $install_st
fi
date
echo "*** Running mysqld-safe"
cd $MYSQL
bin/mysqld_safe --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log &
COUNT=0
while [ -z "$socket_grep" ] ; do
socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH")
echo -n .
sleep 1
if [ $COUNT -eq 100 ]; then
echo "Failed to find socket"
break
fi
COUNT=$(( $COUNT + 1 ))
done
cd -
echo "*** Configuring MySQL"
mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot password 'rootpass'
declare -x admin_st=$?
if [ $admin_st -ne 0 ]
then
echo "*** mysqladmin failed, cannot proceed" >&2
mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown
exit $admin_st
fi
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO root;'
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO consult IDENTIFIED BY \"consult\";'
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'DELETE FROM mysql.user WHERE User = \"\"; FLUSH PRIVILEGES;'
echo "*** Installing Mokka DB"
date
echo "mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s"
time mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s
sleep 5
date\n""" % {'DBDUMP': self.db_dump_name})
#Now take care of the particle tables.
script.write("""
if [ -e "./particle.tbl" ]
then
declare -x PARTICLETBL=./particle.tbl
fi
if [ -d "${MOKKA}" ]
#As the local env does not set MOKKA, it must come from CVMFS
then
declare -x PARTICLETBL=$MOKKA/particle.tbl
fi
if [ -d "${MOKKATARBALL}" ]
#As the local env does set MOKKATARBALL, it must be there
then
declare -x PARTICLETBL=$MOKKATARBALL/ConfigFiles/particle.tbl
fi
echo "/Mokka/init/PDGFile $PARTICLETBL" >> %s
\n"""% mokkasteer)
##Now run Mokka
comm = 'Mokka %s -h localhost:$SOCKETPATH %s %s\n' % (mokkaextraoption, mokkasteer, self.extraCLIarguments)
LOG.info("Command : %s" % (comm))
script.write(comm)
script.write('declare -x appstatus=$?\n')
#Now shutdown the MySQL server
script.write("mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown\n")
script.write("""
while [ -n "$socket_grep" ] ; do
socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH")
echo -n .
sleep 1
done
""")
script.write("rm -f %s\n" % self.db_dump_name)#remove db file
script.write('rm -rf $MYSQLDATA\n')#cleanup
script.write('rm -rf $MOKKADBROOT\n')#cleanup
#script.write('quit\n')
#script.write('EOF\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0o755)
comm = 'sh -c "./%s"' % scriptName
self.setApplicationStatus('Mokka %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
LOG.info("Status after Mokka execution is %s" % str(status))
if not os.path.exists(self.applicationLog):
LOG.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
if not self.ignoreapperrors:
LOG.error("Missing log file")
return S_ERROR('%s did not produce the expected log' % (self.applicationName))
###Now change the name of Mokka output to the specified filename
if os.path.exists("out.slcio"):
if len(self.OutputFile) > 0:
os.rename("out.slcio", self.OutputFile)
failed = False
if status not in [0, 106, 9]:
LOG.error("Mokka execution completed with errors:")
failed = True
elif status in [106, 9]:
LOG.info("Mokka execution reached end of input generator file")
else:
LOG.info("Mokka execution finished successfully")
message = 'Mokka %s Successful' % (self.applicationVersion)
if failed is True:
LOG.error("==================================\n StdError:\n")
LOG.error(self.stdError)
#self.setApplicationStatus('%s Exited With Status %s' %(self.applicationName,status))
LOG.error('Mokka Exited With Status %s' % (status))
message = 'Mokka Exited With Status %s' % (status)
self.setApplicationStatus(message)
if not self.ignoreapperrors:
LOG.error("Application exists with error:", message)
return S_ERROR(message)
else:
if status in [106, 9]:
message = 'Mokka %s reached end of input generator file' % (self.applicationVersion)
self.setApplicationStatus(message)
return S_OK( { 'OutputFile' : self.OutputFile } )
shutil.copy(
os.path.basename(self.detectorModel),
os.path.join(lcsimfolder, "detectors",
os.path.basename(self.detectorModel)))
paths = {}
paths[os.path.basename(self.SteeringFile)] = os.path.basename(
self.SteeringFile)
paths[os.path.basename(self.trackingstrategy)] = os.path.basename(
self.trackingstrategy)
for myfile in paths:
if len(myfile):
#file = os.path.basename(file)
if not os.path.exists(myfile):
res = getSteeringFileDirName(self.platform,
self.applicationName,
self.applicationVersion)
if not res['OK']:
self.log.error(
'Failed finding the steering file folder:',
res["Message"])
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname,
myfile)):
paths[myfile] = os.path.join(steeringfiledirname,
myfile)
if not os.path.exists(paths[myfile]):
self.log.error('Failed finding file', paths[myfile])
return S_ERROR("Could not find file %s" % paths[myfile])
self.SteeringFile = paths[os.path.basename(self.SteeringFile)]
def runIt(self):
"""
Called by JobAgent
Execute the following:
- get the environment variables that should have been set during installation
- download the detector model, using CS query to fetch the address
- prepare the mac file using :any:`prepareMacFile`
- run SLIC on this mac File and catch the exit status
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
LOG.error("Failed to resolve input parameters:", self.result['Message'])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('SLIC should not proceed as previous step did not end properly')
res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript)
if not res['OK']:
LOG.error("Could not obtain the environment script: ", res["Message"])
return res
env_script_path = res["Value"]
retMod = self.getDetectorModel()
if not retMod:
return retMod
slicmac = 'slicmac.mac'
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
LOG.error("Generator file not found")
return res
self.InputFile = res['Value']
if len(self.SteeringFile) > 0:
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion)
if not res['OK']:
LOG.error("Could not find where the steering files are")
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile)
if not os.path.exists(self.SteeringFile):
LOG.error("Missing steering file")
return S_ERROR("Could not find mac file")
##Same as for mokka: using ParticleGun does not imply InputFile
if not len(self.InputFile):
self.InputFile = ['']
macok = prepareMacFile(self.SteeringFile, slicmac, self.InputFile[0],
self.NumberOfEvents, self.startFrom, self.detectorModel,
self.RandomSeed, self.OutputFile, self.debug)
if not macok['OK']:
LOG.error('Failed to create SLIC mac file')
return S_ERROR('Error when creating SLIC mac file')
scriptName = 'SLIC_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/bash \n')
script.write('#####################################################################\n')
script.write('# Dynamically generated script to run a production or analysis job. #\n')
script.write('#####################################################################\n')
script.write("source %s\n" % (env_script_path))
preloadDir = "%s/preload" % os.getcwd()
if os.path.exists( preloadDir ):
preadloadLibs = set( os.listdir( preloadDir ) )
preloadString =":".join( lib for lib in preadloadLibs if ".so" in lib )
script.write( 'declare -x LD_LIBRARY_PATH=%s:${LD_LIBRARY_PATH}\n' % preloadDir )
script.write( 'declare -x LD_PRELOAD=%s\n' % preloadString )
script.write('echo =========\n')
script.write('env | sort >> localEnv.log\n')
script.write('echo SLIC:\n')
script.write("which slic\n")
script.write('echo =========\n')
comm = 'slic -P $PARTICLE_TBL -m %s %s\n' % (slicmac, self.extraCLIarguments)
LOG.info("Command:", comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0o755)
comm = 'sh -c "./%s"' % scriptName
self.setApplicationStatus('SLIC %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
LOG.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' % (self.applicationName))
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
LOG.info("Status after the application execution is %s" % str(status))
return self.finalStatusReport(status)
def runIt(self):
"""
Called by JobAgent
Execute the following:
- get the environment variables that should have been set during installation
- find the detector model xml, using CS query to obtain the path
- prepare the steering file and command line parameters
- run DDSim on this steering file and catch the exit status
:rtype: S_OK, S_ERROR
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:",
self.result['Message'])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'DDSim should not proceed as previous step did not end properly'
)
##TODO: Setup LD_LIBRARY_PATH for extensions
res = getEnvironmentScript(self.platform, self.applicationName,
self.applicationVersion, self.getEnvScript)
if not res['OK']:
self.log.error("Could not obtain the environment script: ",
res["Message"])
return res
envScriptPath = res["Value"]
#get the path to the detector model, either local or from the software
resXML = self._getDetectorXML()
if not resXML['OK']:
self.log.error("Could not obtain the detector XML file: ",
resXML["Message"])
return resXML
compactFile = resXML['Value']
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error("Generator file not found")
return res
self.InputFile = res['Value']
## if steering file is set try to find it
if len(self.SteeringFile) > 0:
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
res = getSteeringFileDirName(self.platform,
self.applicationName,
self.applicationVersion)
if not res['OK']:
self.log.error(
"Could not find where the steering files are")
return res
steeringfiledirname = res['Value']
if os.path.exists(
os.path.join(steeringfiledirname, self.SteeringFile)):
self.SteeringFile = os.path.join(steeringfiledirname,
self.SteeringFile)
if not os.path.exists(self.SteeringFile):
self.log.error("Missing steering file")
return S_ERROR("Could not find steering file")
self.extraCLIarguments += " --steeringFile %s " % self.SteeringFile
if self.startFrom:
self.extraCLIarguments += " --skipNEvents %s " % self.startFrom
if self.debug:
self.extraCLIarguments += " --printLevel DEBUG "
##Same as for mokka: using ParticleGun does not imply InputFile
if self.InputFile:
self.InputFile = [self.InputFile] if isinstance(
self.InputFile, basestring) else self.InputFile
self.extraCLIarguments += " --inputFile %s " % self.InputFile[0]
if self.NumberOfEvents:
self.extraCLIarguments += " --numberOfEvents %s " % self.NumberOfEvents
self.extraCLIarguments += " --random.seed %s " % self.randomSeed
if self.OutputFile:
self.extraCLIarguments += " --outputFile %s " % self.OutputFile
scriptName = 'DDSim_%s_Run_%s.sh' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = []
script.append('#!/bin/bash')
script.append(
'#####################################################################'
)
script.append(
'# Dynamically generated script to run a production or analysis job. #'
)
script.append(
'#####################################################################'
)
script.append('source %s' % envScriptPath)
script.append('echo =========')
## for user provided libraries
if os.path.exists("lib"):
script.append("export LD_LIBRARY_PATH=$PWD/lib:$LD_LIBRARY_PATH")
script.append('env | sort >> localEnv.log')
script.append('echo ddsim:`which ddsim`')
script.append('echo =========')
comm = 'ddsim --compactFile %(compactFile)s %(extraArgs)s' % dict(
compactFile=compactFile, extraArgs=self.extraCLIarguments)
self.log.info("Command:", comm)
script.append(comm)
script.append('declare -x appstatus=$?')
script.append('exit $appstatus')
with open(scriptName, 'w') as scriptFile:
scriptFile.write("\n".join(script))
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'bash "./%s"' % scriptName
self.setApplicationStatus('DDSim %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
comm,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
self.log.error(
"Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed to produce log file' %
(self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log %s' %
(self.applicationName, self.applicationLog))
status = resultTuple[0]
self.log.info("Status after the application execution is %s" % status)
return self.finalStatusReport(status)
def execute(self):
""" Called by Agent
Executes the following:
- read the application parameters that where defined in ILCJob, and stored in the job definition
- setup the SQL server and run it in the background, via a call to L{SQLWrapper}
- prepare the steering fie using L{PrepareSteeringFile}
- run Mokka and catch its return status
@return: S_OK(), S_ERROR()
"""
result = self.resolveInputVariables()
if not result['OK']:
return result
#if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
# self.log.info('Skip this module, failure detected in a previous step :')
# self.log.info('Workflow status : %s' %(self.workflowStatus))
# self.log.info('Step Status %s' %(self.stepStatus))
# return S_OK()
self.result = S_OK()
if not self.systemConfig:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('Mokka should not proceed as previous step did not end properly')
cwd = os.getcwd()
root = gConfig.getValue('/LocalSite/Root', cwd)
self.log.info( "Executing Mokka %s" % ( self.applicationVersion ))
self.log.info("Platform for job is %s" % ( self.systemConfig ) )
self.log.info("Root directory for job is %s" % ( root ) )
mokkaDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig, "mokka",
self.applicationVersion), '')
if not mokkaDir:
self.log.error('Could not get Tar ball name')
return S_ERROR('Failed finding software directory')
mokkaDir = mokkaDir.replace(".tgz", "").replace(".tar.gz", "")
#mokkaDir = 'lddLib' ###Temporary while mokka tar ball are not redone.
mySoftwareRoot = ''
localArea = LocalArea()
sharedArea = SharedArea()
if os.path.exists('%s%s%s' % (localArea, os.sep, mokkaDir)):
mySoftwareRoot = localArea
elif os.path.exists('%s%s%s' % (sharedArea, os.sep, mokkaDir)):
mySoftwareRoot = sharedArea
else:
self.log.error("Mokka: could not find installation directory!")
return S_ERROR("Mokka installation could not be found")
##This is a duplication of what is above, but the above cannot easily be removed...
res = getSoftwareFolder(mokkaDir)
if not res['OK']:
self.log.error("Mokka: could not find installation directory!")
return res
myMokkaDir = res['Value']
if not mySoftwareRoot:
self.log.error('Directory %s was not found in either the local area %s or shared area %s' % (mokkaDir, localArea,
sharedArea))
return S_ERROR('Failed to discover software')
####Setup MySQL instance
MokkaDBrandomName = '/tmp/MokkaDBRoot-' + GenRandString(8)
#sqlwrapper = SQLWrapper(self.dbslice,mySoftwareRoot,"/tmp/MokkaDBRoot")#mySoftwareRoot)
sqlwrapper = SQLWrapper(mySoftwareRoot, MokkaDBrandomName)#mySoftwareRoot)
res = sqlwrapper.setDBpath(myMokkaDir, self.dbSlice)
if not res['OK']:
self.log.error("Failed to find the DB slice")
return res
result = sqlwrapper.makedirs()
if not result['OK']:
self.setApplicationStatus('MySQL setup failed to create directories.')
return result
result = sqlwrapper.mysqlSetup()
if not result['OK']:
self.setApplicationStatus('MySQL setup failed.')
return result
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = GetNewLDLibs(self.systemConfig, "mokka", self.applicationVersion)
##Remove libc
removeLibc(myMokkaDir)
##Get the particle.tbl, if any
path_to_particle_tbl = ''
dir_to_particletbl = os.path.join(myMokkaDir, 'ConfigFiles')
if os.path.exists(dir_to_particletbl):
configdir = os.listdir(dir_to_particletbl)
if 'particle.tbl' in configdir:
path_to_particle_tbl = os.path.join(dir_to_particletbl, 'particle.tbl')
if os.path.exists('./particle.tbl'):
path_to_particle_tbl = "./particle.tbl"
###steering file that will be used to run
mokkasteer = "mokka.steer"
if os.path.exists("mokka.steer"):
try:
shutil.move("mokka.steer", "mymokka.steer")
except:
self.log.error("Failed renaming the steering file")
self.SteeringFile = "mymokka.steer"
###prepare steering file
#first, I need to take the stdhep file, find its path (possible LFN)
if len(self.InputFile) > 0:
#self.InputFile = os.path.basename(self.InputFile)
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error("Generator file not found")
result = sqlwrapper.mysqlCleanUp()
return res
self.InputFile = res['Value']
if len(self.macFile) > 0:
self.macFile = os.path.basename(self.macFile)
##idem for steering file
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
res = getSteeringFileDirName(self.systemConfig, "mokka", self.applicationVersion)
if not res['OK']:
result = sqlwrapper.mysqlCleanUp()
return res
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
try:
shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile )
except Exception, x:
result = sqlwrapper.mysqlCleanUp()
return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(x)))
def runIt(self):
"""
Called by Agent
Execute the following:
- resolve where the soft was installed
- prepare the list of file to feed Marlin with
- create the XML file on which Marlin has to run, done by :any:`prepareXMLFile`
- run Marlin and catch the exit code
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:", self.result["Message"])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName)
#get the path to the detector model, either local or from the software
compactFile = None
if self.detectorModel:
resXML = self._getDetectorXML()
if not resXML['OK']:
self.log.error("Could not obtain the detector XML file: ", resXML["Message"])
return resXML
compactFile = resXML['Value']
res = getEnvironmentScript(self.platform, "marlin", self.applicationVersion, self.getEnvScript)
if not res['OK']:
self.log.error("Failed to get the env script")
return res
env_script_path = res["Value"]
res = self._getInputFiles()
if not res['OK']:
self.log.error("Failed getting input files:", res['Message'])
return res
listofslcio = res['Value']
finalXML = "marlinxml_" + self.STEP_NUMBER + ".xml"
steeringfiledirname = ''
res = getSteeringFileDirName(self.platform, "marlin", self.applicationVersion)
if res['OK']:
steeringfiledirname = res['Value']
else:
self.log.warn('Could not find the steering file directory', res['Message'])
##Handle PandoraSettings.xml
pandorasettings = 'PandoraSettings.xml'
if not os.path.exists(pandorasettings):
if steeringfiledirname and os.path.exists(os.path.join(steeringfiledirname, pandorasettings)):
try:
shutil.copy(os.path.join(steeringfiledirname, pandorasettings),
os.path.join(os.getcwd(), pandorasettings))
except EnvironmentError, x:
self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
def runIt(self):
"""
Called by JobAgent
Execute the following:
- get the environment variables that should have been set during installation
- download the detector model, using CS query to fetch the address
- prepare the mac file using :any:`prepareMacFile`
- run SLIC on this mac File and catch the exit status
:return: S_OK(), S_ERROR()
"""
self.result = S_OK()
if not self.platform:
self.result = S_ERROR('No ILC platform selected')
elif not self.applicationLog:
self.result = S_ERROR('No Log file provided')
if not self.result['OK']:
self.log.error("Failed to resolve input parameters:",
self.result['Message'])
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose(
'Workflow status = %s, step status = %s' %
(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK(
'SLIC should not proceed as previous step did not end properly'
)
res = getEnvironmentScript(self.platform, self.applicationName,
self.applicationVersion, self.getEnvScript)
if not res['OK']:
self.log.error("Could not obtain the environment script: ",
res["Message"])
return res
env_script_path = res["Value"]
retMod = self.getDetectorModel()
if not retMod:
return retMod
slicmac = 'slicmac.mac'
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error("Generator file not found")
return res
self.InputFile = res['Value']
if len(self.SteeringFile) > 0:
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
res = getSteeringFileDirName(self.platform,
self.applicationName,
self.applicationVersion)
if not res['OK']:
self.log.error(
"Could not find where the steering files are")
return res
steeringfiledirname = res['Value']
if os.path.exists(
os.path.join(steeringfiledirname, self.SteeringFile)):
self.SteeringFile = os.path.join(steeringfiledirname,
self.SteeringFile)
if not os.path.exists(self.SteeringFile):
self.log.error("Missing steering file")
return S_ERROR("Could not find mac file")
##Same as for mokka: using ParticleGun does not imply InputFile
if not len(self.InputFile):
self.InputFile = ['']
macok = prepareMacFile(self.SteeringFile, slicmac, self.InputFile[0],
self.NumberOfEvents, self.startFrom,
self.detectorModel, self.RandomSeed,
self.OutputFile, self.debug)
if not macok['OK']:
self.log.error('Failed to create SLIC mac file')
return S_ERROR('Error when creating SLIC mac file')
scriptName = 'SLIC_%s_Run_%s.sh' % (self.applicationVersion,
self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/bash \n')
script.write(
'#####################################################################\n'
)
script.write(
'# Dynamically generated script to run a production or analysis job. #\n'
)
script.write(
'#####################################################################\n'
)
script.write("source %s\n" % (env_script_path))
preloadDir = "%s/preload" % os.getcwd()
if os.path.exists(preloadDir):
preadloadLibs = set(os.listdir(preloadDir))
preloadString = ":".join(lib for lib in preadloadLibs
if ".so" in lib)
script.write('declare -x LD_LIBRARY_PATH=%s:${LD_LIBRARY_PATH}\n' %
preloadDir)
script.write('declare -x LD_PRELOAD=%s\n' % preloadString)
script.write('echo =========\n')
script.write('env | sort >> localEnv.log\n')
script.write('echo SLIC:\n')
script.write("which slic\n")
script.write('echo =========\n')
comm = 'slic -P $PARTICLE_TBL -m %s %s\n' % (slicmac,
self.extraCLIarguments)
self.log.info("Command:", comm)
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'sh -c "./%s"' % scriptName
self.setApplicationStatus('SLIC %s step %s' %
(self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0,
comm,
callbackFunction=self.redirectLogOutput,
bufferLimit=20971520)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
self.log.error(
"Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' %
(self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' %
(self.applicationName))
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
self.log.info("Status after the application execution is %s" %
str(status))
return self.finalStatusReport(status)
def execute(self):
"""
Called by JobAgent
Execute the following:
- get the environment variables that should have been set during installation
- download the detector model, using CS query to fetch the address
- prepare the mac file using L{PrepareMacFile}
- run SLIC on this mac File and catch the exit status
@return: S_OK(), S_ERROR()
"""
self.result = self.resolveInputVariables()
if not self.systemConfig:
self.result = S_ERROR( 'No ILC platform selected' )
elif not self.applicationLog:
self.result = S_ERROR( 'No Log file provided' )
if not self.result['OK']:
return self.result
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK']))
return S_OK('SLIC should not proceed as previous step did not end properly')
if not os.environ.has_key('SLIC_DIR'):
self.log.error('SLIC_DIR not found, probably the software installation failed')
return S_ERROR('SLIC_DIR not found, probably the software installation failed')
if not os.environ.has_key('SLIC_VERSION'):
self.log.error('SLIC_VERSION not found, probably the software installation failed')
return S_ERROR('SLIC_VERSION not found, probably the software installation failed')
if not os.environ.has_key('LCDD_VERSION'):
self.log.error('LCDD_VERSION not found, probably the software installation failed')
return S_ERROR('LCDD_VERSION not found, probably the software installation failed')
#if not os.environ.has_key('XERCES_VERSION'):
# self.log.error('XERCES_VERSION not found, probably the software installation failed')
# return S_ERROR('XERCES_VERSION not found, probably the software installation failed')
slicDir = os.environ['SLIC_DIR']
res = getSoftwareFolder(slicDir)
if not res['OK']:
self.log.error('Directory %s was not found in either the local area or shared area' % (slicDir))
return res
mySoftwareRoot = res['Value']
##Need to fetch the new LD_LIBRARY_PATH
new_ld_lib_path = GetNewLDLibs(self.systemConfig, "slic", self.applicationVersion)
#retrieve detector model from web
detector_urls = self.ops.getValue('/SLICweb/SLICDetectorModels', [''])
if len(detector_urls[0]) < 1:
self.log.error('Could not find in CS the URL for detector model')
return S_ERROR('Could not find in CS the URL for detector model')
if not os.path.exists(self.detectorModel + ".zip"):
for detector_url in detector_urls:
try:
urllib.urlretrieve("%s%s" % (detector_url, self.detectorModel + ".zip"),
self.detectorModel + ".zip")
except:
self.log.error("Download of detector model failed")
continue
if not os.path.exists(self.detectorModel + ".zip"):
self.log.error('Detector model %s was not found neither locally nor on the web, exiting' % self.detectorModel)
return S_ERROR('Detector model %s was not found neither locally nor on the web, exiting' % self.detectorModel)
try:
unzip_file_into_dir(open(self.detectorModel + ".zip"), os.getcwd())
except:
os.unlink(self.detectorModel + ".zip")
self.log.error('Failed to unzip detector model')
return S_ERROR('Failed to unzip detector model')
#unzip detector model
#self.unzip_file_into_dir(open(self.detectorModel+".zip"),os.getcwd())
slicmac = 'slicmac.mac'
if len(self.InputFile):
res = resolveIFpaths(self.InputFile)
if not res['OK']:
self.log.error("Generator file not found")
return res
self.InputFile = res['Value']
if len(self.SteeringFile) > 0:
self.SteeringFile = os.path.basename(self.SteeringFile)
if not os.path.exists(self.SteeringFile):
res = getSteeringFileDirName(self.systemConfig, "slic", self.applicationVersion)
if not res['OK']:
self.log.error("Could not find where the steering files are")
steeringfiledirname = res['Value']
if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile)
if not os.path.exists(self.SteeringFile):
return S_ERROR("Could not find mac file")
##Same as for mokka: using ParticleGun does not imply InputFile
if not len(self.InputFile):
self.InputFile = ['']
macok = PrepareMacFile(self.SteeringFile, slicmac, self.InputFile[0],
self.NumberOfEvents, self.startFrom, self.detectorModel,
self.randomseed, self.OutputFile, self.debug)
if not macok['OK']:
self.log.error('Failed to create SLIC mac file')
return S_ERROR('Error when creating SLIC mac file')
scriptName = 'SLIC_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
if os.path.exists(scriptName):
os.remove(scriptName)
script = open(scriptName, 'w')
script.write('#!/bin/sh \n')
script.write('#####################################################################\n')
script.write('# Dynamically generated script to run a production or analysis job. #\n')
script.write('#####################################################################\n')
if os.environ.has_key('XERCES_VERSION'):
script.write('declare -x XERCES_LIB_DIR=%s/packages/xerces/%s/lib\n' % (mySoftwareRoot,
os.environ['XERCES_VERSION']))
if new_ld_lib_path:
script.write('declare -x LD_LIBRARY_PATH=$XERCES_LIB_DIR:%s\n' % new_ld_lib_path)
else:
script.write('declare -x LD_LIBRARY_PATH=$XERCES_LIB_DIR\n')
script.write('declare -x GEANT4_DATA_ROOT=%s/packages/geant4/data\n' % mySoftwareRoot)
script.write('declare -x G4LEVELGAMMADATA=$(ls -d $GEANT4_DATA_ROOT/PhotonEvaporation*)\n')
script.write('declare -x G4RADIOACTIVEDATA=$(ls -d $GEANT4_DATA_ROOT/RadioactiveDecay*)\n')
script.write('declare -x G4LEDATA=$(ls -d $GEANT4_DATA_ROOT/G4EMLOW*)\n')
script.write('declare -x G4NEUTRONHPDATA=$(ls -d $GEANT4_DATA_ROOT/G4NDL*)\n')
script.write('declare -x GDML_SCHEMA_DIR=%s/packages/lcdd/%s\n' % (mySoftwareRoot, os.environ['LCDD_VERSION']))
script.write('declare -x PARTICLE_TBL=%s/packages/slic/%s/data/particle.tbl\n' % (mySoftwareRoot,
os.environ['SLIC_VERSION']))
script.write('declare -x MALLOC_CHECK_=0\n')
if os.path.exists("%s/lib" % (mySoftwareRoot)):
script.write('declare -x LD_LIBRARY_PATH=%s/lib:$LD_LIBRARY_PATH\n' % (mySoftwareRoot))
script.write('echo =========\n')
script.write('env | sort >> localEnv.log\n')
script.write('echo =========\n')
comm = '%s/packages/slic/%s/bin/Linux-g++/slic -P $PARTICLE_TBL -m %s\n' % (mySoftwareRoot,
os.environ['SLIC_VERSION'],
slicmac)
print comm
script.write(comm)
script.write('declare -x appstatus=$?\n')
script.write('exit $appstatus\n')
script.close()
if os.path.exists(self.applicationLog):
os.remove(self.applicationLog)
os.chmod(scriptName, 0755)
comm = 'sh -c "./%s"' % scriptName
self.setApplicationStatus('SLIC %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
self.stdError = ''
self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520)
#self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
resultTuple = self.result['Value']
if not os.path.exists(self.applicationLog):
self.log.error("Something went terribly wrong, the log file is not present")
self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
if not self.ignoreapperrors:
return S_ERROR('%s did not produce the expected log' % (self.applicationName))
status = resultTuple[0]
# stdOutput = resultTuple[1]
# stdError = resultTuple[2]
self.log.info( "Status after the application execution is %s" % str( status ) )
return self.finalStatusReport(status)
