def runIt(self): """ Called from Workflow """ if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName) if not self.OutputFile: self.log.error("OutputFile not specified") return S_ERROR("OutputFile not specified") res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error('Application %s was not found in either the local area or shared area' % self.applicationName) self.setApplicationStatus('%s: Could not find neither local area not shared area install' % self.applicationName) return res mySoftDir = res['Value'] self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): self.log.verbose('Getting the steering files directory') res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error("Could not locate the steering file directory") return res steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)): try: shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile ) except EnvironmentError, x: self.log.error("Failed to get the cuts file") return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(x)))
def test_getsteerfiledirname_fails( self ): getval_mock = Mock() getval_mock.getValue.return_value = '' ops_mock = Mock() ops_mock.return_value = getval_mock with patch("%s.Operations" % MODULE_NAME, new=ops_mock): assertDiracFailsWith( getSteeringFileDirName( 'mytest_plat', 'myappTestme', 'vTest1.0' ), 'could not find attached steeringfile version', self )
def runIt(self): """ Called by Agent Execute the following: - resolve where the soft was installed - prepare the list of file to feed Marlin with - create the XML file on which Marlin has to run, done by :any:`prepareXMLFile` - run Marlin and catch the exit code :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName) res = getEnvironmentScript(self.platform, "marlin", self.applicationVersion, self.getEnvScript) if not res['OK']: self.log.error("Failed to get the env script") return res env_script_path = res["Value"] res = self.GetInputFiles() if not res['OK']: self.log.error("Failed getting input files:", res['Message']) return res listofslcio = res['Value'] finalXML = "marlinxml_" + self.STEP_NUMBER + ".xml" steeringfiledirname = '' res = getSteeringFileDirName(self.platform, "marlin", self.applicationVersion) if res['OK']: steeringfiledirname = res['Value'] else: self.log.warn('Could not find the steering file directory', res['Message']) ##Handle PandoraSettings.xml pandorasettings = 'PandoraSettings.xml' if not os.path.exists(pandorasettings): if steeringfiledirname and os.path.exists(os.path.join(steeringfiledirname, pandorasettings)): try: shutil.copy(os.path.join(steeringfiledirname, pandorasettings), os.path.join(os.getcwd(), pandorasettings)) except EnvironmentError, x: self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
def test_getsteerfiledirname_fails(self): getval_mock = Mock() getval_mock.getValue.return_value = '' ops_mock = Mock() ops_mock.return_value = getval_mock with patch("%s.Operations" % MODULE_NAME, new=ops_mock): assertDiracFailsWith( getSteeringFileDirName('mytest_plat', 'myappTestme', 'vTest1.0'), 'could not find attached steeringfile version', self)
def test_getsteerfiledirname( self ): getval_mock = Mock() getval_mock.getValue.return_value = 'my_test_getval' ops_mock = Mock() ops_mock.return_value = getval_mock cvmfs_mock = Mock(return_value=S_ERROR('some_err')) soft_mock = Mock(return_value=S_OK('my_test_dir_retval')) check_mock = Mock(return_value=S_OK('ignoreme')) with patch("%s.Operations" % MODULE_NAME, new=ops_mock), \ patch('%s.checkCVMFS' % MODULE_NAME, new=cvmfs_mock), \ patch('%s.getSoftwareFolder' % MODULE_NAME, new=soft_mock), \ patch('%s.check' % MODULE_NAME, new=check_mock): assertDiracSucceedsWith_equals( getSteeringFileDirName( 'mytest_plat', 'myappTestme', 'vTest1.0' ), 'my_test_dir_retval', self ) getval_mock.getValue.assert_called_once_with( '/AvailableTarBalls/mytest_plat/myappTestme/vTest1.0/Dependencies/steeringfiles/version', '' )
def runIt(self): """ Called from Workflow """ if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK( '%s should not proceed as previous step did not end properly' % self.applicationName) if not self.OutputFile: self.log.error("OutputFile not specified") return S_ERROR("OutputFile not specified") res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error( 'Application %s was not found in either the local area or shared area' % self.applicationName) self.setApplicationStatus( '%s: Could not find neither local area not shared area install' % self.applicationName) return res mySoftDir = res['Value'] self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): self.log.verbose('Getting the steering files directory') res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error("Could not locate the steering file directory") return res steeringfiledirname = res['Value'] if os.path.exists( os.path.join(steeringfiledirname, self.SteeringFile)): try: shutil.copy( os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile) except EnvironmentError, x: self.log.error("Failed to get the cuts file") return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(x)))
def test_getsteerfiledirname(self): getval_mock = Mock() getval_mock.getValue.return_value = 'my_test_getval' ops_mock = Mock() ops_mock.return_value = getval_mock cvmfs_mock = Mock(return_value=S_ERROR('some_err')) soft_mock = Mock(return_value=S_OK('my_test_dir_retval')) check_mock = Mock(return_value=S_OK('ignoreme')) with patch("%s.Operations" % MODULE_NAME, new=ops_mock), \ patch('%s.checkCVMFS' % MODULE_NAME, new=cvmfs_mock), \ patch('%s.getSoftwareFolder' % MODULE_NAME, new=soft_mock), \ patch('%s.check' % MODULE_NAME, new=check_mock): assertDiracSucceedsWith_equals( getSteeringFileDirName('mytest_plat', 'myappTestme', 'vTest1.0'), 'my_test_dir_retval', self) getval_mock.getValue.assert_called_once_with( '/AvailableTarBalls/mytest_plat/myappTestme/vTest1.0/Dependencies/steeringfiles/version', '')
def execute(self): """ Called from Workflow """ self.result = self.resolveInputVariables() if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('StdHepCut should not proceed as previous step did not end properly') appDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall'% (self.systemConfig, "stdhepcut", self.applicationVersion), '') if not appDir: self.log.error('Could not get info from CS') self.setApplicationStatus('Failed finding info from CS') return S_ERROR('Failed finding info from CS') appDir = appDir.replace(".tgz", "").replace(".tar.gz", "") res = getSoftwareFolder(appDir) if not res['OK']: self.setApplicationStatus('%s: Could not find neither local area not shared area install' % self.applicationName) return res mySoftDir = res['Value'] new_ld_lib_path = GetNewLDLibs(self.systemConfig, self.applicationName, self.applicationVersion) new_ld_lib_path = mySoftDir + "/lib:" + new_ld_lib_path if os.path.exists("./lib"): new_ld_lib_path = "./lib:" + new_ld_lib_path self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): res = getSteeringFileDirName(self.systemConfig, self.applicationName, self.applicationVersion) if not res['OK']: return res steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)): try: shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile ) except Exception, x: return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(x)))
def runIt(self): """ Called by JobAgent Execute the following: - get the environment variables that should have been set during installation - find the detector model xml, using CS query to obtain the path - prepare the steering file and command line parameters - run DDSim on this steering file and catch the exit status :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result['Message']) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('DDSim should not proceed as previous step did not end properly') ##TODO: Setup LD_LIBRARY_PATH for extensions res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript) if not res['OK']: self.log.error("Could not obtain the environment script: ", res["Message"]) return res envScriptPath = res["Value"] #get the path to the detector model, either local or from the software resXML = self._getDetectorXML() if not resXML['OK']: self.log.error("Could not obtain the detector XML file: ", resXML["Message"]) return resXML compactFile = resXML['Value'] if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Generator file not found") return res self.InputFile = res['Value'] ## if steering file is set try to find it if len(self.SteeringFile) > 0: self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error("Could not find where the steering files are") return res steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)): self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile) if not os.path.exists(self.SteeringFile): self.log.error("Missing steering file") return S_ERROR("Could not find steering file") self.extraCLIarguments += " --steeringFile %s " % self.SteeringFile if self.startFrom: self.extraCLIarguments += " --skipNEvents %s " % self.startFrom if self.debug: self.extraCLIarguments += " --printLevel DEBUG " ##Same as for mokka: using ParticleGun does not imply InputFile if self.InputFile: self.InputFile = [self.InputFile] if isinstance(self.InputFile, basestring) else self.InputFile self.extraCLIarguments += " --inputFile %s " % self.InputFile[0] if self.NumberOfEvents: self.extraCLIarguments += " --numberOfEvents %s " % self.NumberOfEvents self.extraCLIarguments += " --random.seed %s " % self.randomSeed if self.OutputFile: self.extraCLIarguments += " --outputFile %s " % self.OutputFile scriptName = 'DDSim_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = [] script.append('#!/bin/bash') script.append('#####################################################################') script.append('# Dynamically generated script to run a production or analysis job. #') script.append('#####################################################################') script.append('source %s' % envScriptPath) script.append('echo =========') script.append('env | sort >> localEnv.log') script.append('echo ddsim:`which ddsim`') script.append('echo =========') comm = 'ddsim --compactFile %(compactFile)s %(extraArgs)s' % dict(compactFile=compactFile, extraArgs=self.extraCLIarguments) self.log.info("Command:", comm) script.append(comm) script.append('declare -x appstatus=$?') script.append('exit $appstatus') with open(scriptName, 'w') as scriptFile: scriptFile.write( "\n".join(script) ) if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'bash "./%s"' % scriptName self.setApplicationStatus('DDSim %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520) resultTuple = self.result['Value'] if not os.path.exists(self.applicationLog): self.log.error("Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed to produce log file' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log %s' % (self.applicationName, self.applicationLog)) status = resultTuple[0] self.log.info( "Status after the application execution is %s" % status ) return self.finalStatusReport(status)
def runIt(self): """ Called by ModuleBase Executes the following: - read the application parameters that where defined in ILCJob, and stored in the job definition - setup the SQL server and run it in the background - prepare the steering file using :any:`prepareSteeringFile` - run Mokka and catch its return status :return: S_OK(), S_ERROR() """ #if not self.workflowStatus['OK'] or not self.stepStatus['OK']: # self.log.info('Skip this module, failure detected in a previous step :') # self.log.info('Workflow status : %s' %(self.workflowStatus)) # self.log.info('Step Status %s' %(self.stepStatus)) # return S_OK() self.result = S_OK() if not self.platform: self.result = S_ERROR('No ILC platform selected') elif not self.applicationLog: self.result = S_ERROR('No Log file provided') if not self.result['OK']: self.log.error("Failed to resolve the input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK( 'Mokka should not proceed as previous step did not end properly' ) self.log.info("Executing Mokka %s" % (self.applicationVersion)) if self.dbSlice: if self.dbSlice.lower().startswith("lfn"): self.dbSlice = os.path.basename(self.dbSlice) self.db_dump_name = self.dbSlice else: self.db_dump_name = "CLICMokkaDB.sql" res = getEnvironmentScript(self.platform, "mokka", self.applicationVersion, self.getEnvScript) self.log.notice("Got the environment script: %s" % res) if not res['OK']: self.log.error("Error getting the env script: ", res['Message']) return res env_script_path = res['Value'] ####Setup MySQL instance mysqlBasePath = '%s/mysqltmp/MokkaDBRoot-%s' % ( os.getcwd(), generateRandomString(8)) self.log.notice("Placing mysql files in %s" % mysqlBasePath) ###steering file that will be used to run mokkasteer = "mokka.steer" if os.path.exists("mokka.steer"): try: os.rename("mokka.steer", "mymokka.steer") except EnvironmentError as err: self.log.error("Failed renaming the steering file: ", str(err)) self.SteeringFile = "mymokka.steer" ###prepare steering file #first, I need to take the stdhep file, find its path (possible LFN) if len(self.InputFile) > 0: #self.InputFile = os.path.basename(self.InputFile) res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Generator file not found") return res self.InputFile = res['Value'] if len(self.macFile) > 0: self.macFile = os.path.basename(self.macFile) ##idem for steering file self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): self.log.verbose("Steering file %s not found locally" % self.SteeringFile) res = getSteeringFileDirName(self.platform, "mokka", self.applicationVersion) if not res['OK']: self.log.error("Missing Steering file directory:", res['Message']) return res steeringfiledirname = res['Value'] if os.path.exists( os.path.join(steeringfiledirname, self.SteeringFile)): try: shutil.copy( os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile) except EnvironmentError as err: self.log.error("Failed copying file", self.SteeringFile) return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(err))) #self.steeringFile = os.path.join(mySoftwareRoot,"steeringfiles",self.steeringFile) if not os.path.exists(self.SteeringFile): self.log.error("Missing steering file, should not happen!") return S_ERROR("Could not find steering file") else: self.log.verbose("Found local copy of %s" % self.SteeringFile) ### The following is because if someone uses particle gun, there is no InputFile if not len(self.InputFile): self.InputFile = [''] steerok = prepareSteeringFile( self.SteeringFile, mokkasteer, self.detectorModel, self.InputFile[0], self.macFile, self.NumberOfEvents, self.startFrom, self.RandomSeed, self.mcRunNumber, self.ProcessID, self.debug, self.OutputFile, self.inputdataMeta) if not steerok['OK']: self.log.error('Failed to create MOKKA steering file') return S_ERROR('Failed to create MOKKA steering file') ###Extra option depending on mokka version mokkaextraoption = "" if self.applicationVersion not in [ "v07-02", "v07-02fw", "v07-02fwhp", "MokkaRevision42", "MokkaRevision43", "MokkaRevision44", "Revision45" ]: mokkaextraoption = "-U" scriptName = 'Mokka_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/bash \n') script.write( '#####################################################################\n' ) script.write( '# Dynamically generated script to run a production or analysis job. #\n' ) script.write( '#####################################################################\n' ) script.write("source %s\n" % env_script_path) if os.path.exists("./lib"): script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n') script.write('echo =============================\n') script.write('echo Content of mokka.steer:\n') script.write('cat mokka.steer\n') if self.macFile: script.write('echo =============================\n') script.write('echo Content of %s\n' % os.path.basename(self.macFile)) script.write("cat %s\n" % os.path.basename(self.macFile)) elif os.path.exists("./mokkamac.mac"): script.write(""" echo ============================= echo Content of mokkamac.mac: cat mokkamac.mac """) script.write('echo =============================\n') script.write('echo LD_LIBRARY_PATH is\n') script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n') script.write('echo =============================\n') script.write('echo PATH is\n') script.write('echo $PATH | tr ":" "\n"\n') if self.debug: script.write('echo ldd of Mokka is\n') script.write('ldd `which Mokka` \n') script.write('echo =============================\n') script.write('echo ldd of mysql is\n') script.write('ldd `which mysql` \n') script.write('echo =============================\n') script.write('env | sort >> localEnv.log\n') script.write('echo =============================\n') script.write("""if [ ! -e ./%(DBDUMP)s ]; then echo "Missing DB dump!" >&2 exit 1 fi """ % {'DBDUMP': self.db_dump_name}) ##Now start the MySQL server and configure. script.write("declare -x MOKKADBROOT=%s\n" % mysqlBasePath) script.write("declare -x WORKDIR=%s\n" % os.getcwd()) script.write("declare -x LOGDIR=$WORKDIR/mysqllogs/\n") script.write("mkdir -p $LOGDIR\n") #mysql socket must not be longer than 107 characters! script.write("declare -x SOCKETBASE=/tmp/mokka-%s\n" % generateRandomString(8)) script.write("mkdir -p $SOCKETBASE\n") script.write("declare -x SOCKETPATH=$SOCKETBASE/mysql.sock\n") script.write("mkdir -p $MOKKADBROOT\n") script.write("declare -x MYSQLDATA=$MOKKADBROOT/data\n") script.write("rm -rf $MYSQLDATA\n") script.write("mkdir -p $MYSQLDATA\n") script.write("date\n") script.write("""echo "*** Installing MySQL" echo "mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log" mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log install_st=$? if [ $install_st -ne 0 ] then exit $install_st fi date echo "*** Running mysqld-safe" cd $MYSQL bin/mysqld_safe --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log & COUNT=0 while [ -z "$socket_grep" ] ; do socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH") echo -n . sleep 1 if [ $COUNT -eq 100 ]; then echo "Failed to find socket" break fi COUNT=$(( $COUNT + 1 )) done cd - echo "*** Configuring MySQL" mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot password 'rootpass' declare -x admin_st=$? if [ $admin_st -ne 0 ] then echo "*** mysqladmin failed, cannot proceed" >&2 mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown exit $admin_st fi mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO root;' mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO consult IDENTIFIED BY \"consult\";' mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'DELETE FROM mysql.user WHERE User = \"\"; FLUSH PRIVILEGES;' echo "*** Installing Mokka DB" date echo "mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s" time mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s sleep 5 date\n""" % {'DBDUMP': self.db_dump_name}) #Now take care of the particle tables. script.write(""" if [ -e "./particle.tbl" ] then declare -x PARTICLETBL=./particle.tbl fi if [ -d "${MOKKA}" ] #As the local env does not set MOKKA, it must come from CVMFS then declare -x PARTICLETBL=$MOKKA/particle.tbl fi if [ -d "${MOKKATARBALL}" ] #As the local env does set MOKKATARBALL, it must be there then declare -x PARTICLETBL=$MOKKATARBALL/ConfigFiles/particle.tbl fi echo "/Mokka/init/PDGFile $PARTICLETBL" >> %s \n""" % mokkasteer) ##Now run Mokka comm = 'Mokka %s -h localhost:$SOCKETPATH %s %s\n' % ( mokkaextraoption, mokkasteer, self.extraCLIarguments) self.log.info("Command : %s" % (comm)) script.write(comm) script.write('declare -x appstatus=$?\n') #Now shutdown the MySQL server script.write( "mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown\n" ) script.write(""" while [ -n "$socket_grep" ] ; do socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH") echo -n . sleep 1 done """) script.write("rm -f %s\n" % self.db_dump_name) #remove db file script.write('rm -rf $MYSQLDATA\n') #cleanup script.write('rm -rf $MOKKADBROOT\n') #cleanup #script.write('quit\n') #script.write('EOF\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'sh -c "./%s"' % scriptName self.setApplicationStatus('Mokka %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction=self.redirectLogOutput, bufferLimit=20971520) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} resultTuple = self.result['Value'] status = resultTuple[0] # stdOutput = resultTuple[1] # stdError = resultTuple[2] self.log.info("Status after Mokka execution is %s" % str(status)) if not os.path.exists(self.applicationLog): self.log.error( "Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: self.log.error("Missing log file") return S_ERROR('%s did not produce the expected log' % (self.applicationName)) ###Now change the name of Mokka output to the specified filename if os.path.exists("out.slcio"): if len(self.OutputFile) > 0: os.rename("out.slcio", self.OutputFile) failed = False if status not in [0, 106, 9]: self.log.error("Mokka execution completed with errors:") failed = True elif status in [106, 9]: self.log.info( "Mokka execution reached end of input generator file") else: self.log.info("Mokka execution finished successfully") message = 'Mokka %s Successful' % (self.applicationVersion) if failed is True: self.log.error("==================================\n StdError:\n") self.log.error(self.stdError) #self.setApplicationStatus('%s Exited With Status %s' %(self.applicationName,status)) self.log.error('Mokka Exited With Status %s' % (status)) message = 'Mokka Exited With Status %s' % (status) self.setApplicationStatus(message) if not self.ignoreapperrors: self.log.error("Application exists with error:", message) return S_ERROR(message) else: if status in [106, 9]: message = 'Mokka %s reached end of input generator file' % ( self.applicationVersion) self.setApplicationStatus(message) return S_OK({'OutputFile': self.OutputFile})
os.mkdir(os.path.join(lcsimfolder, "detectors")) except OSError, x: self.log.error("Could not create detectors folder !", str(x)) if os.path.exists(os.path.join(lcsimfolder, "detectors")): self.log.verbose("Copy detector model.zip into the .lcsim/detectors folder") shutil.copy(os.path.basename(self.detectorModel), os.path.join(lcsimfolder, "detectors", os.path.basename(self.detectorModel))) paths = {} paths[os.path.basename(self.SteeringFile)] = os.path.basename(self.SteeringFile) paths[os.path.basename(self.trackingstrategy)] = os.path.basename(self.trackingstrategy) for myfile in paths.keys(): if len(myfile): #file = os.path.basename(file) if not os.path.exists(myfile): res = getSteeringFileDirName(self.systemConfig, "lcsim", self.applicationVersion) if not res['OK']: return res steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, myfile)): paths[myfile] = os.path.join(steeringfiledirname, myfile) if not os.path.exists(paths[myfile]): return S_ERROR("Could not find file %s" % paths[myfile]) self.SteeringFile = paths[os.path.basename(self.SteeringFile)] self.trackingstrategy = paths[os.path.basename(self.trackingstrategy)] lcsimfile = "job_%s.lcsim" % self.STEP_NUMBER res = PrepareLCSIMFile(self.SteeringFile, lcsimfile, self.NumberOfEvents, self.trackingstrategy, runonslcio, jars, cachedir, self.OutputFile, self.outputREC, self.outputDST, self.debug) if not res['OK']:
def execute(self): """ Called by Agent Execute the following: - resolve where the soft was installed - prepare the list of file to feed Marlin with - create the XML file on which Marlin has to run, done by L{PrepareXMLFile} - run Marlin and catch the exit code @return: S_OK(), S_ERROR() """ self.result = self.resolveInputVariables() if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName) marlinDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig, "marlin", self.applicationVersion), '') marlinDir = marlinDir.replace(".tgz", "").replace(".tar.gz", "") res = getSoftwareFolder(marlinDir) if not res['OK']: self.setApplicationStatus('Marlin: Could not find neither local area not shared area install') return res myMarlinDir = res['Value'] ##Remove libc removeLibc(myMarlinDir + "/LDLibs") ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = GetNewLDLibs(self.systemConfig, "marlin", self.applicationVersion) res = self.GetInputFiles() if not res['OK']: self.log.error(res['Message']) return res listofslcio = res['Value'] finalXML = "marlinxml_" + self.STEP_NUMBER + ".xml" steeringfiledirname = '' res = getSteeringFileDirName(self.systemConfig, "marlin", self.applicationVersion) if res['OK']: steeringfiledirname = res['Value'] else: self.log.warn('Could not find the steering file directory') ##Handle PandoraSettings.xml pandorasettings = 'PandoraSettings.xml' if not os.path.exists(pandorasettings): if os.path.exists(os.path.join(myMarlinDir, 'Settings', pandorasettings)): try: shutil.copy(os.path.join(myMarlinDir, 'Settings', pandorasettings), os.path.join(os.getcwd(), pandorasettings)) except Exception, x: self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x) elif steeringfiledirname and os.path.exists(os.path.join(steeringfiledirname,pandorasettings)): try: shutil.copy(os.path.join(steeringfiledirname, pandorasettings), os.path.join(os.getcwd(), pandorasettings)) except Exception, x: self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
os.mkdir(os.path.join(lcsimfolder, "detectors")) except OSError, x: LOG.error("Could not create detectors folder !", str(x)) if os.path.exists(os.path.join(lcsimfolder, "detectors")): LOG.verbose("Copy detector model.zip into the .lcsim/detectors folder") shutil.copy(os.path.basename(self.detectorModel), os.path.join(lcsimfolder, "detectors", os.path.basename(self.detectorModel))) paths = {} paths[os.path.basename(self.SteeringFile)] = os.path.basename(self.SteeringFile) paths[os.path.basename(self.trackingstrategy)] = os.path.basename(self.trackingstrategy) for myfile in paths: if len(myfile): #file = os.path.basename(file) if not os.path.exists(myfile): res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: LOG.error('Failed finding the steering file folder:', res["Message"]) return res steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, myfile)): paths[myfile] = os.path.join(steeringfiledirname, myfile) if not os.path.exists(paths[myfile]): LOG.error('Failed finding file', paths[myfile]) return S_ERROR("Could not find file %s" % paths[myfile]) self.SteeringFile = paths[os.path.basename(self.SteeringFile)] self.trackingstrategy = paths[os.path.basename(self.trackingstrategy)] lcsimfile = "job_%s.lcsim" % self.STEP_NUMBER res = prepareLCSIMFile(self.SteeringFile, lcsimfile, self.NumberOfEvents, self.trackingstrategy, runonslcio, jars, cachedir,
def runIt(self): """ Called by ModuleBase Executes the following: - read the application parameters that where defined in ILCJob, and stored in the job definition - setup the SQL server and run it in the background - prepare the steering file using :any:`prepareSteeringFile` - run Mokka and catch its return status :return: S_OK(), S_ERROR() """ #if not self.workflowStatus['OK'] or not self.stepStatus['OK']: # LOG.info('Skip this module, failure detected in a previous step :') # LOG.info('Workflow status : %s' %(self.workflowStatus)) # LOG.info('Step Status %s' %(self.stepStatus)) # return S_OK() self.result = S_OK() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: LOG.error("Failed to resolve the input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('Mokka should not proceed as previous step did not end properly') LOG.info("Executing Mokka %s" % (self.applicationVersion)) if self.dbSlice: if self.dbSlice.lower().startswith("lfn"): self.dbSlice = os.path.basename(self.dbSlice) self.db_dump_name = self.dbSlice else: self.db_dump_name = "CLICMokkaDB.sql" res = getEnvironmentScript(self.platform, "mokka", self.applicationVersion, self.getEnvScript) LOG.notice("Got the environment script: %s" % res) if not res['OK']: LOG.error("Error getting the env script: ", res['Message']) return res env_script_path = res['Value'] ####Setup MySQL instance mysqlBasePath = '%s/mysqltmp/MokkaDBRoot-%s' %(os.getcwd(), generateRandomString(8)) LOG.notice("Placing mysql files in %s" % mysqlBasePath) ###steering file that will be used to run mokkasteer = "mokka.steer" if os.path.exists("mokka.steer"): try: os.rename("mokka.steer", "mymokka.steer") except EnvironmentError as err: LOG.error("Failed renaming the steering file: ", str(err)) self.SteeringFile = "mymokka.steer" ###prepare steering file #first, I need to take the stdhep file, find its path (possible LFN) if len(self.InputFile) > 0: #self.InputFile = os.path.basename(self.InputFile) res = resolveIFpaths(self.InputFile) if not res['OK']: LOG.error("Generator file not found") return res self.InputFile = res['Value'] if len(self.macFile) > 0: self.macFile = os.path.basename(self.macFile) ##idem for steering file self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): LOG.verbose("Steering file %s not found locally" % self.SteeringFile) res = getSteeringFileDirName(self.platform, "mokka", self.applicationVersion) if not res['OK']: LOG.error("Missing Steering file directory:", res['Message']) return res steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)): try: shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile ) except EnvironmentError as err: LOG.error("Failed copying file", self.SteeringFile) return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(err))) #self.steeringFile = os.path.join(mySoftwareRoot,"steeringfiles",self.steeringFile) if not os.path.exists(self.SteeringFile): LOG.error("Missing steering file, should not happen!") return S_ERROR("Could not find steering file") else: LOG.verbose("Found local copy of %s" % self.SteeringFile) ### The following is because if someone uses particle gun, there is no InputFile if not len(self.InputFile): self.InputFile = [''] steerok = prepareSteeringFile(self.SteeringFile, mokkasteer, self.detectorModel, self.InputFile[0], self.macFile, self.NumberOfEvents, self.startFrom, self.RandomSeed, self.mcRunNumber, self.ProcessID, self.debug, self.OutputFile, self.inputdataMeta) if not steerok['OK']: LOG.error('Failed to create MOKKA steering file') return S_ERROR('Failed to create MOKKA steering file') ###Extra option depending on mokka version mokkaextraoption = "" if self.applicationVersion not in ["v07-02", "v07-02fw", "v07-02fwhp", "MokkaRevision42", "MokkaRevision43", "MokkaRevision44", "Revision45"]: mokkaextraoption = "-U" scriptName = 'Mokka_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/bash \n') script.write('#####################################################################\n') script.write('# Dynamically generated script to run a production or analysis job. #\n') script.write('#####################################################################\n') script.write("source %s\n" % env_script_path) if os.path.exists("./lib"): script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n' ) script.write('echo =============================\n') script.write('echo Content of mokka.steer:\n') script.write('cat mokka.steer\n') if self.macFile: script.write('echo =============================\n') script.write('echo Content of %s\n' % os.path.basename(self.macFile)) script.write("cat %s\n" % os.path.basename(self.macFile)) elif os.path.exists("./mokkamac.mac"): script.write(""" echo ============================= echo Content of mokkamac.mac: cat mokkamac.mac """) script.write('echo =============================\n') script.write('echo LD_LIBRARY_PATH is\n') script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n') script.write('echo =============================\n') script.write('echo PATH is\n') script.write('echo $PATH | tr ":" "\n"\n') if self.debug: script.write('echo ldd of Mokka is\n') script.write('ldd `which Mokka` \n' ) script.write('echo =============================\n') script.write('echo ldd of mysql is\n') script.write('ldd `which mysql` \n' ) script.write('echo =============================\n') script.write('env | sort >> localEnv.log\n') script.write('echo =============================\n') script.write("""if [ ! -e ./%(DBDUMP)s ]; then echo "Missing DB dump!" >&2 exit 1 fi """ % {'DBDUMP': self.db_dump_name} ) ##Now start the MySQL server and configure. script.write("declare -x MOKKADBROOT=%s\n" % mysqlBasePath) script.write("declare -x WORKDIR=%s\n" % os.getcwd()) script.write("declare -x LOGDIR=$WORKDIR/mysqllogs/\n") script.write("mkdir -p $LOGDIR\n") #mysql socket must not be longer than 107 characters! script.write("declare -x SOCKETBASE=/tmp/mokka-%s\n" % generateRandomString(8) ) script.write("mkdir -p $SOCKETBASE\n") script.write("declare -x SOCKETPATH=$SOCKETBASE/mysql.sock\n" ) script.write("mkdir -p $MOKKADBROOT\n") script.write("declare -x MYSQLDATA=$MOKKADBROOT/data\n") script.write("rm -rf $MYSQLDATA\n") script.write("mkdir -p $MYSQLDATA\n") script.write("date\n") script.write("""echo "*** Installing MySQL" echo "mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log" mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log install_st=$? if [ $install_st -ne 0 ] then exit $install_st fi date echo "*** Running mysqld-safe" cd $MYSQL bin/mysqld_safe --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log & COUNT=0 while [ -z "$socket_grep" ] ; do socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH") echo -n . sleep 1 if [ $COUNT -eq 100 ]; then echo "Failed to find socket" break fi COUNT=$(( $COUNT + 1 )) done cd - echo "*** Configuring MySQL" mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot password 'rootpass' declare -x admin_st=$? if [ $admin_st -ne 0 ] then echo "*** mysqladmin failed, cannot proceed" >&2 mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown exit $admin_st fi mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO root;' mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO consult IDENTIFIED BY \"consult\";' mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'DELETE FROM mysql.user WHERE User = \"\"; FLUSH PRIVILEGES;' echo "*** Installing Mokka DB" date echo "mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s" time mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s sleep 5 date\n""" % {'DBDUMP': self.db_dump_name}) #Now take care of the particle tables. script.write(""" if [ -e "./particle.tbl" ] then declare -x PARTICLETBL=./particle.tbl fi if [ -d "${MOKKA}" ] #As the local env does not set MOKKA, it must come from CVMFS then declare -x PARTICLETBL=$MOKKA/particle.tbl fi if [ -d "${MOKKATARBALL}" ] #As the local env does set MOKKATARBALL, it must be there then declare -x PARTICLETBL=$MOKKATARBALL/ConfigFiles/particle.tbl fi echo "/Mokka/init/PDGFile $PARTICLETBL" >> %s \n"""% mokkasteer) ##Now run Mokka comm = 'Mokka %s -h localhost:$SOCKETPATH %s %s\n' % (mokkaextraoption, mokkasteer, self.extraCLIarguments) LOG.info("Command : %s" % (comm)) script.write(comm) script.write('declare -x appstatus=$?\n') #Now shutdown the MySQL server script.write("mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown\n") script.write(""" while [ -n "$socket_grep" ] ; do socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH") echo -n . sleep 1 done """) script.write("rm -f %s\n" % self.db_dump_name)#remove db file script.write('rm -rf $MYSQLDATA\n')#cleanup script.write('rm -rf $MOKKADBROOT\n')#cleanup #script.write('quit\n') #script.write('EOF\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0o755) comm = 'sh -c "./%s"' % scriptName self.setApplicationStatus('Mokka %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} resultTuple = self.result['Value'] status = resultTuple[0] # stdOutput = resultTuple[1] # stdError = resultTuple[2] LOG.info("Status after Mokka execution is %s" % str(status)) if not os.path.exists(self.applicationLog): LOG.error("Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: LOG.error("Missing log file") return S_ERROR('%s did not produce the expected log' % (self.applicationName)) ###Now change the name of Mokka output to the specified filename if os.path.exists("out.slcio"): if len(self.OutputFile) > 0: os.rename("out.slcio", self.OutputFile) failed = False if status not in [0, 106, 9]: LOG.error("Mokka execution completed with errors:") failed = True elif status in [106, 9]: LOG.info("Mokka execution reached end of input generator file") else: LOG.info("Mokka execution finished successfully") message = 'Mokka %s Successful' % (self.applicationVersion) if failed is True: LOG.error("==================================\n StdError:\n") LOG.error(self.stdError) #self.setApplicationStatus('%s Exited With Status %s' %(self.applicationName,status)) LOG.error('Mokka Exited With Status %s' % (status)) message = 'Mokka Exited With Status %s' % (status) self.setApplicationStatus(message) if not self.ignoreapperrors: LOG.error("Application exists with error:", message) return S_ERROR(message) else: if status in [106, 9]: message = 'Mokka %s reached end of input generator file' % (self.applicationVersion) self.setApplicationStatus(message) return S_OK( { 'OutputFile' : self.OutputFile } )
shutil.copy( os.path.basename(self.detectorModel), os.path.join(lcsimfolder, "detectors", os.path.basename(self.detectorModel))) paths = {} paths[os.path.basename(self.SteeringFile)] = os.path.basename( self.SteeringFile) paths[os.path.basename(self.trackingstrategy)] = os.path.basename( self.trackingstrategy) for myfile in paths: if len(myfile): #file = os.path.basename(file) if not os.path.exists(myfile): res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error( 'Failed finding the steering file folder:', res["Message"]) return res steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, myfile)): paths[myfile] = os.path.join(steeringfiledirname, myfile) if not os.path.exists(paths[myfile]): self.log.error('Failed finding file', paths[myfile]) return S_ERROR("Could not find file %s" % paths[myfile]) self.SteeringFile = paths[os.path.basename(self.SteeringFile)]
def runIt(self): """ Called by JobAgent Execute the following: - get the environment variables that should have been set during installation - download the detector model, using CS query to fetch the address - prepare the mac file using :any:`prepareMacFile` - run SLIC on this mac File and catch the exit status :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: LOG.error("Failed to resolve input parameters:", self.result['Message']) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('SLIC should not proceed as previous step did not end properly') res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript) if not res['OK']: LOG.error("Could not obtain the environment script: ", res["Message"]) return res env_script_path = res["Value"] retMod = self.getDetectorModel() if not retMod: return retMod slicmac = 'slicmac.mac' if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: LOG.error("Generator file not found") return res self.InputFile = res['Value'] if len(self.SteeringFile) > 0: self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: LOG.error("Could not find where the steering files are") return res steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)): self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile) if not os.path.exists(self.SteeringFile): LOG.error("Missing steering file") return S_ERROR("Could not find mac file") ##Same as for mokka: using ParticleGun does not imply InputFile if not len(self.InputFile): self.InputFile = [''] macok = prepareMacFile(self.SteeringFile, slicmac, self.InputFile[0], self.NumberOfEvents, self.startFrom, self.detectorModel, self.RandomSeed, self.OutputFile, self.debug) if not macok['OK']: LOG.error('Failed to create SLIC mac file') return S_ERROR('Error when creating SLIC mac file') scriptName = 'SLIC_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/bash \n') script.write('#####################################################################\n') script.write('# Dynamically generated script to run a production or analysis job. #\n') script.write('#####################################################################\n') script.write("source %s\n" % (env_script_path)) preloadDir = "%s/preload" % os.getcwd() if os.path.exists( preloadDir ): preadloadLibs = set( os.listdir( preloadDir ) ) preloadString =":".join( lib for lib in preadloadLibs if ".so" in lib ) script.write( 'declare -x LD_LIBRARY_PATH=%s:${LD_LIBRARY_PATH}\n' % preloadDir ) script.write( 'declare -x LD_PRELOAD=%s\n' % preloadString ) script.write('echo =========\n') script.write('env | sort >> localEnv.log\n') script.write('echo SLIC:\n') script.write("which slic\n") script.write('echo =========\n') comm = 'slic -P $PARTICLE_TBL -m %s %s\n' % (slicmac, self.extraCLIarguments) LOG.info("Command:", comm) script.write(comm) script.write('declare -x appstatus=$?\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0o755) comm = 'sh -c "./%s"' % scriptName self.setApplicationStatus('SLIC %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} resultTuple = self.result['Value'] if not os.path.exists(self.applicationLog): LOG.error("Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log' % (self.applicationName)) status = resultTuple[0] # stdOutput = resultTuple[1] # stdError = resultTuple[2] LOG.info("Status after the application execution is %s" % str(status)) return self.finalStatusReport(status)
def runIt(self): """ Called by JobAgent Execute the following: - get the environment variables that should have been set during installation - find the detector model xml, using CS query to obtain the path - prepare the steering file and command line parameters - run DDSim on this steering file and catch the exit status :rtype: S_OK, S_ERROR """ self.result = S_OK() if not self.platform: self.result = S_ERROR('No ILC platform selected') elif not self.applicationLog: self.result = S_ERROR('No Log file provided') if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result['Message']) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK( 'DDSim should not proceed as previous step did not end properly' ) ##TODO: Setup LD_LIBRARY_PATH for extensions res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript) if not res['OK']: self.log.error("Could not obtain the environment script: ", res["Message"]) return res envScriptPath = res["Value"] #get the path to the detector model, either local or from the software resXML = self._getDetectorXML() if not resXML['OK']: self.log.error("Could not obtain the detector XML file: ", resXML["Message"]) return resXML compactFile = resXML['Value'] if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Generator file not found") return res self.InputFile = res['Value'] ## if steering file is set try to find it if len(self.SteeringFile) > 0: self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error( "Could not find where the steering files are") return res steeringfiledirname = res['Value'] if os.path.exists( os.path.join(steeringfiledirname, self.SteeringFile)): self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile) if not os.path.exists(self.SteeringFile): self.log.error("Missing steering file") return S_ERROR("Could not find steering file") self.extraCLIarguments += " --steeringFile %s " % self.SteeringFile if self.startFrom: self.extraCLIarguments += " --skipNEvents %s " % self.startFrom if self.debug: self.extraCLIarguments += " --printLevel DEBUG " ##Same as for mokka: using ParticleGun does not imply InputFile if self.InputFile: self.InputFile = [self.InputFile] if isinstance( self.InputFile, basestring) else self.InputFile self.extraCLIarguments += " --inputFile %s " % self.InputFile[0] if self.NumberOfEvents: self.extraCLIarguments += " --numberOfEvents %s " % self.NumberOfEvents self.extraCLIarguments += " --random.seed %s " % self.randomSeed if self.OutputFile: self.extraCLIarguments += " --outputFile %s " % self.OutputFile scriptName = 'DDSim_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = [] script.append('#!/bin/bash') script.append( '#####################################################################' ) script.append( '# Dynamically generated script to run a production or analysis job. #' ) script.append( '#####################################################################' ) script.append('source %s' % envScriptPath) script.append('echo =========') ## for user provided libraries if os.path.exists("lib"): script.append("export LD_LIBRARY_PATH=$PWD/lib:$LD_LIBRARY_PATH") script.append('env | sort >> localEnv.log') script.append('echo ddsim:`which ddsim`') script.append('echo =========') comm = 'ddsim --compactFile %(compactFile)s %(extraArgs)s' % dict( compactFile=compactFile, extraArgs=self.extraCLIarguments) self.log.info("Command:", comm) script.append(comm) script.append('declare -x appstatus=$?') script.append('exit $appstatus') with open(scriptName, 'w') as scriptFile: scriptFile.write("\n".join(script)) if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'bash "./%s"' % scriptName self.setApplicationStatus('DDSim %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction=self.redirectLogOutput, bufferLimit=20971520) resultTuple = self.result['Value'] if not os.path.exists(self.applicationLog): self.log.error( "Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed to produce log file' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log %s' % (self.applicationName, self.applicationLog)) status = resultTuple[0] self.log.info("Status after the application execution is %s" % status) return self.finalStatusReport(status)
def execute(self): """ Called by Agent Executes the following: - read the application parameters that where defined in ILCJob, and stored in the job definition - setup the SQL server and run it in the background, via a call to L{SQLWrapper} - prepare the steering fie using L{PrepareSteeringFile} - run Mokka and catch its return status @return: S_OK(), S_ERROR() """ result = self.resolveInputVariables() if not result['OK']: return result #if not self.workflowStatus['OK'] or not self.stepStatus['OK']: # self.log.info('Skip this module, failure detected in a previous step :') # self.log.info('Workflow status : %s' %(self.workflowStatus)) # self.log.info('Step Status %s' %(self.stepStatus)) # return S_OK() self.result = S_OK() if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('Mokka should not proceed as previous step did not end properly') cwd = os.getcwd() root = gConfig.getValue('/LocalSite/Root', cwd) self.log.info( "Executing Mokka %s" % ( self.applicationVersion )) self.log.info("Platform for job is %s" % ( self.systemConfig ) ) self.log.info("Root directory for job is %s" % ( root ) ) mokkaDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig, "mokka", self.applicationVersion), '') if not mokkaDir: self.log.error('Could not get Tar ball name') return S_ERROR('Failed finding software directory') mokkaDir = mokkaDir.replace(".tgz", "").replace(".tar.gz", "") #mokkaDir = 'lddLib' ###Temporary while mokka tar ball are not redone. mySoftwareRoot = '' localArea = LocalArea() sharedArea = SharedArea() if os.path.exists('%s%s%s' % (localArea, os.sep, mokkaDir)): mySoftwareRoot = localArea elif os.path.exists('%s%s%s' % (sharedArea, os.sep, mokkaDir)): mySoftwareRoot = sharedArea else: self.log.error("Mokka: could not find installation directory!") return S_ERROR("Mokka installation could not be found") ##This is a duplication of what is above, but the above cannot easily be removed... res = getSoftwareFolder(mokkaDir) if not res['OK']: self.log.error("Mokka: could not find installation directory!") return res myMokkaDir = res['Value'] if not mySoftwareRoot: self.log.error('Directory %s was not found in either the local area %s or shared area %s' % (mokkaDir, localArea, sharedArea)) return S_ERROR('Failed to discover software') ####Setup MySQL instance MokkaDBrandomName = '/tmp/MokkaDBRoot-' + GenRandString(8) #sqlwrapper = SQLWrapper(self.dbslice,mySoftwareRoot,"/tmp/MokkaDBRoot")#mySoftwareRoot) sqlwrapper = SQLWrapper(mySoftwareRoot, MokkaDBrandomName)#mySoftwareRoot) res = sqlwrapper.setDBpath(myMokkaDir, self.dbSlice) if not res['OK']: self.log.error("Failed to find the DB slice") return res result = sqlwrapper.makedirs() if not result['OK']: self.setApplicationStatus('MySQL setup failed to create directories.') return result result = sqlwrapper.mysqlSetup() if not result['OK']: self.setApplicationStatus('MySQL setup failed.') return result ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = GetNewLDLibs(self.systemConfig, "mokka", self.applicationVersion) ##Remove libc removeLibc(myMokkaDir) ##Get the particle.tbl, if any path_to_particle_tbl = '' dir_to_particletbl = os.path.join(myMokkaDir, 'ConfigFiles') if os.path.exists(dir_to_particletbl): configdir = os.listdir(dir_to_particletbl) if 'particle.tbl' in configdir: path_to_particle_tbl = os.path.join(dir_to_particletbl, 'particle.tbl') if os.path.exists('./particle.tbl'): path_to_particle_tbl = "./particle.tbl" ###steering file that will be used to run mokkasteer = "mokka.steer" if os.path.exists("mokka.steer"): try: shutil.move("mokka.steer", "mymokka.steer") except: self.log.error("Failed renaming the steering file") self.SteeringFile = "mymokka.steer" ###prepare steering file #first, I need to take the stdhep file, find its path (possible LFN) if len(self.InputFile) > 0: #self.InputFile = os.path.basename(self.InputFile) res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Generator file not found") result = sqlwrapper.mysqlCleanUp() return res self.InputFile = res['Value'] if len(self.macFile) > 0: self.macFile = os.path.basename(self.macFile) ##idem for steering file self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): res = getSteeringFileDirName(self.systemConfig, "mokka", self.applicationVersion) if not res['OK']: result = sqlwrapper.mysqlCleanUp() return res steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)): try: shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile ) except Exception, x: result = sqlwrapper.mysqlCleanUp() return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(x)))
def runIt(self): """ Called by Agent Execute the following: - resolve where the soft was installed - prepare the list of file to feed Marlin with - create the XML file on which Marlin has to run, done by :any:`prepareXMLFile` - run Marlin and catch the exit code :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('%s should not proceed as previous step did not end properly' % self.applicationName) #get the path to the detector model, either local or from the software compactFile = None if self.detectorModel: resXML = self._getDetectorXML() if not resXML['OK']: self.log.error("Could not obtain the detector XML file: ", resXML["Message"]) return resXML compactFile = resXML['Value'] res = getEnvironmentScript(self.platform, "marlin", self.applicationVersion, self.getEnvScript) if not res['OK']: self.log.error("Failed to get the env script") return res env_script_path = res["Value"] res = self._getInputFiles() if not res['OK']: self.log.error("Failed getting input files:", res['Message']) return res listofslcio = res['Value'] finalXML = "marlinxml_" + self.STEP_NUMBER + ".xml" steeringfiledirname = '' res = getSteeringFileDirName(self.platform, "marlin", self.applicationVersion) if res['OK']: steeringfiledirname = res['Value'] else: self.log.warn('Could not find the steering file directory', res['Message']) ##Handle PandoraSettings.xml pandorasettings = 'PandoraSettings.xml' if not os.path.exists(pandorasettings): if steeringfiledirname and os.path.exists(os.path.join(steeringfiledirname, pandorasettings)): try: shutil.copy(os.path.join(steeringfiledirname, pandorasettings), os.path.join(os.getcwd(), pandorasettings)) except EnvironmentError, x: self.log.warn('Could not copy PandoraSettings.xml, exception: %s' % x)
def runIt(self): """ Called by JobAgent Execute the following: - get the environment variables that should have been set during installation - download the detector model, using CS query to fetch the address - prepare the mac file using :any:`prepareMacFile` - run SLIC on this mac File and catch the exit status :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR('No ILC platform selected') elif not self.applicationLog: self.result = S_ERROR('No Log file provided') if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result['Message']) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK( 'SLIC should not proceed as previous step did not end properly' ) res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript) if not res['OK']: self.log.error("Could not obtain the environment script: ", res["Message"]) return res env_script_path = res["Value"] retMod = self.getDetectorModel() if not retMod: return retMod slicmac = 'slicmac.mac' if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Generator file not found") return res self.InputFile = res['Value'] if len(self.SteeringFile) > 0: self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error( "Could not find where the steering files are") return res steeringfiledirname = res['Value'] if os.path.exists( os.path.join(steeringfiledirname, self.SteeringFile)): self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile) if not os.path.exists(self.SteeringFile): self.log.error("Missing steering file") return S_ERROR("Could not find mac file") ##Same as for mokka: using ParticleGun does not imply InputFile if not len(self.InputFile): self.InputFile = [''] macok = prepareMacFile(self.SteeringFile, slicmac, self.InputFile[0], self.NumberOfEvents, self.startFrom, self.detectorModel, self.RandomSeed, self.OutputFile, self.debug) if not macok['OK']: self.log.error('Failed to create SLIC mac file') return S_ERROR('Error when creating SLIC mac file') scriptName = 'SLIC_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/bash \n') script.write( '#####################################################################\n' ) script.write( '# Dynamically generated script to run a production or analysis job. #\n' ) script.write( '#####################################################################\n' ) script.write("source %s\n" % (env_script_path)) preloadDir = "%s/preload" % os.getcwd() if os.path.exists(preloadDir): preadloadLibs = set(os.listdir(preloadDir)) preloadString = ":".join(lib for lib in preadloadLibs if ".so" in lib) script.write('declare -x LD_LIBRARY_PATH=%s:${LD_LIBRARY_PATH}\n' % preloadDir) script.write('declare -x LD_PRELOAD=%s\n' % preloadString) script.write('echo =========\n') script.write('env | sort >> localEnv.log\n') script.write('echo SLIC:\n') script.write("which slic\n") script.write('echo =========\n') comm = 'slic -P $PARTICLE_TBL -m %s %s\n' % (slicmac, self.extraCLIarguments) self.log.info("Command:", comm) script.write(comm) script.write('declare -x appstatus=$?\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'sh -c "./%s"' % scriptName self.setApplicationStatus('SLIC %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction=self.redirectLogOutput, bufferLimit=20971520) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} resultTuple = self.result['Value'] if not os.path.exists(self.applicationLog): self.log.error( "Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log' % (self.applicationName)) status = resultTuple[0] # stdOutput = resultTuple[1] # stdError = resultTuple[2] self.log.info("Status after the application execution is %s" % str(status)) return self.finalStatusReport(status)
def execute(self): """ Called by JobAgent Execute the following: - get the environment variables that should have been set during installation - download the detector model, using CS query to fetch the address - prepare the mac file using L{PrepareMacFile} - run SLIC on this mac File and catch the exit status @return: S_OK(), S_ERROR() """ self.result = self.resolveInputVariables() if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('SLIC should not proceed as previous step did not end properly') if not os.environ.has_key('SLIC_DIR'): self.log.error('SLIC_DIR not found, probably the software installation failed') return S_ERROR('SLIC_DIR not found, probably the software installation failed') if not os.environ.has_key('SLIC_VERSION'): self.log.error('SLIC_VERSION not found, probably the software installation failed') return S_ERROR('SLIC_VERSION not found, probably the software installation failed') if not os.environ.has_key('LCDD_VERSION'): self.log.error('LCDD_VERSION not found, probably the software installation failed') return S_ERROR('LCDD_VERSION not found, probably the software installation failed') #if not os.environ.has_key('XERCES_VERSION'): # self.log.error('XERCES_VERSION not found, probably the software installation failed') # return S_ERROR('XERCES_VERSION not found, probably the software installation failed') slicDir = os.environ['SLIC_DIR'] res = getSoftwareFolder(slicDir) if not res['OK']: self.log.error('Directory %s was not found in either the local area or shared area' % (slicDir)) return res mySoftwareRoot = res['Value'] ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = GetNewLDLibs(self.systemConfig, "slic", self.applicationVersion) #retrieve detector model from web detector_urls = self.ops.getValue('/SLICweb/SLICDetectorModels', ['']) if len(detector_urls[0]) < 1: self.log.error('Could not find in CS the URL for detector model') return S_ERROR('Could not find in CS the URL for detector model') if not os.path.exists(self.detectorModel + ".zip"): for detector_url in detector_urls: try: urllib.urlretrieve("%s%s" % (detector_url, self.detectorModel + ".zip"), self.detectorModel + ".zip") except: self.log.error("Download of detector model failed") continue if not os.path.exists(self.detectorModel + ".zip"): self.log.error('Detector model %s was not found neither locally nor on the web, exiting' % self.detectorModel) return S_ERROR('Detector model %s was not found neither locally nor on the web, exiting' % self.detectorModel) try: unzip_file_into_dir(open(self.detectorModel + ".zip"), os.getcwd()) except: os.unlink(self.detectorModel + ".zip") self.log.error('Failed to unzip detector model') return S_ERROR('Failed to unzip detector model') #unzip detector model #self.unzip_file_into_dir(open(self.detectorModel+".zip"),os.getcwd()) slicmac = 'slicmac.mac' if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Generator file not found") return res self.InputFile = res['Value'] if len(self.SteeringFile) > 0: self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): res = getSteeringFileDirName(self.systemConfig, "slic", self.applicationVersion) if not res['OK']: self.log.error("Could not find where the steering files are") steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)): self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile) if not os.path.exists(self.SteeringFile): return S_ERROR("Could not find mac file") ##Same as for mokka: using ParticleGun does not imply InputFile if not len(self.InputFile): self.InputFile = [''] macok = PrepareMacFile(self.SteeringFile, slicmac, self.InputFile[0], self.NumberOfEvents, self.startFrom, self.detectorModel, self.randomseed, self.OutputFile, self.debug) if not macok['OK']: self.log.error('Failed to create SLIC mac file') return S_ERROR('Error when creating SLIC mac file') scriptName = 'SLIC_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/sh \n') script.write('#####################################################################\n') script.write('# Dynamically generated script to run a production or analysis job. #\n') script.write('#####################################################################\n') if os.environ.has_key('XERCES_VERSION'): script.write('declare -x XERCES_LIB_DIR=%s/packages/xerces/%s/lib\n' % (mySoftwareRoot, os.environ['XERCES_VERSION'])) if new_ld_lib_path: script.write('declare -x LD_LIBRARY_PATH=$XERCES_LIB_DIR:%s\n' % new_ld_lib_path) else: script.write('declare -x LD_LIBRARY_PATH=$XERCES_LIB_DIR\n') script.write('declare -x GEANT4_DATA_ROOT=%s/packages/geant4/data\n' % mySoftwareRoot) script.write('declare -x G4LEVELGAMMADATA=$(ls -d $GEANT4_DATA_ROOT/PhotonEvaporation*)\n') script.write('declare -x G4RADIOACTIVEDATA=$(ls -d $GEANT4_DATA_ROOT/RadioactiveDecay*)\n') script.write('declare -x G4LEDATA=$(ls -d $GEANT4_DATA_ROOT/G4EMLOW*)\n') script.write('declare -x G4NEUTRONHPDATA=$(ls -d $GEANT4_DATA_ROOT/G4NDL*)\n') script.write('declare -x GDML_SCHEMA_DIR=%s/packages/lcdd/%s\n' % (mySoftwareRoot, os.environ['LCDD_VERSION'])) script.write('declare -x PARTICLE_TBL=%s/packages/slic/%s/data/particle.tbl\n' % (mySoftwareRoot, os.environ['SLIC_VERSION'])) script.write('declare -x MALLOC_CHECK_=0\n') if os.path.exists("%s/lib" % (mySoftwareRoot)): script.write('declare -x LD_LIBRARY_PATH=%s/lib:$LD_LIBRARY_PATH\n' % (mySoftwareRoot)) script.write('echo =========\n') script.write('env | sort >> localEnv.log\n') script.write('echo =========\n') comm = '%s/packages/slic/%s/bin/Linux-g++/slic -P $PARTICLE_TBL -m %s\n' % (mySoftwareRoot, os.environ['SLIC_VERSION'], slicmac) print comm script.write(comm) script.write('declare -x appstatus=$?\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'sh -c "./%s"' % scriptName self.setApplicationStatus('SLIC %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} resultTuple = self.result['Value'] if not os.path.exists(self.applicationLog): self.log.error("Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log' % (self.applicationName)) status = resultTuple[0] # stdOutput = resultTuple[1] # stdError = resultTuple[2] self.log.info( "Status after the application execution is %s" % str( status ) ) return self.finalStatusReport(status)