def test_resolvepaths(self): """test ResolvePathsAndNames resolvePaths.......................................................""" res = resolveIFpaths(self.inputfiles) self.assertTrue('OK' in res) self.assertEqual(res['OK'], True, res) self.assertTrue('Value' in res, res.keys()) self.assertEqual(res['Value'], [os.path.abspath(self.realloc)])
def test_resolvepaths(self): """test ResolvePathsAndNames resolvePaths.......................................................""" res = resolveIFpaths(self.inputfiles) self.assertTrue('OK' in res) self.assertEqual(res['OK'], True, res) self.assertTrue('Value' in res, res.keys() ) self.assertEqual(res['Value'], [ os.path.abspath(self.realloc) ])
def GetInputFiles(self): """ Resolve the input files. But not if in the application definition it was decided that it should forget about the input. """ if self.ignoremissingInput: return S_OK("") res = resolveIFpaths(self.InputFile) if not res['OK']: self.setApplicationStatus('%s: missing slcio file' % self.applicationName) return S_ERROR('Missing slcio file!') runonslcio = res['Value'] listofslcio = string.join(runonslcio, " ") return S_OK(listofslcio)
def _getInputFiles(self): """ Resolve the input files. But not if in the application definition it was decided that it should forget about the input. """ if self.ignoremissingInput: return S_OK("") res = resolveIFpaths(self.InputFile) if not res['OK']: self.setApplicationStatus('%s: missing slcio file' % self.applicationName) return S_ERROR('Missing slcio file!') runonslcio = res['Value'] listofslcio = " ".join(runonslcio) return S_OK(listofslcio)
def runIt(self): """ Called by JobAgent Execute the following: - prepend in the LD_LIBRARY_PATH any lib directory of any dependency (e.g. root) - prepare the list of files to run on - set the cacheDirectory and put in there the alias.properties - set the lcsim file using :any:`prepareLCSIMFile` - run java and catch the exit code :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: LOG.error("Failed to resolve input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('LCSIM should not proceed as previous step did not end properly') res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: LOG.error('LCSIM was not found in either the local area or shared area:', res['Message']) return res lcsim_name = res['Value'] ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName, self.applicationVersion) runonslcio = [] if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.setApplicationStatus('LCSIM: missing input slcio file') return S_ERROR('Missing slcio file!') runonslcio = res['Value'] #for inputfile in inputfilelist: # LOG.verbose("Will try using %s"%(os.path.basename(inputfile))) # runonslcio.append(os.path.join(os.getcwd(),os.path.basename(inputfile))) retMod = self.downloadDetectorZip() if not retMod: return retMod ##Collect jar files to put in classspath jars = [] if os.path.exists("lib"): for libs in os.listdir("lib"): if os.path.basename(libs).find(".jar") > 0: jars.append(os.path.abspath(os.path.join("lib", libs))) new_ld_lib_path = "./lib:%s" % new_ld_lib_path #Remove any libc remaining in .lib removeLibc("./lib") ###Define cache directory as local folder aliasproperties = os.path.basename(self.aliasproperties) cachedir = os.getcwd() if not os.path.isdir(os.path.join(cachedir, ".lcsim")): try: os.mkdir(os.path.join(cachedir, ".lcsim")) except OSError, x: LOG.error("Could not create .lcsim folder !", str(x))
def runIt(self): """ Called by ModuleBase Executes the following: - read the application parameters that where defined in ILCJob, and stored in the job definition - setup the SQL server and run it in the background - prepare the steering file using :any:`prepareSteeringFile` - run Mokka and catch its return status :return: S_OK(), S_ERROR() """ #if not self.workflowStatus['OK'] or not self.stepStatus['OK']: # LOG.info('Skip this module, failure detected in a previous step :') # LOG.info('Workflow status : %s' %(self.workflowStatus)) # LOG.info('Step Status %s' %(self.stepStatus)) # return S_OK() self.result = S_OK() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: LOG.error("Failed to resolve the input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('Mokka should not proceed as previous step did not end properly') LOG.info("Executing Mokka %s" % (self.applicationVersion)) if self.dbSlice: if self.dbSlice.lower().startswith("lfn"): self.dbSlice = os.path.basename(self.dbSlice) self.db_dump_name = self.dbSlice else: self.db_dump_name = "CLICMokkaDB.sql" res = getEnvironmentScript(self.platform, "mokka", self.applicationVersion, self.getEnvScript) LOG.notice("Got the environment script: %s" % res) if not res['OK']: LOG.error("Error getting the env script: ", res['Message']) return res env_script_path = res['Value'] ####Setup MySQL instance mysqlBasePath = '%s/mysqltmp/MokkaDBRoot-%s' %(os.getcwd(), generateRandomString(8)) LOG.notice("Placing mysql files in %s" % mysqlBasePath) ###steering file that will be used to run mokkasteer = "mokka.steer" if os.path.exists("mokka.steer"): try: os.rename("mokka.steer", "mymokka.steer") except EnvironmentError as err: LOG.error("Failed renaming the steering file: ", str(err)) self.SteeringFile = "mymokka.steer" ###prepare steering file #first, I need to take the stdhep file, find its path (possible LFN) if len(self.InputFile) > 0: #self.InputFile = os.path.basename(self.InputFile) res = resolveIFpaths(self.InputFile) if not res['OK']: LOG.error("Generator file not found") return res self.InputFile = res['Value'] if len(self.macFile) > 0: self.macFile = os.path.basename(self.macFile) ##idem for steering file self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): LOG.verbose("Steering file %s not found locally" % self.SteeringFile) res = getSteeringFileDirName(self.platform, "mokka", self.applicationVersion) if not res['OK']: LOG.error("Missing Steering file directory:", res['Message']) return res steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)): try: shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile ) except EnvironmentError as err: LOG.error("Failed copying file", self.SteeringFile) return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(err))) #self.steeringFile = os.path.join(mySoftwareRoot,"steeringfiles",self.steeringFile) if not os.path.exists(self.SteeringFile): LOG.error("Missing steering file, should not happen!") return S_ERROR("Could not find steering file") else: LOG.verbose("Found local copy of %s" % self.SteeringFile) ### The following is because if someone uses particle gun, there is no InputFile if not len(self.InputFile): self.InputFile = [''] steerok = prepareSteeringFile(self.SteeringFile, mokkasteer, self.detectorModel, self.InputFile[0], self.macFile, self.NumberOfEvents, self.startFrom, self.RandomSeed, self.mcRunNumber, self.ProcessID, self.debug, self.OutputFile, self.inputdataMeta) if not steerok['OK']: LOG.error('Failed to create MOKKA steering file') return S_ERROR('Failed to create MOKKA steering file') ###Extra option depending on mokka version mokkaextraoption = "" if self.applicationVersion not in ["v07-02", "v07-02fw", "v07-02fwhp", "MokkaRevision42", "MokkaRevision43", "MokkaRevision44", "Revision45"]: mokkaextraoption = "-U" scriptName = 'Mokka_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/bash \n') script.write('#####################################################################\n') script.write('# Dynamically generated script to run a production or analysis job. #\n') script.write('#####################################################################\n') script.write("source %s\n" % env_script_path) if os.path.exists("./lib"): script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n' ) script.write('echo =============================\n') script.write('echo Content of mokka.steer:\n') script.write('cat mokka.steer\n') if self.macFile: script.write('echo =============================\n') script.write('echo Content of %s\n' % os.path.basename(self.macFile)) script.write("cat %s\n" % os.path.basename(self.macFile)) elif os.path.exists("./mokkamac.mac"): script.write(""" echo ============================= echo Content of mokkamac.mac: cat mokkamac.mac """) script.write('echo =============================\n') script.write('echo LD_LIBRARY_PATH is\n') script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n') script.write('echo =============================\n') script.write('echo PATH is\n') script.write('echo $PATH | tr ":" "\n"\n') if self.debug: script.write('echo ldd of Mokka is\n') script.write('ldd `which Mokka` \n' ) script.write('echo =============================\n') script.write('echo ldd of mysql is\n') script.write('ldd `which mysql` \n' ) script.write('echo =============================\n') script.write('env | sort >> localEnv.log\n') script.write('echo =============================\n') script.write("""if [ ! -e ./%(DBDUMP)s ]; then echo "Missing DB dump!" >&2 exit 1 fi """ % {'DBDUMP': self.db_dump_name} ) ##Now start the MySQL server and configure. script.write("declare -x MOKKADBROOT=%s\n" % mysqlBasePath) script.write("declare -x WORKDIR=%s\n" % os.getcwd()) script.write("declare -x LOGDIR=$WORKDIR/mysqllogs/\n") script.write("mkdir -p $LOGDIR\n") #mysql socket must not be longer than 107 characters! script.write("declare -x SOCKETBASE=/tmp/mokka-%s\n" % generateRandomString(8) ) script.write("mkdir -p $SOCKETBASE\n") script.write("declare -x SOCKETPATH=$SOCKETBASE/mysql.sock\n" ) script.write("mkdir -p $MOKKADBROOT\n") script.write("declare -x MYSQLDATA=$MOKKADBROOT/data\n") script.write("rm -rf $MYSQLDATA\n") script.write("mkdir -p $MYSQLDATA\n") script.write("date\n") script.write("""echo "*** Installing MySQL" echo "mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log" mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log install_st=$? if [ $install_st -ne 0 ] then exit $install_st fi date echo "*** Running mysqld-safe" cd $MYSQL bin/mysqld_safe --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log & COUNT=0 while [ -z "$socket_grep" ] ; do socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH") echo -n . sleep 1 if [ $COUNT -eq 100 ]; then echo "Failed to find socket" break fi COUNT=$(( $COUNT + 1 )) done cd - echo "*** Configuring MySQL" mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot password 'rootpass' declare -x admin_st=$? if [ $admin_st -ne 0 ] then echo "*** mysqladmin failed, cannot proceed" >&2 mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown exit $admin_st fi mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO root;' mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO consult IDENTIFIED BY \"consult\";' mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'DELETE FROM mysql.user WHERE User = \"\"; FLUSH PRIVILEGES;' echo "*** Installing Mokka DB" date echo "mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s" time mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s sleep 5 date\n""" % {'DBDUMP': self.db_dump_name}) #Now take care of the particle tables. script.write(""" if [ -e "./particle.tbl" ] then declare -x PARTICLETBL=./particle.tbl fi if [ -d "${MOKKA}" ] #As the local env does not set MOKKA, it must come from CVMFS then declare -x PARTICLETBL=$MOKKA/particle.tbl fi if [ -d "${MOKKATARBALL}" ] #As the local env does set MOKKATARBALL, it must be there then declare -x PARTICLETBL=$MOKKATARBALL/ConfigFiles/particle.tbl fi echo "/Mokka/init/PDGFile $PARTICLETBL" >> %s \n"""% mokkasteer) ##Now run Mokka comm = 'Mokka %s -h localhost:$SOCKETPATH %s %s\n' % (mokkaextraoption, mokkasteer, self.extraCLIarguments) LOG.info("Command : %s" % (comm)) script.write(comm) script.write('declare -x appstatus=$?\n') #Now shutdown the MySQL server script.write("mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown\n") script.write(""" while [ -n "$socket_grep" ] ; do socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH") echo -n . sleep 1 done """) script.write("rm -f %s\n" % self.db_dump_name)#remove db file script.write('rm -rf $MYSQLDATA\n')#cleanup script.write('rm -rf $MOKKADBROOT\n')#cleanup #script.write('quit\n') #script.write('EOF\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0o755) comm = 'sh -c "./%s"' % scriptName self.setApplicationStatus('Mokka %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} resultTuple = self.result['Value'] status = resultTuple[0] # stdOutput = resultTuple[1] # stdError = resultTuple[2] LOG.info("Status after Mokka execution is %s" % str(status)) if not os.path.exists(self.applicationLog): LOG.error("Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: LOG.error("Missing log file") return S_ERROR('%s did not produce the expected log' % (self.applicationName)) ###Now change the name of Mokka output to the specified filename if os.path.exists("out.slcio"): if len(self.OutputFile) > 0: os.rename("out.slcio", self.OutputFile) failed = False if status not in [0, 106, 9]: LOG.error("Mokka execution completed with errors:") failed = True elif status in [106, 9]: LOG.info("Mokka execution reached end of input generator file") else: LOG.info("Mokka execution finished successfully") message = 'Mokka %s Successful' % (self.applicationVersion) if failed is True: LOG.error("==================================\n StdError:\n") LOG.error(self.stdError) #self.setApplicationStatus('%s Exited With Status %s' %(self.applicationName,status)) LOG.error('Mokka Exited With Status %s' % (status)) message = 'Mokka Exited With Status %s' % (status) self.setApplicationStatus(message) if not self.ignoreapperrors: LOG.error("Application exists with error:", message) return S_ERROR(message) else: if status in [106, 9]: message = 'Mokka %s reached end of input generator file' % (self.applicationVersion) self.setApplicationStatus(message) return S_OK( { 'OutputFile' : self.OutputFile } )
def execute(self): """ Called by JobAgent Execute the following: - prepend in the LD_LIBRARY_PATH any lib directory of any dependency (e.g. root) - prepare the list of files to run on - set the cacheDirectory and put in there the alias.properties - set the lcsim file using L{PrepareLCSIMFile} - run java and catch the exit code @return: S_OK(), S_ERROR() """ self.result = self.resolveInputVariables() if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('LCSIM should not proceed as previous step did not end properly') #look for lcsim filename lcsim_name = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall'%(self.systemConfig, "lcsim", self.applicationVersion), '') if not lcsim_name: self.log.error("Could not find lcsim file name from CS") return S_ERROR("Could not find lcsim file name from CS") res = getSoftwareFolder(lcsim_name) if not res['OK']: self.log.error('Application %s was not found in either the local area or shared area' % (lcsim_name)) return res lcsim_name = res['Value'] ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = GetNewLDLibs(self.systemConfig, "lcsim", self.applicationVersion) runonslcio = [] if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.setApplicationStatus('LCSIM: missing input slcio file') return S_ERROR('Missing slcio file!') runonslcio = res['Value'] #for inputfile in inputfilelist: # self.log.verbose("Will try using %s"%(os.path.basename(inputfile))) # runonslcio.append(os.path.join(os.getcwd(),os.path.basename(inputfile))) ##Collect jar files to put in classspath jars = [] if os.path.exists("lib"): for libs in os.listdir("lib"): if os.path.basename(libs).find(".jar") > 0: jars.append(os.path.abspath(os.path.join("lib", libs))) new_ld_lib_path = "./lib:%s" % new_ld_lib_path #Remove any libc remaining in .lib removeLibc("./lib") ###Define cache directory as local folder aliasproperties = os.path.basename(self.aliasproperties) cachedir = os.getcwd() if not os.path.isdir(os.path.join(cachedir, ".lcsim")): try: os.mkdir(os.path.join(cachedir, ".lcsim")) except OSError, x: self.log.error("Could not create .lcsim folder !", str(x))
def runIt(self): """ Called by ModuleBase Executes the following: - read the application parameters that where defined in ILCJob, and stored in the job definition - setup the SQL server and run it in the background - prepare the steering file using :any:`prepareSteeringFile` - run Mokka and catch its return status :return: S_OK(), S_ERROR() """ #if not self.workflowStatus['OK'] or not self.stepStatus['OK']: # self.log.info('Skip this module, failure detected in a previous step :') # self.log.info('Workflow status : %s' %(self.workflowStatus)) # self.log.info('Step Status %s' %(self.stepStatus)) # return S_OK() self.result = S_OK() if not self.platform: self.result = S_ERROR('No ILC platform selected') elif not self.applicationLog: self.result = S_ERROR('No Log file provided') if not self.result['OK']: self.log.error("Failed to resolve the input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK( 'Mokka should not proceed as previous step did not end properly' ) self.log.info("Executing Mokka %s" % (self.applicationVersion)) if self.dbSlice: if self.dbSlice.lower().startswith("lfn"): self.dbSlice = os.path.basename(self.dbSlice) self.db_dump_name = self.dbSlice else: self.db_dump_name = "CLICMokkaDB.sql" res = getEnvironmentScript(self.platform, "mokka", self.applicationVersion, self.getEnvScript) self.log.notice("Got the environment script: %s" % res) if not res['OK']: self.log.error("Error getting the env script: ", res['Message']) return res env_script_path = res['Value'] ####Setup MySQL instance mysqlBasePath = '%s/mysqltmp/MokkaDBRoot-%s' % ( os.getcwd(), generateRandomString(8)) self.log.notice("Placing mysql files in %s" % mysqlBasePath) ###steering file that will be used to run mokkasteer = "mokka.steer" if os.path.exists("mokka.steer"): try: os.rename("mokka.steer", "mymokka.steer") except EnvironmentError as err: self.log.error("Failed renaming the steering file: ", str(err)) self.SteeringFile = "mymokka.steer" ###prepare steering file #first, I need to take the stdhep file, find its path (possible LFN) if len(self.InputFile) > 0: #self.InputFile = os.path.basename(self.InputFile) res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Generator file not found") return res self.InputFile = res['Value'] if len(self.macFile) > 0: self.macFile = os.path.basename(self.macFile) ##idem for steering file self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): self.log.verbose("Steering file %s not found locally" % self.SteeringFile) res = getSteeringFileDirName(self.platform, "mokka", self.applicationVersion) if not res['OK']: self.log.error("Missing Steering file directory:", res['Message']) return res steeringfiledirname = res['Value'] if os.path.exists( os.path.join(steeringfiledirname, self.SteeringFile)): try: shutil.copy( os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile) except EnvironmentError as err: self.log.error("Failed copying file", self.SteeringFile) return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(err))) #self.steeringFile = os.path.join(mySoftwareRoot,"steeringfiles",self.steeringFile) if not os.path.exists(self.SteeringFile): self.log.error("Missing steering file, should not happen!") return S_ERROR("Could not find steering file") else: self.log.verbose("Found local copy of %s" % self.SteeringFile) ### The following is because if someone uses particle gun, there is no InputFile if not len(self.InputFile): self.InputFile = [''] steerok = prepareSteeringFile( self.SteeringFile, mokkasteer, self.detectorModel, self.InputFile[0], self.macFile, self.NumberOfEvents, self.startFrom, self.RandomSeed, self.mcRunNumber, self.ProcessID, self.debug, self.OutputFile, self.inputdataMeta) if not steerok['OK']: self.log.error('Failed to create MOKKA steering file') return S_ERROR('Failed to create MOKKA steering file') ###Extra option depending on mokka version mokkaextraoption = "" if self.applicationVersion not in [ "v07-02", "v07-02fw", "v07-02fwhp", "MokkaRevision42", "MokkaRevision43", "MokkaRevision44", "Revision45" ]: mokkaextraoption = "-U" scriptName = 'Mokka_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/bash \n') script.write( '#####################################################################\n' ) script.write( '# Dynamically generated script to run a production or analysis job. #\n' ) script.write( '#####################################################################\n' ) script.write("source %s\n" % env_script_path) if os.path.exists("./lib"): script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n') script.write('echo =============================\n') script.write('echo Content of mokka.steer:\n') script.write('cat mokka.steer\n') if self.macFile: script.write('echo =============================\n') script.write('echo Content of %s\n' % os.path.basename(self.macFile)) script.write("cat %s\n" % os.path.basename(self.macFile)) elif os.path.exists("./mokkamac.mac"): script.write(""" echo ============================= echo Content of mokkamac.mac: cat mokkamac.mac """) script.write('echo =============================\n') script.write('echo LD_LIBRARY_PATH is\n') script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n') script.write('echo =============================\n') script.write('echo PATH is\n') script.write('echo $PATH | tr ":" "\n"\n') if self.debug: script.write('echo ldd of Mokka is\n') script.write('ldd `which Mokka` \n') script.write('echo =============================\n') script.write('echo ldd of mysql is\n') script.write('ldd `which mysql` \n') script.write('echo =============================\n') script.write('env | sort >> localEnv.log\n') script.write('echo =============================\n') script.write("""if [ ! -e ./%(DBDUMP)s ]; then echo "Missing DB dump!" >&2 exit 1 fi """ % {'DBDUMP': self.db_dump_name}) ##Now start the MySQL server and configure. script.write("declare -x MOKKADBROOT=%s\n" % mysqlBasePath) script.write("declare -x WORKDIR=%s\n" % os.getcwd()) script.write("declare -x LOGDIR=$WORKDIR/mysqllogs/\n") script.write("mkdir -p $LOGDIR\n") #mysql socket must not be longer than 107 characters! script.write("declare -x SOCKETBASE=/tmp/mokka-%s\n" % generateRandomString(8)) script.write("mkdir -p $SOCKETBASE\n") script.write("declare -x SOCKETPATH=$SOCKETBASE/mysql.sock\n") script.write("mkdir -p $MOKKADBROOT\n") script.write("declare -x MYSQLDATA=$MOKKADBROOT/data\n") script.write("rm -rf $MYSQLDATA\n") script.write("mkdir -p $MYSQLDATA\n") script.write("date\n") script.write("""echo "*** Installing MySQL" echo "mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log" mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log install_st=$? if [ $install_st -ne 0 ] then exit $install_st fi date echo "*** Running mysqld-safe" cd $MYSQL bin/mysqld_safe --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log & COUNT=0 while [ -z "$socket_grep" ] ; do socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH") echo -n . sleep 1 if [ $COUNT -eq 100 ]; then echo "Failed to find socket" break fi COUNT=$(( $COUNT + 1 )) done cd - echo "*** Configuring MySQL" mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot password 'rootpass' declare -x admin_st=$? if [ $admin_st -ne 0 ] then echo "*** mysqladmin failed, cannot proceed" >&2 mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown exit $admin_st fi mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO root;' mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO consult IDENTIFIED BY \"consult\";' mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'DELETE FROM mysql.user WHERE User = \"\"; FLUSH PRIVILEGES;' echo "*** Installing Mokka DB" date echo "mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s" time mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s sleep 5 date\n""" % {'DBDUMP': self.db_dump_name}) #Now take care of the particle tables. script.write(""" if [ -e "./particle.tbl" ] then declare -x PARTICLETBL=./particle.tbl fi if [ -d "${MOKKA}" ] #As the local env does not set MOKKA, it must come from CVMFS then declare -x PARTICLETBL=$MOKKA/particle.tbl fi if [ -d "${MOKKATARBALL}" ] #As the local env does set MOKKATARBALL, it must be there then declare -x PARTICLETBL=$MOKKATARBALL/ConfigFiles/particle.tbl fi echo "/Mokka/init/PDGFile $PARTICLETBL" >> %s \n""" % mokkasteer) ##Now run Mokka comm = 'Mokka %s -h localhost:$SOCKETPATH %s %s\n' % ( mokkaextraoption, mokkasteer, self.extraCLIarguments) self.log.info("Command : %s" % (comm)) script.write(comm) script.write('declare -x appstatus=$?\n') #Now shutdown the MySQL server script.write( "mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown\n" ) script.write(""" while [ -n "$socket_grep" ] ; do socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH") echo -n . sleep 1 done """) script.write("rm -f %s\n" % self.db_dump_name) #remove db file script.write('rm -rf $MYSQLDATA\n') #cleanup script.write('rm -rf $MOKKADBROOT\n') #cleanup #script.write('quit\n') #script.write('EOF\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'sh -c "./%s"' % scriptName self.setApplicationStatus('Mokka %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction=self.redirectLogOutput, bufferLimit=20971520) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} resultTuple = self.result['Value'] status = resultTuple[0] # stdOutput = resultTuple[1] # stdError = resultTuple[2] self.log.info("Status after Mokka execution is %s" % str(status)) if not os.path.exists(self.applicationLog): self.log.error( "Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: self.log.error("Missing log file") return S_ERROR('%s did not produce the expected log' % (self.applicationName)) ###Now change the name of Mokka output to the specified filename if os.path.exists("out.slcio"): if len(self.OutputFile) > 0: os.rename("out.slcio", self.OutputFile) failed = False if status not in [0, 106, 9]: self.log.error("Mokka execution completed with errors:") failed = True elif status in [106, 9]: self.log.info( "Mokka execution reached end of input generator file") else: self.log.info("Mokka execution finished successfully") message = 'Mokka %s Successful' % (self.applicationVersion) if failed is True: self.log.error("==================================\n StdError:\n") self.log.error(self.stdError) #self.setApplicationStatus('%s Exited With Status %s' %(self.applicationName,status)) self.log.error('Mokka Exited With Status %s' % (status)) message = 'Mokka Exited With Status %s' % (status) self.setApplicationStatus(message) if not self.ignoreapperrors: self.log.error("Application exists with error:", message) return S_ERROR(message) else: if status in [106, 9]: message = 'Mokka %s reached end of input generator file' % ( self.applicationVersion) self.setApplicationStatus(message) return S_OK({'OutputFile': self.OutputFile})
def execute(self): """ Run the module """ result = self.resolveInputVariables() if not result['OK']: return result self.result = S_OK() if not self.applicationLog: self.result = S_ERROR('No Log file provided') if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK( '%s should not proceed as previous step did not end properly' % self.applicationName) ### Now remove the files in the FC lfns = self.InputFile ##Check that all the files are here: res = resolveIFpaths(lfns) if not res['OK']: self.log.error(res['Message']) return S_ERROR("Failed to find a file locally") #All files are here and available paths = res['Value'] localpaths = [] for inputfile in paths: basename = os.path.basename(inputfile) locname = os.path.join(os.getcwd(), basename) if not locname == inputfile: try: shutil.copy(inputfile, locname) except shutil.Error: self.log.error( "Failed to copy file locally, will have to stop") return S_ERROR("Failed copy to local directory") localpaths.append(locname) try: os.unlink(inputfile) except OSError: self.log.warn("Failed to remove initial file, increased \ disk space usage") #all the files are in the run directory #get all metadata, ancestor/daughter relations, etc. for all the files #Update the listoutput if self.listoutput: outputlist = [] for localFile in localpaths: item = {} item['outputFile'] = localFile item['outputPath'] = self.listoutput['outputPath'] item['outputDataSE'] = self.listoutput['outputDataSE'] outputlist.append(item) if self.enable: self.step_commons['listoutput'] = outputlist else: self.log.info("listoutput would have been ", outputlist) ## Make sure the path contains / at the end as we are going to ## concatenate final path and local files if not self.outputpath[-1] == '/': self.outputpath += "/" if 'ProductionOutputData' in self.workflow_commons: file_list = ";".join([ self.outputpath + name for name in [os.path.basename(fin) for fin in localpaths] ]) if self.enable: self.workflow_commons['ProductionOutputData'] = file_list else: self.log.info("ProductionOutputData would have been", file_list) #Now remove them if self.enable: res = self.repMan.removeFile(lfns, force=True) if not res['OK']: self.log.error("Failed to remove the files") self.setApplicationStatus("Failed to remove the file") #return S_ERROR("Failed to remove the files") else: self.log.info("Would have removed: ", "%s" % str(lfns)) ## Now the files are not on the storage anymore, they exist only locally. We can hope ## that the job will not be killed between now and the time the UploadOutputData module ## is called return self.finalStatusReport(0)
def runIt(self): """ Called by JobAgent Execute the following: - get the environment variables that should have been set during installation - find the detector model xml, using CS query to obtain the path - prepare the steering file and command line parameters - run DDSim on this steering file and catch the exit status :rtype: S_OK, S_ERROR """ self.result = S_OK() if not self.platform: self.result = S_ERROR('No ILC platform selected') elif not self.applicationLog: self.result = S_ERROR('No Log file provided') if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result['Message']) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK( 'DDSim should not proceed as previous step did not end properly' ) ##TODO: Setup LD_LIBRARY_PATH for extensions res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript) if not res['OK']: self.log.error("Could not obtain the environment script: ", res["Message"]) return res envScriptPath = res["Value"] #get the path to the detector model, either local or from the software resXML = self._getDetectorXML() if not resXML['OK']: self.log.error("Could not obtain the detector XML file: ", resXML["Message"]) return resXML compactFile = resXML['Value'] if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Generator file not found") return res self.InputFile = res['Value'] ## if steering file is set try to find it if len(self.SteeringFile) > 0: self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error( "Could not find where the steering files are") return res steeringfiledirname = res['Value'] if os.path.exists( os.path.join(steeringfiledirname, self.SteeringFile)): self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile) if not os.path.exists(self.SteeringFile): self.log.error("Missing steering file") return S_ERROR("Could not find steering file") self.extraCLIarguments += " --steeringFile %s " % self.SteeringFile if self.startFrom: self.extraCLIarguments += " --skipNEvents %s " % self.startFrom if self.debug: self.extraCLIarguments += " --printLevel DEBUG " ##Same as for mokka: using ParticleGun does not imply InputFile if self.InputFile: self.InputFile = [self.InputFile] if isinstance( self.InputFile, basestring) else self.InputFile self.extraCLIarguments += " --inputFile %s " % self.InputFile[0] if self.NumberOfEvents: self.extraCLIarguments += " --numberOfEvents %s " % self.NumberOfEvents self.extraCLIarguments += " --random.seed %s " % self.randomSeed if self.OutputFile: self.extraCLIarguments += " --outputFile %s " % self.OutputFile scriptName = 'DDSim_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = [] script.append('#!/bin/bash') script.append( '#####################################################################' ) script.append( '# Dynamically generated script to run a production or analysis job. #' ) script.append( '#####################################################################' ) script.append('source %s' % envScriptPath) script.append('echo =========') ## for user provided libraries if os.path.exists("lib"): script.append("export LD_LIBRARY_PATH=$PWD/lib:$LD_LIBRARY_PATH") script.append('env | sort >> localEnv.log') script.append('echo ddsim:`which ddsim`') script.append('echo =========') comm = 'ddsim --compactFile %(compactFile)s %(extraArgs)s' % dict( compactFile=compactFile, extraArgs=self.extraCLIarguments) self.log.info("Command:", comm) script.append(comm) script.append('declare -x appstatus=$?') script.append('exit $appstatus') with open(scriptName, 'w') as scriptFile: scriptFile.write("\n".join(script)) if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'bash "./%s"' % scriptName self.setApplicationStatus('DDSim %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction=self.redirectLogOutput, bufferLimit=20971520) resultTuple = self.result['Value'] if not os.path.exists(self.applicationLog): self.log.error( "Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed to produce log file' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log %s' % (self.applicationName, self.applicationLog)) status = resultTuple[0] self.log.info("Status after the application execution is %s" % status) return self.finalStatusReport(status)
def execute(self): """ Execute the module, called by JobAgent """ # Checks resultRIV = self.resolveInputVariables() if not resultRIV['OK']: self.log.error("Failed to resolve input variables", resultRIV['Message']) return resultRIV if not self.platform: return S_ERROR('No ILC platform selected') if "LCIO" not in os.environ: self.log.error( "Environment variable LCIO was not defined, cannot do anything" ) return S_ERROR( "Environment variable LCIO was not defined, cannot do anything" ) if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Missing slcio file!") self.setApplicationStatus( 'LCIOSplit: missing input slcio file') return S_ERROR('Missing slcio file!') runonslcio = res['Value'][0] else: return S_OK("No files found to process") removeLibc(os.path.join(os.environ["LCIO"], "lib")) # Setting up script LD_LIBRARY_PATH = os.path.join("$LCIO", "lib") if 'LD_LIBRARY_PATH' in os.environ: LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH'] PATH = "$LCIO/bin" if 'PATH' in os.environ: PATH += ":" + os.environ['PATH'] scriptContent = """ #!/bin/sh ################################################################################ # Dynamically generated script by LCIOConcatenate module # ################################################################################ declare -x LD_LIBRARY_PATH=%s declare -x PATH=%s lcio split -i %s -n %s exit $? """ % (LD_LIBRARY_PATH, PATH, runonslcio, self.nbEventsPerSlice) # Write script to file scriptPath = 'LCIOSplit_%s_Run_%s.tcl' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptPath): os.remove(scriptPath) script = open(scriptPath, 'w') script.write(scriptContent) script.close() # Setup log file for application stdout if os.path.exists(self.applicationLog): os.remove(self.applicationLog) # Run code os.chmod(scriptPath, 0755) command = '"./%s"' % (scriptPath) self.setApplicationStatus('LCIOSplit %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' result = shellCall(0, command, callbackFunction=self.redirectLogOutput, bufferLimit=20971520) # Check results status = result['Value'][0] if not os.path.exists(self.applicationLog): self.log.error("Cannot access log file, cannot proceed") return S_ERROR("Failed reading the log file") baseinputfilename = os.path.basename(runonslcio).split(".slcio")[0] output_file_base_name = '' if self.OutputFile: output_file_base_name = self.OutputFile.split('.slcio')[0] self.log.info("Will rename all files using '%s' as base." % output_file_base_name) numberofeventsdict = {} fname = '' with open(self.applicationLog, "r") as logf: for line in logf: line = line.rstrip() if line.count(baseinputfilename): #First, we need to rename those guys current_file = os.path.basename(line).replace(".slcio", "") current_file_extension = current_file.replace( baseinputfilename, "") newfile = output_file_base_name + current_file_extension + ".slcio" os.rename(line, newfile) fname = newfile numberofeventsdict[fname] = 0 elif line.count("events"): numberofeventsdict[fname] = int(line.split()[0]) self.log.verbose("Number of eventsdict dict: %s" % numberofeventsdict) ##Now update the workflow_commons dict with the relation between filename and number of events: needed for #the registerOutputData self.workflow_commons[ 'file_number_of_event_relation'] = numberofeventsdict if self.listoutput: outputlist = [] for fileName in numberofeventsdict: item = {} item['outputFile'] = fileName item['outputPath'] = self.listoutput['outputPath'] item['outputDataSE'] = self.listoutput['outputDataSE'] outputlist.append(item) self.step_commons['listoutput'] = outputlist #Not only the step_commons must be updated if 'ProductionOutputData' in self.workflow_commons: proddata = self.workflow_commons['ProductionOutputData'].split(";") finalproddata = [] this_split_data = '' for item in proddata: if not item.count(output_file_base_name): finalproddata.append(item) else: this_split_data = item path = os.path.dirname(this_split_data) for fileName in numberofeventsdict: finalproddata.append(os.path.join(path, fileName)) self.workflow_commons['ProductionOutputData'] = ";".join( finalproddata) self.log.info("Status after the application execution is %s" % str(status)) self.listDir() return self.finalStatusReport(status)
def execute(self): """ Called by Agent Executes the following: - read the application parameters that where defined in ILCJob, and stored in the job definition - setup the SQL server and run it in the background, via a call to L{SQLWrapper} - prepare the steering fie using L{PrepareSteeringFile} - run Mokka and catch its return status @return: S_OK(), S_ERROR() """ result = self.resolveInputVariables() if not result['OK']: return result #if not self.workflowStatus['OK'] or not self.stepStatus['OK']: # self.log.info('Skip this module, failure detected in a previous step :') # self.log.info('Workflow status : %s' %(self.workflowStatus)) # self.log.info('Step Status %s' %(self.stepStatus)) # return S_OK() self.result = S_OK() if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('Mokka should not proceed as previous step did not end properly') cwd = os.getcwd() root = gConfig.getValue('/LocalSite/Root', cwd) self.log.info( "Executing Mokka %s" % ( self.applicationVersion )) self.log.info("Platform for job is %s" % ( self.systemConfig ) ) self.log.info("Root directory for job is %s" % ( root ) ) mokkaDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig, "mokka", self.applicationVersion), '') if not mokkaDir: self.log.error('Could not get Tar ball name') return S_ERROR('Failed finding software directory') mokkaDir = mokkaDir.replace(".tgz", "").replace(".tar.gz", "") #mokkaDir = 'lddLib' ###Temporary while mokka tar ball are not redone. mySoftwareRoot = '' localArea = LocalArea() sharedArea = SharedArea() if os.path.exists('%s%s%s' % (localArea, os.sep, mokkaDir)): mySoftwareRoot = localArea elif os.path.exists('%s%s%s' % (sharedArea, os.sep, mokkaDir)): mySoftwareRoot = sharedArea else: self.log.error("Mokka: could not find installation directory!") return S_ERROR("Mokka installation could not be found") ##This is a duplication of what is above, but the above cannot easily be removed... res = getSoftwareFolder(mokkaDir) if not res['OK']: self.log.error("Mokka: could not find installation directory!") return res myMokkaDir = res['Value'] if not mySoftwareRoot: self.log.error('Directory %s was not found in either the local area %s or shared area %s' % (mokkaDir, localArea, sharedArea)) return S_ERROR('Failed to discover software') ####Setup MySQL instance MokkaDBrandomName = '/tmp/MokkaDBRoot-' + GenRandString(8) #sqlwrapper = SQLWrapper(self.dbslice,mySoftwareRoot,"/tmp/MokkaDBRoot")#mySoftwareRoot) sqlwrapper = SQLWrapper(mySoftwareRoot, MokkaDBrandomName)#mySoftwareRoot) res = sqlwrapper.setDBpath(myMokkaDir, self.dbSlice) if not res['OK']: self.log.error("Failed to find the DB slice") return res result = sqlwrapper.makedirs() if not result['OK']: self.setApplicationStatus('MySQL setup failed to create directories.') return result result = sqlwrapper.mysqlSetup() if not result['OK']: self.setApplicationStatus('MySQL setup failed.') return result ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = GetNewLDLibs(self.systemConfig, "mokka", self.applicationVersion) ##Remove libc removeLibc(myMokkaDir) ##Get the particle.tbl, if any path_to_particle_tbl = '' dir_to_particletbl = os.path.join(myMokkaDir, 'ConfigFiles') if os.path.exists(dir_to_particletbl): configdir = os.listdir(dir_to_particletbl) if 'particle.tbl' in configdir: path_to_particle_tbl = os.path.join(dir_to_particletbl, 'particle.tbl') if os.path.exists('./particle.tbl'): path_to_particle_tbl = "./particle.tbl" ###steering file that will be used to run mokkasteer = "mokka.steer" if os.path.exists("mokka.steer"): try: shutil.move("mokka.steer", "mymokka.steer") except: self.log.error("Failed renaming the steering file") self.SteeringFile = "mymokka.steer" ###prepare steering file #first, I need to take the stdhep file, find its path (possible LFN) if len(self.InputFile) > 0: #self.InputFile = os.path.basename(self.InputFile) res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Generator file not found") result = sqlwrapper.mysqlCleanUp() return res self.InputFile = res['Value'] if len(self.macFile) > 0: self.macFile = os.path.basename(self.macFile) ##idem for steering file self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): res = getSteeringFileDirName(self.systemConfig, "mokka", self.applicationVersion) if not res['OK']: result = sqlwrapper.mysqlCleanUp() return res steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)): try: shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile ) except Exception, x: result = sqlwrapper.mysqlCleanUp() return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(x)))
def runIt(self): """ Called by JobAgent Execute the following: - get the environment variables that should have been set during installation - download the detector model, using CS query to fetch the address - prepare the mac file using :any:`prepareMacFile` - run SLIC on this mac File and catch the exit status :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR('No ILC platform selected') elif not self.applicationLog: self.result = S_ERROR('No Log file provided') if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result['Message']) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK( 'SLIC should not proceed as previous step did not end properly' ) res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript) if not res['OK']: self.log.error("Could not obtain the environment script: ", res["Message"]) return res env_script_path = res["Value"] retMod = self.getDetectorModel() if not retMod: return retMod slicmac = 'slicmac.mac' if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Generator file not found") return res self.InputFile = res['Value'] if len(self.SteeringFile) > 0: self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error( "Could not find where the steering files are") return res steeringfiledirname = res['Value'] if os.path.exists( os.path.join(steeringfiledirname, self.SteeringFile)): self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile) if not os.path.exists(self.SteeringFile): self.log.error("Missing steering file") return S_ERROR("Could not find mac file") ##Same as for mokka: using ParticleGun does not imply InputFile if not len(self.InputFile): self.InputFile = [''] macok = prepareMacFile(self.SteeringFile, slicmac, self.InputFile[0], self.NumberOfEvents, self.startFrom, self.detectorModel, self.RandomSeed, self.OutputFile, self.debug) if not macok['OK']: self.log.error('Failed to create SLIC mac file') return S_ERROR('Error when creating SLIC mac file') scriptName = 'SLIC_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/bash \n') script.write( '#####################################################################\n' ) script.write( '# Dynamically generated script to run a production or analysis job. #\n' ) script.write( '#####################################################################\n' ) script.write("source %s\n" % (env_script_path)) preloadDir = "%s/preload" % os.getcwd() if os.path.exists(preloadDir): preadloadLibs = set(os.listdir(preloadDir)) preloadString = ":".join(lib for lib in preadloadLibs if ".so" in lib) script.write('declare -x LD_LIBRARY_PATH=%s:${LD_LIBRARY_PATH}\n' % preloadDir) script.write('declare -x LD_PRELOAD=%s\n' % preloadString) script.write('echo =========\n') script.write('env | sort >> localEnv.log\n') script.write('echo SLIC:\n') script.write("which slic\n") script.write('echo =========\n') comm = 'slic -P $PARTICLE_TBL -m %s %s\n' % (slicmac, self.extraCLIarguments) self.log.info("Command:", comm) script.write(comm) script.write('declare -x appstatus=$?\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'sh -c "./%s"' % scriptName self.setApplicationStatus('SLIC %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction=self.redirectLogOutput, bufferLimit=20971520) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} resultTuple = self.result['Value'] if not os.path.exists(self.applicationLog): self.log.error( "Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log' % (self.applicationName)) status = resultTuple[0] # stdOutput = resultTuple[1] # stdError = resultTuple[2] self.log.info("Status after the application execution is %s" % str(status)) return self.finalStatusReport(status)
def runIt(self): """ Called from ModuleBase """ self.result = S_OK() if not self.platform: self.result = S_ERROR('No ILC platform selected') elif not self.applicationLog: self.result = S_ERROR('No Log file provided') if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK( 'SLIC Pandora should not proceed as previous step did not end properly' ) ###Get the env script res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript) if not res['OK']: self.log.error("Failed to get the environment script:", res["Message"]) return res env_script_path = res["Value"] res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error('Could not find input files') self.setApplicationStatus('SLICPandora: missing slcio file') return S_ERROR('Missing slcio file!') runonslcio = res['Value'][0] if not self.detectorxml.count(".xml") or not os.path.exists( os.path.basename(self.detectorxml)): detmodel = self.detectorxml.replace("_pandora.xml", "") if os.path.exists(detmodel + ".zip"): try: unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd()) except (RuntimeError, OSError) as err: self.log.error( "Exception when unpacking detectormodel zip file:", str(err)) os.unlink(detmodel + ".zip") if not os.path.exists(detmodel + ".zip"): #retrieve detector model from web detector_urls = self.ops.getValue( '/SLICweb/SLICDetectorModels', ['']) if len(detector_urls[0]) < 1: self.log.error( 'Could not find in CS the URL for detector model') return S_ERROR( 'Could not find in CS the URL for detector model') for detector_url in detector_urls: try: urllib.urlretrieve( "%s%s" % (detector_url, detmodel + ".zip"), detmodel + ".zip") except IOError: self.log.error("Download of detector model failed") continue try: unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd()) break except (RuntimeError, OSError) as err: self.log.error("Exception for zip file obtained from ", detector_url) self.log.error("Exception:", str(err)) os.unlink(detmodel + ".zip") continue #if os.path.exists(detmodel): #and os.path.isdir(detmodel): self.detectorxml = os.path.join(os.getcwd(), self.detectorxml) if not self.detectorxml.endswith('xml'): self.detectorxml = self.detectorxml + "_pandora.xml" if not os.path.exists(self.detectorxml): self.log.error('Detector model xml %s was not found, exiting' % self.detectorxml) return S_ERROR('Detector model xml was not found, exiting') oldversion = False if self.applicationVersion in [ 'CLIC_CDR', 'CDR1', 'CDR2', 'CDR0', 'V2', 'V3', 'V4' ]: oldversion = True scriptName = 'SLICPandora_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/bash \n') script.write( '#####################################################################\n' ) script.write( '# Dynamically generated script to run a production or analysis job. #\n' ) script.write( '#####################################################################\n' ) script.write("source %s\n" % env_script_path) script.write("declare -x file=./Settings/%s\n" % self.pandorasettings) script.write(""" if [ -e "${file}" ] then declare -x PANDORASETTINGS=$file else if [ -d "${PANDORASETTINGSDIR}" ] then cp $PANDORASETTINGSDIR/*.xml . declare -x PANDORASETTINGS=%s fi fi if [ ! -e "${PANDORASETTINGS}" ] then echo "Missing PandoraSettings file" exit 1 fi """ % self.pandorasettings) script.write('echo =============================\n') script.write('echo PATH is \n') script.write('echo $PATH | tr ":" "\n" \n') script.write('echo ==============\n') if os.path.exists("./lib"): script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n') script.write('echo =============================\n') script.write('echo LD_LIBRARY_PATH is \n') script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n') script.write('echo ============================= \n') script.write('env | sort >> localEnv.log\n') #Now run it if oldversion: comm = 'PandoraFrontend %s $PANDORASETTINGS %s %s %s' % ( self.detectorxml, runonslcio, self.OutputFile, str(self.NumberOfEvents)) else: comm = 'PandoraFrontend -g %s -c $PANDORASETTINGS -i %s -o %s -r %s' % ( self.detectorxml, runonslcio, self.OutputFile, str(self.NumberOfEvents)) comm = "%s %s\n" % (comm, self.extraCLIarguments) self.log.info("Will run %s" % comm) script.write(comm) script.write('declare -x appstatus=$?\n') #script.write('where\n') #script.write('quit\n') #script.write('EOF\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'sh -c "./%s"' % (scriptName) self.setApplicationStatus('SLICPandora %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction=self.redirectLogOutput, bufferLimit=20971520) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} resultTuple = self.result['Value'] if resultTuple[0]: self.log.error("There was an error during the execution") if not os.path.exists(self.applicationLog): self.log.error( "Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log' % (self.applicationName)) logf = open(self.applicationLog, 'r') if "Missing PandoraSettings file" in logf.readlines()[-1]: self.log.error("Issue with the Pandora Settings file.") logf.close() status = resultTuple[0] # stdOutput = resultTuple[1] # stdError = resultTuple[2] self.log.info("Status after the application execution is %s" % str(status)) return self.finalStatusReport(status)
def execute(self): """ Called by JobAgent Execute the following: - get the environment variables that should have been set during installation - download the detector model, using CS query to fetch the address - prepare the mac file using L{PrepareMacFile} - run SLIC on this mac File and catch the exit status @return: S_OK(), S_ERROR() """ self.result = self.resolveInputVariables() if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('SLIC should not proceed as previous step did not end properly') if not os.environ.has_key('SLIC_DIR'): self.log.error('SLIC_DIR not found, probably the software installation failed') return S_ERROR('SLIC_DIR not found, probably the software installation failed') if not os.environ.has_key('SLIC_VERSION'): self.log.error('SLIC_VERSION not found, probably the software installation failed') return S_ERROR('SLIC_VERSION not found, probably the software installation failed') if not os.environ.has_key('LCDD_VERSION'): self.log.error('LCDD_VERSION not found, probably the software installation failed') return S_ERROR('LCDD_VERSION not found, probably the software installation failed') #if not os.environ.has_key('XERCES_VERSION'): # self.log.error('XERCES_VERSION not found, probably the software installation failed') # return S_ERROR('XERCES_VERSION not found, probably the software installation failed') slicDir = os.environ['SLIC_DIR'] res = getSoftwareFolder(slicDir) if not res['OK']: self.log.error('Directory %s was not found in either the local area or shared area' % (slicDir)) return res mySoftwareRoot = res['Value'] ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = GetNewLDLibs(self.systemConfig, "slic", self.applicationVersion) #retrieve detector model from web detector_urls = self.ops.getValue('/SLICweb/SLICDetectorModels', ['']) if len(detector_urls[0]) < 1: self.log.error('Could not find in CS the URL for detector model') return S_ERROR('Could not find in CS the URL for detector model') if not os.path.exists(self.detectorModel + ".zip"): for detector_url in detector_urls: try: urllib.urlretrieve("%s%s" % (detector_url, self.detectorModel + ".zip"), self.detectorModel + ".zip") except: self.log.error("Download of detector model failed") continue if not os.path.exists(self.detectorModel + ".zip"): self.log.error('Detector model %s was not found neither locally nor on the web, exiting' % self.detectorModel) return S_ERROR('Detector model %s was not found neither locally nor on the web, exiting' % self.detectorModel) try: unzip_file_into_dir(open(self.detectorModel + ".zip"), os.getcwd()) except: os.unlink(self.detectorModel + ".zip") self.log.error('Failed to unzip detector model') return S_ERROR('Failed to unzip detector model') #unzip detector model #self.unzip_file_into_dir(open(self.detectorModel+".zip"),os.getcwd()) slicmac = 'slicmac.mac' if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Generator file not found") return res self.InputFile = res['Value'] if len(self.SteeringFile) > 0: self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): res = getSteeringFileDirName(self.systemConfig, "slic", self.applicationVersion) if not res['OK']: self.log.error("Could not find where the steering files are") steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)): self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile) if not os.path.exists(self.SteeringFile): return S_ERROR("Could not find mac file") ##Same as for mokka: using ParticleGun does not imply InputFile if not len(self.InputFile): self.InputFile = [''] macok = PrepareMacFile(self.SteeringFile, slicmac, self.InputFile[0], self.NumberOfEvents, self.startFrom, self.detectorModel, self.randomseed, self.OutputFile, self.debug) if not macok['OK']: self.log.error('Failed to create SLIC mac file') return S_ERROR('Error when creating SLIC mac file') scriptName = 'SLIC_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/sh \n') script.write('#####################################################################\n') script.write('# Dynamically generated script to run a production or analysis job. #\n') script.write('#####################################################################\n') if os.environ.has_key('XERCES_VERSION'): script.write('declare -x XERCES_LIB_DIR=%s/packages/xerces/%s/lib\n' % (mySoftwareRoot, os.environ['XERCES_VERSION'])) if new_ld_lib_path: script.write('declare -x LD_LIBRARY_PATH=$XERCES_LIB_DIR:%s\n' % new_ld_lib_path) else: script.write('declare -x LD_LIBRARY_PATH=$XERCES_LIB_DIR\n') script.write('declare -x GEANT4_DATA_ROOT=%s/packages/geant4/data\n' % mySoftwareRoot) script.write('declare -x G4LEVELGAMMADATA=$(ls -d $GEANT4_DATA_ROOT/PhotonEvaporation*)\n') script.write('declare -x G4RADIOACTIVEDATA=$(ls -d $GEANT4_DATA_ROOT/RadioactiveDecay*)\n') script.write('declare -x G4LEDATA=$(ls -d $GEANT4_DATA_ROOT/G4EMLOW*)\n') script.write('declare -x G4NEUTRONHPDATA=$(ls -d $GEANT4_DATA_ROOT/G4NDL*)\n') script.write('declare -x GDML_SCHEMA_DIR=%s/packages/lcdd/%s\n' % (mySoftwareRoot, os.environ['LCDD_VERSION'])) script.write('declare -x PARTICLE_TBL=%s/packages/slic/%s/data/particle.tbl\n' % (mySoftwareRoot, os.environ['SLIC_VERSION'])) script.write('declare -x MALLOC_CHECK_=0\n') if os.path.exists("%s/lib" % (mySoftwareRoot)): script.write('declare -x LD_LIBRARY_PATH=%s/lib:$LD_LIBRARY_PATH\n' % (mySoftwareRoot)) script.write('echo =========\n') script.write('env | sort >> localEnv.log\n') script.write('echo =========\n') comm = '%s/packages/slic/%s/bin/Linux-g++/slic -P $PARTICLE_TBL -m %s\n' % (mySoftwareRoot, os.environ['SLIC_VERSION'], slicmac) print comm script.write(comm) script.write('declare -x appstatus=$?\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'sh -c "./%s"' % scriptName self.setApplicationStatus('SLIC %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} resultTuple = self.result['Value'] if not os.path.exists(self.applicationLog): self.log.error("Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log' % (self.applicationName)) status = resultTuple[0] # stdOutput = resultTuple[1] # stdError = resultTuple[2] self.log.info( "Status after the application execution is %s" % str( status ) ) return self.finalStatusReport(status)
def execute(self): """ Execute the module, called by JobAgent """ # Get input variables self.result = self.resolveInputVariables() # Checks if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result["Message"]) return self.result if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Cannot find input file") self.setApplicationStatus('StdHepSplit: missing input stdhep file') return S_ERROR('Missing stdhep file!') runonstdhep = res['Value'][0] else: self.log.warn("No files found to split") return S_OK("No files found to process") # removeLibc #removeLibc( os.path.join( os.environ["LCIO"], "lib" ) ) prefix = '' if self.OutputFile: prefix = self.OutputFile.split('.stdhep')[0] else: prefix = "this_split" #because we need to make sure the files end up in the base directory at the end self.OutputFile = prefix self.log.info("Will rename all files using '%s' as base." % prefix) # Setting up script res = getSoftwareFolder(self.platform, "stdhepsplit", self.applicationVersion) if not res['OK']: self.log.error("Failed to find the software") self.setApplicationStatus('StdHepSplit: Could not find neither local area not shared area install') return res mysplitDir = res['Value'] new_ld_lib = getNewLDLibs(self.platform, "stdhepsplit", self.applicationVersion) LD_LIBRARY_PATH = os.path.join(mysplitDir, "lib") + ":" + new_ld_lib scriptContent = """ #!/bin/sh ################################################################################ # Dynamically generated script by LCIOConcatenate module # ################################################################################ declare -x LD_LIBRARY_PATH=%s %s/hepsplit --infile %s --nw_per_file %s --outpref %s exit $? """ % ( LD_LIBRARY_PATH, mysplitDir, runonstdhep, self.nbEventsPerSlice, prefix ) # Write script to file scriptPath = 'StdHepSplit_%s_Run_%s.tcl' % ( self.applicationVersion, self.STEP_NUMBER ) if os.path.exists(scriptPath): os.remove(scriptPath) script = open( scriptPath, 'w' ) script.write( scriptContent ) script.close() # Setup log file for application stdout if os.path.exists(self.applicationLog): os.remove(self.applicationLog) # Run code os.chmod( scriptPath, 0755 ) command = '"./%s"' % ( scriptPath ) self.setApplicationStatus( 'StdHepSplit %s step %s' % ( self.applicationVersion, self.STEP_NUMBER ) ) self.stdError = '' self.result = shellCall( 0, command, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520 ) # Check results resultTuple = self.result['Value'] status = resultTuple[0] if not os.path.exists(self.applicationLog): self.log.error("Cannot access log file, cannot proceed") return S_ERROR("Failed reading the log file") with open(self.applicationLog, "r") as logf: numberofeventsdict = {} fname = '' for line in logf: line = line.rstrip() if line.count('Open output file'): fname = line.split()[-1].rstrip().rstrip("\0") numberofeventsdict[fname] = 0 elif line.count("Record") and not line.count('Output Begin Run') : #print line val = line.split("=")[1].rstrip().lstrip() if val != '0': numberofeventsdict[fname] = int(val) self.log.verbose("numberofeventsdict dict: %s" % numberofeventsdict) ##Now update the workflow_commons dict with the relation between filename and number of events: #needed for the registerOutputData self.workflow_commons['file_number_of_event_relation'] = numberofeventsdict if self.listoutput: outputlist = [] for of in numberofeventsdict.keys(): item = {} item['outputFile'] = of item['outputPath'] = self.listoutput['outputPath'] item['outputDataSE'] = self.listoutput['outputDataSE'] outputlist.append(item) self.step_commons['listoutput'] = outputlist #Not only the step_commons must be updated if self.workflow_commons.has_key('ProductionOutputData'): proddata = self.workflow_commons['ProductionOutputData'].split(";") finalproddata = [] this_split_data = '' for item in proddata: if not item.count(prefix): finalproddata.append(item) else: this_split_data = item path = os.path.dirname(this_split_data) for of in numberofeventsdict.keys(): finalproddata.append(os.path.join(path, of)) self.workflow_commons['ProductionOutputData'] = ";".join(finalproddata) self.log.info( "Status after the application execution is %s" % str( status ) ) if status == 2: self.log.info("Reached end of input file") status = 0 self.listDir() return self.finalStatusReport(status)
def runIt(self): """ Called by JobAgent Execute the following: - get the environment variables that should have been set during installation - download the detector model, using CS query to fetch the address - prepare the mac file using :any:`prepareMacFile` - run SLIC on this mac File and catch the exit status :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: LOG.error("Failed to resolve input parameters:", self.result['Message']) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('SLIC should not proceed as previous step did not end properly') res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript) if not res['OK']: LOG.error("Could not obtain the environment script: ", res["Message"]) return res env_script_path = res["Value"] retMod = self.getDetectorModel() if not retMod: return retMod slicmac = 'slicmac.mac' if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: LOG.error("Generator file not found") return res self.InputFile = res['Value'] if len(self.SteeringFile) > 0: self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: LOG.error("Could not find where the steering files are") return res steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)): self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile) if not os.path.exists(self.SteeringFile): LOG.error("Missing steering file") return S_ERROR("Could not find mac file") ##Same as for mokka: using ParticleGun does not imply InputFile if not len(self.InputFile): self.InputFile = [''] macok = prepareMacFile(self.SteeringFile, slicmac, self.InputFile[0], self.NumberOfEvents, self.startFrom, self.detectorModel, self.RandomSeed, self.OutputFile, self.debug) if not macok['OK']: LOG.error('Failed to create SLIC mac file') return S_ERROR('Error when creating SLIC mac file') scriptName = 'SLIC_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/bash \n') script.write('#####################################################################\n') script.write('# Dynamically generated script to run a production or analysis job. #\n') script.write('#####################################################################\n') script.write("source %s\n" % (env_script_path)) preloadDir = "%s/preload" % os.getcwd() if os.path.exists( preloadDir ): preadloadLibs = set( os.listdir( preloadDir ) ) preloadString =":".join( lib for lib in preadloadLibs if ".so" in lib ) script.write( 'declare -x LD_LIBRARY_PATH=%s:${LD_LIBRARY_PATH}\n' % preloadDir ) script.write( 'declare -x LD_PRELOAD=%s\n' % preloadString ) script.write('echo =========\n') script.write('env | sort >> localEnv.log\n') script.write('echo SLIC:\n') script.write("which slic\n") script.write('echo =========\n') comm = 'slic -P $PARTICLE_TBL -m %s %s\n' % (slicmac, self.extraCLIarguments) LOG.info("Command:", comm) script.write(comm) script.write('declare -x appstatus=$?\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0o755) comm = 'sh -c "./%s"' % scriptName self.setApplicationStatus('SLIC %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} resultTuple = self.result['Value'] if not os.path.exists(self.applicationLog): LOG.error("Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log' % (self.applicationName)) status = resultTuple[0] # stdOutput = resultTuple[1] # stdError = resultTuple[2] LOG.info("Status after the application execution is %s" % str(status)) return self.finalStatusReport(status)
def execute(self): """ Execute the module, called by JobAgent """ # Get input variables self.result = self.resolveInputVariables() # Checks if not self.platform: self.result = S_ERROR('No ILC platform selected') if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result["Message"]) return self.result if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Cannot find input file") self.setApplicationStatus( 'StdHepSplit: missing input stdhep file') return S_ERROR('Missing stdhep file!') runonstdhep = res['Value'][0] else: self.log.warn("No files found to split") return S_OK("No files found to process") prefix = '' if self.OutputFile: prefix = self.OutputFile.split('.stdhep')[0] else: prefix = "this_split" #because we need to make sure the files end up in the base directory at the end self.OutputFile = prefix self.log.info("Will rename all files using '%s' as base." % prefix) # Setting up script res = getSoftwareFolder(self.platform, "stdhepsplit", self.applicationVersion) if not res['OK']: self.log.error("Failed to find the software") self.setApplicationStatus( 'StdHepSplit: Could not find neither local area not shared area install' ) return res mysplitDir = res['Value'] new_ld_lib = getNewLDLibs(self.platform, "stdhepsplit", self.applicationVersion) LD_LIBRARY_PATH = os.path.join(mysplitDir, "lib") + ":" + new_ld_lib maxReadString = '' ## hepsplit has a off-by-one error so we add 1 to max read to actually read maxRead events if self.maxRead: self.maxRead += 1 maxReadString = " --maxread %s" % self.maxRead scriptContent = """ #!/bin/sh ################################################################################ # Dynamically generated script by LCIOConcatenate module # ################################################################################ declare -x LD_LIBRARY_PATH=%s %s/hepsplit --infile %s --nw_per_file %s --outpref %s%s exit $? """ % ( LD_LIBRARY_PATH, mysplitDir, runonstdhep, self.nbEventsPerSlice, prefix, maxReadString, ) # Write script to file scriptPath = 'StdHepSplit_%s_Run_%s.tcl' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptPath): os.remove(scriptPath) script = open(scriptPath, 'w') script.write(scriptContent) script.close() # Setup log file for application stdout if os.path.exists(self.applicationLog): os.remove(self.applicationLog) # Run code os.chmod(scriptPath, 0755) command = '"./%s"' % (scriptPath) self.setApplicationStatus('StdHepSplit %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, command, callbackFunction=self.redirectLogOutput, bufferLimit=20971520) resultTuple = self.result['Value'] status = resultTuple[0] if not os.path.exists(self.applicationLog): self.log.error("Cannot access log file, cannot proceed") return S_ERROR("Failed reading the log file") with open(self.applicationLog, "r") as logf: numberofeventsdict = {} fname = '' for line in logf: line = line.rstrip() if line.count('Open output file'): fname = line.split()[-1].rstrip().rstrip("\0") numberofeventsdict[fname] = 0 elif line.count( "Record") and not line.count('Output Begin Run'): #print line val = line.split("=")[1].rstrip().lstrip() if val != '0': numberofeventsdict[fname] = int(val) self.log.verbose("numberofeventsdict dict: %s" % numberofeventsdict) ##Now update the workflow_commons dict with the relation between filename and number of events: #needed for the registerOutputData self.workflow_commons[ 'file_number_of_event_relation'] = numberofeventsdict if self.listoutput: outputlist = [] for of in numberofeventsdict: item = {} item['outputFile'] = of item['outputPath'] = self.listoutput['outputPath'] item['outputDataSE'] = self.listoutput['outputDataSE'] outputlist.append(item) self.step_commons['listoutput'] = outputlist #Not only the step_commons must be updated if 'ProductionOutputData' in self.workflow_commons: proddata = self.workflow_commons['ProductionOutputData'].split(";") finalproddata = [] this_split_data = '' for item in proddata: if not item.count(prefix): finalproddata.append(item) else: this_split_data = item path = os.path.dirname(this_split_data) for of in numberofeventsdict: finalproddata.append(os.path.join(path, of)) self.workflow_commons['ProductionOutputData'] = ";".join( finalproddata) self.log.info("Status after the application execution is %s" % str(status)) if status == 2: self.log.info("Reached end of input file") status = 0 self.listDir() return self.finalStatusReport(status)
def runIt(self): """ Called from ModuleBase """ self.result = S_OK() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('SLIC Pandora should not proceed as previous step did not end properly') ###Get the env script res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript) if not res['OK']: self.log.error("Failed to get the environment script:", res["Message"]) return res env_script_path = res["Value"] res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error('Could not find input files') self.setApplicationStatus('SLICPandora: missing slcio file') return S_ERROR('Missing slcio file!') runonslcio = res['Value'][0] if not self.detectorxml.count(".xml") or not os.path.exists(os.path.basename(self.detectorxml)): detmodel = self.detectorxml.replace("_pandora.xml", "") if os.path.exists(detmodel + ".zip"): try: unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd()) except (RuntimeError, OSError) as err: self.log.error("Exception when unpacking detectormodel zip file:", str(err)) os.unlink(detmodel + ".zip") if not os.path.exists(detmodel + ".zip"): #retrieve detector model from web detector_urls = self.ops.getValue('/SLICweb/SLICDetectorModels', ['']) if len(detector_urls[0]) < 1: self.log.error('Could not find in CS the URL for detector model') return S_ERROR('Could not find in CS the URL for detector model') for detector_url in detector_urls: try: urllib.urlretrieve("%s%s"%(detector_url, detmodel + ".zip"), detmodel + ".zip") except IOError: self.log.error("Download of detector model failed") continue try: unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd()) break except (RuntimeError, OSError) as err: self.log.error("Exception for zip file obtained from ", detector_url) self.log.error("Exception:", str(err)) os.unlink(detmodel + ".zip") continue #if os.path.exists(detmodel): #and os.path.isdir(detmodel): self.detectorxml = os.path.join(os.getcwd(), self.detectorxml) self.detectorxml = self.detectorxml + "_pandora.xml" if not os.path.exists(self.detectorxml): self.log.error('Detector model xml %s was not found, exiting' % self.detectorxml) return S_ERROR('Detector model xml was not found, exiting') oldversion = False if self.applicationVersion in ['CLIC_CDR', 'CDR1', 'CDR2', 'CDR0', 'V2', 'V3', 'V4']: oldversion = True scriptName = 'SLICPandora_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = open(scriptName, 'w') script.write('#!/bin/bash \n') script.write('#####################################################################\n') script.write('# Dynamically generated script to run a production or analysis job. #\n') script.write('#####################################################################\n') script.write("source %s\n" % env_script_path) script.write("declare -x file=./Settings/%s\n" % self.pandorasettings) script.write(""" if [ -e "${file}" ] then declare -x PANDORASETTINGS=$file else if [ -d "${PANDORASETTINGSDIR}" ] then cp $PANDORASETTINGSDIR/*.xml . declare -x PANDORASETTINGS=%s fi fi if [ ! -e "${PANDORASETTINGS}" ] then echo "Missing PandoraSettings file" exit 1 fi """ % self.pandorasettings ) script.write('echo =============================\n') script.write('echo PATH is \n') script.write('echo $PATH | tr ":" "\n" \n') script.write('echo ==============\n') if os.path.exists("./lib"): script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n') script.write('echo =============================\n') script.write('echo LD_LIBRARY_PATH is \n') script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n') script.write('echo ============================= \n') script.write('env | sort >> localEnv.log\n') #Now run it if oldversion: comm = 'PandoraFrontend %s $PANDORASETTINGS %s %s %s' % (self.detectorxml, runonslcio, self.OutputFile, str(self.NumberOfEvents)) else: comm = 'PandoraFrontend -g %s -c $PANDORASETTINGS -i %s -o %s -r %s' % (self.detectorxml, runonslcio, self.OutputFile, str(self.NumberOfEvents)) comm = "%s %s\n" % (comm, self.extraCLIarguments) self.log.info("Will run %s" % comm) script.write(comm) script.write('declare -x appstatus=$?\n') #script.write('where\n') #script.write('quit\n') #script.write('EOF\n') script.write('exit $appstatus\n') script.close() if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'sh -c "./%s"' % (scriptName) self.setApplicationStatus('SLICPandora %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520) #self.result = {'OK':True,'Value':(0,'Disabled Execution','')} resultTuple = self.result['Value'] if resultTuple[0]: self.log.error("There was an error during the execution") if not os.path.exists(self.applicationLog): self.log.error("Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log' % (self.applicationName)) logf = open(self.applicationLog,'r') if "Missing PandoraSettings file" in logf.readlines()[-1]: self.log.error("Issue with the Pandora Settings file.") logf.close() status = resultTuple[0] # stdOutput = resultTuple[1] # stdError = resultTuple[2] self.log.info( "Status after the application execution is %s" % str( status ) ) return self.finalStatusReport(status)
def runIt(self): """ Called by JobAgent Execute the following: - prepend in the LD_LIBRARY_PATH any lib directory of any dependency (e.g. root) - prepare the list of files to run on - set the cacheDirectory and put in there the alias.properties - set the lcsim file using :any:`prepareLCSIMFile` - run java and catch the exit code :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR('No ILC platform selected') elif not self.applicationLog: self.result = S_ERROR('No Log file provided') if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result["Message"]) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose( 'Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK( 'LCSIM should not proceed as previous step did not end properly' ) res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error( 'LCSIM was not found in either the local area or shared area:', res['Message']) return res lcsim_name = res['Value'] ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName, self.applicationVersion) runonslcio = [] if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.setApplicationStatus('LCSIM: missing input slcio file') return S_ERROR('Missing slcio file!') runonslcio = res['Value'] #for inputfile in inputfilelist: # self.log.verbose("Will try using %s"%(os.path.basename(inputfile))) # runonslcio.append(os.path.join(os.getcwd(),os.path.basename(inputfile))) retMod = self.downloadDetectorZip() if not retMod: return retMod ##Collect jar files to put in classspath jars = [] if os.path.exists("lib"): for libs in os.listdir("lib"): if os.path.basename(libs).find(".jar") > 0: jars.append(os.path.abspath(os.path.join("lib", libs))) new_ld_lib_path = "./lib:%s" % new_ld_lib_path #Remove any libc remaining in .lib removeLibc("./lib") ###Define cache directory as local folder aliasproperties = os.path.basename(self.aliasproperties) cachedir = os.getcwd() if not os.path.isdir(os.path.join(cachedir, ".lcsim")): try: os.mkdir(os.path.join(cachedir, ".lcsim")) except OSError, x: self.log.error("Could not create .lcsim folder !", str(x))
def execute(self): """ Run the module """ result = self.resolveInputVariables() if not result['OK']: return result self.result = S_OK() if not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('%s should not proceed as previous step did not end properly'% self.applicationName) ### Now remove the files in the FC lfns = self.InputFile ##Check that all the files are here: res = resolveIFpaths(lfns) if not res['OK']: LOG.error(res['Message']) return S_ERROR("Failed to find a file locally") #All files are here and available paths = res['Value'] localpaths = [] for inputfile in paths: basename = os.path.basename(inputfile) locname = os.path.join(os.getcwd(),basename) if not locname == inputfile: try: shutil.copy(inputfile, locname) except shutil.Error: LOG.error("Failed to copy file locally, will have to stop") return S_ERROR("Failed copy to local directory") localpaths.append(locname) try: os.unlink(inputfile) except OSError: LOG.warn("Failed to remove initial file, increased \ disk space usage") #all the files are in the run directory #get all metadata, ancestor/daughter relations, etc. for all the files #Update the listoutput if self.listoutput: outputlist = [] for localFile in localpaths: item = {} item['outputFile'] = localFile item['outputPath'] = self.listoutput['outputPath'] item['outputDataSE'] = self.listoutput['outputDataSE'] outputlist.append(item) if self.enable: self.step_commons['listoutput'] = outputlist else: LOG.info("listoutput would have been ", outputlist) ## Make sure the path contains / at the end as we are going to ## concatenate final path and local files if not self.outputpath[-1]=='/': self.outputpath += "/" if 'ProductionOutputData' in self.workflow_commons: file_list = ";".join([self.outputpath+name for name in [os.path.basename(fin) for fin in localpaths]]) if self.enable: self.workflow_commons['ProductionOutputData'] = file_list else: LOG.info("ProductionOutputData would have been", file_list) #Now remove them if self.enable: res = self.repMan.removeFile(lfns, force=True) if not res['OK']: LOG.error("Failed to remove the files") self.setApplicationStatus("Failed to remove the file") #return S_ERROR("Failed to remove the files") else: LOG.info("Would have removed: ", "%s" % str(lfns)) ## Now the files are not on the storage anymore, they exist only locally. We can hope ## that the job will not be killed between now and the time the UploadOutputData module ## is called return self.finalStatusReport(0)
def execute(self): """ Execute the module, called by JobAgent """ # Get input variables self.result = self.resolveInputVariables() # Checks if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) if not self.result['OK']: return self.result if not os.environ.has_key("LCIO"): self.log.error("Environment variable LCIO was not defined, cannot do anything") return S_ERROR("Environment variable LCIO was not defined, cannot do anything") if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.setApplicationStatus('LCIOSplit: missing input slcio file') return S_ERROR('Missing slcio file!') runonslcio = res['Value'][0] else: return S_OK("No files found to process") # removeLibc removeLibc( os.path.join( os.environ["LCIO"], "lib" ) ) # Setting up script LD_LIBRARY_PATH = os.path.join( "$LCIO", "lib" ) if os.environ.has_key('LD_LIBRARY_PATH'): LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH'] PATH = "$LCIO/bin" if os.environ.has_key('PATH'): PATH += ":" + os.environ['PATH'] scriptContent = """ #!/bin/sh ################################################################################ # Dynamically generated script by LCIOConcatenate module # ################################################################################ declare -x LD_LIBRARY_PATH=%s declare -x PATH=%s lcio split -i %s -n %s exit $? """ % ( LD_LIBRARY_PATH, PATH, runonslcio, self.nbEventsPerSlice ) # Write script to file scriptPath = 'LCIOSplit_%s_Run_%s.tcl' % ( self.applicationVersion, self.STEP_NUMBER ) if os.path.exists(scriptPath): os.remove(scriptPath) script = open( scriptPath, 'w' ) script.write( scriptContent ) script.close() # Setup log file for application stdout if os.path.exists(self.applicationLog): os.remove(self.applicationLog) # Run code os.chmod( scriptPath, 0755 ) command = '"./%s"' % ( scriptPath ) self.setApplicationStatus( 'LCIOSplit %s step %s' % ( self.applicationVersion, self.STEP_NUMBER ) ) self.stdError = '' self.result = shellCall( 0, command, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520 ) # Check results resultTuple = self.result['Value'] status = resultTuple[0] if not os.path.exists(self.applicationLog): self.log.error("Cannot access log file, cannot proceed") return S_ERROR("Failed reading the log file") logf = file(self.applicationLog,"r") baseinputfilename = os.path.basename(runonslcio).split(".slcio")[0] output_file_base_name = '' if self.OutputFile: output_file_base_name = self.OutputFile.split('.slcio')[0] self.log.info("Will rename all files using '%s' as base." % output_file_base_name) numberofeventsdict = {} fname = '' for line in logf: line = line.rstrip() if line.count(baseinputfilename): #First, we need to rename those guys current_file = os.path.basename(line).replace(".slcio", "") current_file_extension = current_file.replace(baseinputfilename, "") newfile = output_file_base_name + current_file_extension + ".slcio" os.rename(line, newfile) fname = newfile numberofeventsdict[fname] = 0 elif line.count("events"): numberofeventsdict[fname] = int(line.split()[0]) self.log.verbose("numberofeventsdict dict: %s" % numberofeventsdict) ##Now update the workflow_commons dict with the relation between filename and number of events: needed for #the registerOutputData self.workflow_commons['file_number_of_event_relation'] = numberofeventsdict if self.listoutput: outputlist = [] for f in numberofeventsdict.keys(): item = {} item['outputFile'] = f item['outputPath'] = self.listoutput['outputPath'] item['outputDataSE'] = self.listoutput['outputDataSE'] outputlist.append(item) self.step_commons['listoutput'] = outputlist #Not only the step_commons must be updated if self.workflow_commons.has_key('ProductionOutputData'): proddata = self.workflow_commons['ProductionOutputData'].split(";") finalproddata = [] this_split_data = '' for item in proddata: if not item.count(output_file_base_name): finalproddata.append(item) else: this_split_data = item path = os.path.dirname(this_split_data) for f in numberofeventsdict.keys(): finalproddata.append(os.path.join(path, f)) self.workflow_commons['ProductionOutputData'] = ";".join(finalproddata) self.log.info( "Status after the application execution is %s" % str( status ) ) return self.finalStatusReport(status)
def runIt(self): """ Called by JobAgent Execute the following: - get the environment variables that should have been set during installation - find the detector model xml, using CS query to obtain the path - prepare the steering file and command line parameters - run DDSim on this steering file and catch the exit status :return: S_OK(), S_ERROR() """ self.result = S_OK() if not self.platform: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: self.log.error("Failed to resolve input parameters:", self.result['Message']) return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('DDSim should not proceed as previous step did not end properly') ##TODO: Setup LD_LIBRARY_PATH for extensions res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript) if not res['OK']: self.log.error("Could not obtain the environment script: ", res["Message"]) return res envScriptPath = res["Value"] #get the path to the detector model, either local or from the software resXML = self._getDetectorXML() if not resXML['OK']: self.log.error("Could not obtain the detector XML file: ", resXML["Message"]) return resXML compactFile = resXML['Value'] if len(self.InputFile): res = resolveIFpaths(self.InputFile) if not res['OK']: self.log.error("Generator file not found") return res self.InputFile = res['Value'] ## if steering file is set try to find it if len(self.SteeringFile) > 0: self.SteeringFile = os.path.basename(self.SteeringFile) if not os.path.exists(self.SteeringFile): res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion) if not res['OK']: self.log.error("Could not find where the steering files are") return res steeringfiledirname = res['Value'] if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)): self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile) if not os.path.exists(self.SteeringFile): self.log.error("Missing steering file") return S_ERROR("Could not find steering file") self.extraCLIarguments += " --steeringFile %s " % self.SteeringFile if self.startFrom: self.extraCLIarguments += " --skipNEvents %s " % self.startFrom if self.debug: self.extraCLIarguments += " --printLevel DEBUG " ##Same as for mokka: using ParticleGun does not imply InputFile if self.InputFile: self.InputFile = [self.InputFile] if isinstance(self.InputFile, basestring) else self.InputFile self.extraCLIarguments += " --inputFile %s " % self.InputFile[0] if self.NumberOfEvents: self.extraCLIarguments += " --numberOfEvents %s " % self.NumberOfEvents self.extraCLIarguments += " --random.seed %s " % self.randomSeed if self.OutputFile: self.extraCLIarguments += " --outputFile %s " % self.OutputFile scriptName = 'DDSim_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER) if os.path.exists(scriptName): os.remove(scriptName) script = [] script.append('#!/bin/bash') script.append('#####################################################################') script.append('# Dynamically generated script to run a production or analysis job. #') script.append('#####################################################################') script.append('source %s' % envScriptPath) script.append('echo =========') script.append('env | sort >> localEnv.log') script.append('echo ddsim:`which ddsim`') script.append('echo =========') comm = 'ddsim --compactFile %(compactFile)s %(extraArgs)s' % dict(compactFile=compactFile, extraArgs=self.extraCLIarguments) self.log.info("Command:", comm) script.append(comm) script.append('declare -x appstatus=$?') script.append('exit $appstatus') with open(scriptName, 'w') as scriptFile: scriptFile.write( "\n".join(script) ) if os.path.exists(self.applicationLog): os.remove(self.applicationLog) os.chmod(scriptName, 0755) comm = 'bash "./%s"' % scriptName self.setApplicationStatus('DDSim %s step %s' % (self.applicationVersion, self.STEP_NUMBER)) self.stdError = '' self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520) resultTuple = self.result['Value'] if not os.path.exists(self.applicationLog): self.log.error("Something went terribly wrong, the log file is not present") self.setApplicationStatus('%s failed to produce log file' % (self.applicationName)) if not self.ignoreapperrors: return S_ERROR('%s did not produce the expected log %s' % (self.applicationName, self.applicationLog)) status = resultTuple[0] self.log.info( "Status after the application execution is %s" % status ) return self.finalStatusReport(status)
def execute(self): """ Called from Workflow """ self.result = self.resolveInputVariables() if not self.systemConfig: self.result = S_ERROR( 'No ILC platform selected' ) elif not self.applicationLog: self.result = S_ERROR( 'No Log file provided' ) if not self.result['OK']: return self.result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('SLIC Pandora should not proceed as previous step did not end properly') slicPandoraDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig, "slicpandora", self.applicationVersion), '') slicPandoraDir = slicPandoraDir.replace(".tgz", "").replace(".tar.gz", "") res = getSoftwareFolder(slicPandoraDir) if not res['OK']: self.setApplicationStatus('SLICPandora: Could not find neither local area not shared area install') return res myslicPandoraDir = res['Value'] ##Remove libc lib removeLibc(myslicPandoraDir + "/LDLibs") ##Need to fetch the new LD_LIBRARY_PATH new_ld_lib_path = GetNewLDLibs(self.systemConfig, "slicpandora", self.applicationVersion) new_path = GetNewPATH(self.systemConfig, "slicpandora", self.applicationVersion) res = resolveIFpaths(self.InputFile) if not res['OK']: self.setApplicationStatus('SLICPandora: missing slcio file') return S_ERROR('Missing slcio file!') runonslcio = res['Value'][0] if not self.detectorxml.count(".xml") or not os.path.exists(os.path.basename(self.detectorxml)): detmodel = self.detectorxml.replace("_pandora.xml", "") if os.path.exists(detmodel + ".zip"): try: unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd()) except: os.unlink(detmodel + ".zip") if not os.path.exists(detmodel + ".zip"): #retrieve detector model from web detector_urls = self.ops.getValue('/SLICweb/SLICDetectorModels', ['']) if len(detector_urls[0]) < 1: self.log.error('Could not find in CS the URL for detector model') return S_ERROR('Could not find in CS the URL for detector model') for detector_url in detector_urls: try: urllib.urlretrieve("%s%s"%(detector_url, detmodel + ".zip"), detmodel + ".zip") except: self.log.error("Download of detector model failed") continue try: unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd()) break except: os.unlink(detmodel + ".zip") continue #if os.path.exists(detmodel): #and os.path.isdir(detmodel): self.detectorxml = os.path.join(os.getcwd(), self.detectorxml) self.detectorxml = self.detectorxml + "_pandora.xml" if not os.path.exists(self.detectorxml): self.log.error('Detector model xml %s was not found, exiting' % self.detectorxml) return S_ERROR('Detector model xml %s was not found, exiting' % self.detectorxml) if not os.path.exists(self.pandorasettings): if os.path.exists("./Settings/%s" % self.pandorasettings): self.pandorasettings = "./Settings/%s" % self.pandorasettings elif os.path.exists(os.path.join(myslicPandoraDir, 'Settings')): xmllist = glob.glob(os.path.join(myslicPandoraDir, 'Settings', "*.xml")) for f in xmllist: try: shutil.copy(f, os.path.join(os.getcwd(), os.path.basename(f))) except Exception, x: self.log.error('Could not copy %s, exception: %s' % (f, str(x))) return S_ERROR('Could not copy PandoraSettings file') else: self.log.error("Failed to find PandoraSettings anywhere, possibly SLICPandora install broken") return S_ERROR("Failed to find PandoraSettings anywhere")