def test_resolvepaths(self):
   """test ResolvePathsAndNames resolvePaths......................................................."""
   res = resolveIFpaths(self.inputfiles)
   self.assertTrue('OK' in res)
   self.assertEqual(res['OK'], True, res)
   self.assertTrue('Value' in res, res.keys())
   self.assertEqual(res['Value'], [os.path.abspath(self.realloc)])
 def test_resolvepaths(self):
   """test ResolvePathsAndNames resolvePaths......................................................."""
   res = resolveIFpaths(self.inputfiles)
   self.assertTrue('OK' in res)
   self.assertEqual(res['OK'], True, res)
   self.assertTrue('Value' in res, res.keys() )
   self.assertEqual(res['Value'], [ os.path.abspath(self.realloc) ])
Exemplo n.º 3
0
  def GetInputFiles(self):
    """ Resolve the input files. But not if in the application definition it was decided
    that it should forget about the input.
    """
    if self.ignoremissingInput:
      return S_OK("")
    res = resolveIFpaths(self.InputFile)
    if not res['OK']:
      self.setApplicationStatus('%s: missing slcio file' % self.applicationName)
      return S_ERROR('Missing slcio file!')
    runonslcio = res['Value']

    listofslcio = string.join(runonslcio, " ")
    
    return S_OK(listofslcio)
Exemplo n.º 4
0
  def _getInputFiles(self):
    """ Resolve the input files. But not if in the application definition it was decided
    that it should forget about the input.
    """
    if self.ignoremissingInput:
      return S_OK("")
    res = resolveIFpaths(self.InputFile)
    if not res['OK']:
      self.setApplicationStatus('%s: missing slcio file' % self.applicationName)
      return S_ERROR('Missing slcio file!')
    runonslcio = res['Value']

    listofslcio = " ".join(runonslcio)
    
    return S_OK(listofslcio)
Exemplo n.º 5
0
  def runIt(self):
    """
    Called by JobAgent
    
    Execute the following:
      - prepend in the LD_LIBRARY_PATH any lib directory of any dependency (e.g. root)
      - prepare the list of files to run on
      - set the cacheDirectory and put in there the alias.properties
      - set the lcsim file using :any:`prepareLCSIMFile`
      - run java and catch the exit code

    :return: S_OK(), S_ERROR()
    """
    self.result = S_OK()
    if not self.platform:
      self.result = S_ERROR( 'No ILC platform selected' )
    elif not self.applicationLog:
      self.result = S_ERROR( 'No Log file provided' )
    if not self.result['OK']:
      LOG.error("Failed to resolve input parameters:", self.result["Message"])
      return self.result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('LCSIM should not proceed as previous step did not end properly')
    
    
    res = getSoftwareFolder(self.platform, self.applicationName, self.applicationVersion)
    if not res['OK']:
      LOG.error('LCSIM was not found in either the local area or shared area:', res['Message'])
      return res
    lcsim_name = res['Value']
    ##Need to fetch the new LD_LIBRARY_PATH
    new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName, self.applicationVersion)

    runonslcio = []
    if len(self.InputFile):
      res = resolveIFpaths(self.InputFile)
      if not res['OK']:
        self.setApplicationStatus('LCSIM: missing input slcio file')
        return S_ERROR('Missing slcio file!')
      runonslcio = res['Value']
    #for inputfile in inputfilelist:
    #  LOG.verbose("Will try using %s"%(os.path.basename(inputfile)))
    #  runonslcio.append(os.path.join(os.getcwd(),os.path.basename(inputfile)))


    retMod = self.downloadDetectorZip()
    if not retMod:
      return retMod

    ##Collect jar files to put in classspath
    jars = []
    if os.path.exists("lib"):
      for libs in os.listdir("lib"):
        if os.path.basename(libs).find(".jar") > 0:
          jars.append(os.path.abspath(os.path.join("lib", libs)))
      new_ld_lib_path = "./lib:%s" % new_ld_lib_path
      #Remove any libc remaining in .lib
      removeLibc("./lib")
    
    
    ###Define cache directory as local folder
    aliasproperties = os.path.basename(self.aliasproperties)
    cachedir = os.getcwd()
    if not os.path.isdir(os.path.join(cachedir, ".lcsim")):
      try:
        os.mkdir(os.path.join(cachedir, ".lcsim"))
      except OSError, x:
        LOG.error("Could not create .lcsim folder !", str(x))
Exemplo n.º 6
0
  def runIt(self):
    """ Called by ModuleBase
      
    Executes the following:
      - read the application parameters that where defined in ILCJob, and stored in the job definition
      - setup the SQL server and run it in the background
      - prepare the steering file using :any:`prepareSteeringFile`
      - run Mokka and catch its return status

    :return: S_OK(), S_ERROR()
    """
    
    #if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
    #  LOG.info('Skip this module, failure detected in a previous step :')
    #  LOG.info('Workflow status : %s' %(self.workflowStatus))
    #  LOG.info('Step Status %s' %(self.stepStatus))
    #  return S_OK()

    self.result = S_OK()
       
    if not self.platform:
      self.result = S_ERROR( 'No ILC platform selected' )
    elif not self.applicationLog:
      self.result = S_ERROR( 'No Log file provided' )

    if not self.result['OK']:
      LOG.error("Failed to resolve the input parameters:", self.result["Message"])
      return self.result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('Mokka should not proceed as previous step did not end properly')

    LOG.info("Executing Mokka %s" % (self.applicationVersion))

    if self.dbSlice:
      if self.dbSlice.lower().startswith("lfn"):
        self.dbSlice = os.path.basename(self.dbSlice)
      self.db_dump_name = self.dbSlice
    else:
      self.db_dump_name = "CLICMokkaDB.sql"

    
    res = getEnvironmentScript(self.platform, "mokka", self.applicationVersion, self.getEnvScript)
    LOG.notice("Got the environment script: %s" % res)
    if not res['OK']:
      LOG.error("Error getting the env script: ", res['Message'])
      return res
    env_script_path = res['Value']
    

    ####Setup MySQL instance      
    mysqlBasePath = '%s/mysqltmp/MokkaDBRoot-%s' %(os.getcwd(), generateRandomString(8))
    LOG.notice("Placing mysql files in %s" % mysqlBasePath)
    
    ###steering file that will be used to run
    mokkasteer = "mokka.steer"
    if os.path.exists("mokka.steer"):
      try:
        os.rename("mokka.steer", "mymokka.steer")
      except EnvironmentError as err:
        LOG.error("Failed renaming the steering file: ", str(err))
      self.SteeringFile = "mymokka.steer"
        
    ###prepare steering file
    #first, I need to take the stdhep file, find its path (possible LFN)      
    if len(self.InputFile) > 0:
      #self.InputFile = os.path.basename(self.InputFile)
      res = resolveIFpaths(self.InputFile)
      if not res['OK']:
        LOG.error("Generator file not found")
        return res
      self.InputFile = res['Value']
    if len(self.macFile) > 0:
      self.macFile = os.path.basename(self.macFile)
    ##idem for steering file
      
    self.SteeringFile = os.path.basename(self.SteeringFile)
    if not os.path.exists(self.SteeringFile):
      LOG.verbose("Steering file %s not found locally" % self.SteeringFile)
      res =  getSteeringFileDirName(self.platform, "mokka", self.applicationVersion)
      if not res['OK']:
        LOG.error("Missing Steering file directory:", res['Message'])
        return res
      steeringfiledirname = res['Value']
      if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
        try:
          shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile )
        except EnvironmentError as err:
          LOG.error("Failed copying file", self.SteeringFile)
          return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(err)))
          #self.steeringFile = os.path.join(mySoftwareRoot,"steeringfiles",self.steeringFile)
    if not os.path.exists(self.SteeringFile):
      LOG.error("Missing steering file, should not happen!")
      return S_ERROR("Could not find steering file")
    else:
      LOG.verbose("Found local copy of %s" % self.SteeringFile)
    ### The following is because if someone uses particle gun, there is no InputFile
    if not len(self.InputFile):
      self.InputFile = ['']
    steerok = prepareSteeringFile(self.SteeringFile, mokkasteer, self.detectorModel, self.InputFile[0],
                                  self.macFile, self.NumberOfEvents, self.startFrom, self.RandomSeed,
                                  self.mcRunNumber,
                                  self.ProcessID,
                                  self.debug,
                                  self.OutputFile,
                                  self.inputdataMeta)
    if not steerok['OK']:
      LOG.error('Failed to create MOKKA steering file')
      return S_ERROR('Failed to create MOKKA steering file')

    ###Extra option depending on mokka version
    mokkaextraoption = ""
    if self.applicationVersion not in ["v07-02", "v07-02fw", "v07-02fwhp", "MokkaRevision42", "MokkaRevision43", 
                                       "MokkaRevision44", "Revision45"]:
      mokkaextraoption = "-U"

    scriptName = 'Mokka_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)

    if os.path.exists(scriptName): 
      os.remove(scriptName)
    script = open(scriptName, 'w')
    script.write('#!/bin/bash \n')
    script.write('#####################################################################\n')
    script.write('# Dynamically generated script to run a production or analysis job. #\n')
    script.write('#####################################################################\n')
    script.write("source %s\n" % env_script_path)
    if os.path.exists("./lib"):
      script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n' )
    script.write('echo =============================\n')
    script.write('echo Content of mokka.steer:\n')
    script.write('cat mokka.steer\n')
    if self.macFile:
      script.write('echo =============================\n')
      script.write('echo Content of %s\n' % os.path.basename(self.macFile))
      script.write("cat %s\n" % os.path.basename(self.macFile))
    elif os.path.exists("./mokkamac.mac"):
      script.write("""
echo =============================
echo Content of mokkamac.mac:
cat mokkamac.mac
""")
    script.write('echo =============================\n')
    script.write('echo LD_LIBRARY_PATH is\n')
    script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n')
    script.write('echo =============================\n')
    script.write('echo PATH is\n')
    script.write('echo $PATH | tr ":" "\n"\n')
    if self.debug:
      script.write('echo ldd of Mokka is\n')
      script.write('ldd `which Mokka` \n' )
      script.write('echo =============================\n')
      script.write('echo ldd of mysql is\n')
      script.write('ldd `which mysql` \n' )
      script.write('echo =============================\n')
    script.write('env | sort >> localEnv.log\n')      
    script.write('echo =============================\n')
    script.write("""if [ ! -e ./%(DBDUMP)s ]; then
    echo "Missing DB dump!" >&2
    exit 1
fi    
    """ % {'DBDUMP': self.db_dump_name} )
    ##Now start the MySQL server and configure.
    script.write("declare -x MOKKADBROOT=%s\n" % mysqlBasePath)
    script.write("declare -x WORKDIR=%s\n" % os.getcwd())
    script.write("declare -x LOGDIR=$WORKDIR/mysqllogs/\n")
    script.write("mkdir -p $LOGDIR\n")

    #mysql socket must not be longer than 107 characters!
    script.write("declare -x SOCKETBASE=/tmp/mokka-%s\n" % generateRandomString(8) )
    script.write("mkdir -p $SOCKETBASE\n")
    script.write("declare -x SOCKETPATH=$SOCKETBASE/mysql.sock\n" )
    script.write("mkdir -p $MOKKADBROOT\n")
    script.write("declare -x MYSQLDATA=$MOKKADBROOT/data\n")
    script.write("rm -rf $MYSQLDATA\n")
    script.write("mkdir -p $MYSQLDATA\n")
    script.write("date\n")
    script.write("""echo "*** Installing MySQL"
echo "mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log"
mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log
install_st=$?
if [ $install_st -ne 0 ]
then
      exit $install_st
fi
date
echo "*** Running mysqld-safe"
cd $MYSQL
bin/mysqld_safe --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log &
COUNT=0
while [ -z "$socket_grep" ] ; do
    socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH")
    echo -n .
    sleep 1
    if [ $COUNT -eq 100 ]; then
        echo "Failed to find socket"
        break
    fi
    COUNT=$(( $COUNT + 1 ))
done
cd -
echo "*** Configuring MySQL"
mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot password 'rootpass'
declare -x admin_st=$?
if [ $admin_st -ne 0 ]
then
  echo "*** mysqladmin failed, cannot proceed" >&2
  mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown
  exit $admin_st
fi

mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO root;'
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO consult IDENTIFIED BY \"consult\";'
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'DELETE FROM mysql.user WHERE User = \"\"; FLUSH PRIVILEGES;'
echo "*** Installing Mokka DB"
date
echo "mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s"
time mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s
sleep 5
date\n""" % {'DBDUMP': self.db_dump_name})
    #Now take care of the particle tables.
    script.write("""
if [ -e "./particle.tbl" ]
then
   declare -x PARTICLETBL=./particle.tbl
fi
if [ -d "${MOKKA}" ]
#As the local env does not set MOKKA, it must come from CVMFS
then
   declare -x PARTICLETBL=$MOKKA/particle.tbl
fi
if [ -d "${MOKKATARBALL}" ]
#As the local env does set MOKKATARBALL, it must be there
then
   declare -x PARTICLETBL=$MOKKATARBALL/ConfigFiles/particle.tbl
fi
echo "/Mokka/init/PDGFile $PARTICLETBL" >> %s
\n"""% mokkasteer)

    ##Now run Mokka
    comm = 'Mokka %s -h localhost:$SOCKETPATH %s %s\n' % (mokkaextraoption, mokkasteer, self.extraCLIarguments)
    LOG.info("Command : %s" % (comm))
    script.write(comm)
    script.write('declare -x appstatus=$?\n')

    #Now shutdown the MySQL server
    script.write("mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown\n")
    script.write("""
while [ -n "$socket_grep" ] ; do
    socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH")
    echo -n .
    sleep 1
done 
""")
    script.write("rm -f %s\n" % self.db_dump_name)#remove db file
    script.write('rm -rf $MYSQLDATA\n')#cleanup
    script.write('rm -rf $MOKKADBROOT\n')#cleanup
    #script.write('quit\n')
    #script.write('EOF\n')

    script.write('exit $appstatus\n')

    script.close()
    if os.path.exists(self.applicationLog): 
      os.remove(self.applicationLog)

    os.chmod(scriptName, 0o755)
    comm = 'sh -c "./%s"' % scriptName
    self.setApplicationStatus('Mokka %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
    self.stdError = ''
    self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520)
    #self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
    resultTuple = self.result['Value']

    status = resultTuple[0]
    # stdOutput = resultTuple[1]
    # stdError = resultTuple[2]
    LOG.info("Status after Mokka execution is %s" % str(status))
    if not os.path.exists(self.applicationLog):
      LOG.error("Something went terribly wrong, the log file is not present")
      self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
      if not self.ignoreapperrors:
        LOG.error("Missing log file")
        return S_ERROR('%s did not produce the expected log' % (self.applicationName))
    ###Now change the name of Mokka output to the specified filename
    if os.path.exists("out.slcio"):
      if len(self.OutputFile) > 0:
        os.rename("out.slcio", self.OutputFile)

    failed = False
    if status not in [0, 106, 9]:
      LOG.error("Mokka execution completed with errors:")
      failed = True
    elif status in [106, 9]:
      LOG.info("Mokka execution reached end of input generator file")
    else:
      LOG.info("Mokka execution finished successfully")

    message = 'Mokka %s Successful' % (self.applicationVersion)
    if failed is True:
      LOG.error("==================================\n StdError:\n")
      LOG.error(self.stdError)
      #self.setApplicationStatus('%s Exited With Status %s' %(self.applicationName,status))
      LOG.error('Mokka Exited With Status %s' % (status))
      message = 'Mokka Exited With Status %s' % (status)
      self.setApplicationStatus(message)
      if not self.ignoreapperrors:
        LOG.error("Application exists with error:", message)
        return S_ERROR(message)
    else:
      if status in [106, 9]:
        message = 'Mokka %s reached end of input generator file' % (self.applicationVersion)
      self.setApplicationStatus(message)
    return S_OK( { 'OutputFile' : self.OutputFile } )
Exemplo n.º 7
0
  def execute(self):
    """
    Called by JobAgent
    
    Execute the following:
      - prepend in the LD_LIBRARY_PATH any lib directory of any dependency (e.g. root)
      - prepare the list of files to run on
      - set the cacheDirectory and put in there the alias.properties
      - set the lcsim file using L{PrepareLCSIMFile}
      - run java and catch the exit code
    @return: S_OK(), S_ERROR()
    """
    self.result = self.resolveInputVariables()
    if not self.systemConfig:
      self.result = S_ERROR( 'No ILC platform selected' )
    elif not self.applicationLog:
      self.result = S_ERROR( 'No Log file provided' )
    if not self.result['OK']:
      return self.result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('LCSIM should not proceed as previous step did not end properly')
    
    #look for lcsim filename
    lcsim_name = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall'%(self.systemConfig, 
                                                                          "lcsim", 
                                                                          self.applicationVersion), '')
    if not lcsim_name:
      self.log.error("Could not find lcsim file name from CS")
      return S_ERROR("Could not find lcsim file name from CS")
    
    res = getSoftwareFolder(lcsim_name)
    if not res['OK']:
      self.log.error('Application %s was not found in either the local area or shared area' % (lcsim_name))
      return res
    lcsim_name = res['Value']
    ##Need to fetch the new LD_LIBRARY_PATH
    new_ld_lib_path = GetNewLDLibs(self.systemConfig, "lcsim", self.applicationVersion)

    runonslcio = []
    if len(self.InputFile):
      res = resolveIFpaths(self.InputFile)
      if not res['OK']:
        self.setApplicationStatus('LCSIM: missing input slcio file')
        return S_ERROR('Missing slcio file!')
      runonslcio = res['Value']
    #for inputfile in inputfilelist:
    #  self.log.verbose("Will try using %s"%(os.path.basename(inputfile)))
    #  runonslcio.append(os.path.join(os.getcwd(),os.path.basename(inputfile)))


    ##Collect jar files to put in classspath
    jars = []
    if os.path.exists("lib"):
      for libs in os.listdir("lib"):
        if os.path.basename(libs).find(".jar") > 0:
          jars.append(os.path.abspath(os.path.join("lib", libs)))
      new_ld_lib_path = "./lib:%s" % new_ld_lib_path
      #Remove any libc remaining in .lib
      removeLibc("./lib")
    
    
    ###Define cache directory as local folder
    aliasproperties = os.path.basename(self.aliasproperties)
    cachedir = os.getcwd()
    if not os.path.isdir(os.path.join(cachedir, ".lcsim")):
      try:
        os.mkdir(os.path.join(cachedir, ".lcsim"))
      except OSError, x:
        self.log.error("Could not create .lcsim folder !", str(x))
Exemplo n.º 8
0
    def runIt(self):
        """ Called by ModuleBase
      
    Executes the following:
      - read the application parameters that where defined in ILCJob, and stored in the job definition
      - setup the SQL server and run it in the background
      - prepare the steering file using :any:`prepareSteeringFile`
      - run Mokka and catch its return status

    :return: S_OK(), S_ERROR()
    """

        #if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
        #  self.log.info('Skip this module, failure detected in a previous step :')
        #  self.log.info('Workflow status : %s' %(self.workflowStatus))
        #  self.log.info('Step Status %s' %(self.stepStatus))
        #  return S_OK()

        self.result = S_OK()

        if not self.platform:
            self.result = S_ERROR('No ILC platform selected')
        elif not self.applicationLog:
            self.result = S_ERROR('No Log file provided')

        if not self.result['OK']:
            self.log.error("Failed to resolve the input parameters:",
                           self.result["Message"])
            return self.result

        if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
            self.log.verbose(
                'Workflow status = %s, step status = %s' %
                (self.workflowStatus['OK'], self.stepStatus['OK']))
            return S_OK(
                'Mokka should not proceed as previous step did not end properly'
            )

        self.log.info("Executing Mokka %s" % (self.applicationVersion))

        if self.dbSlice:
            if self.dbSlice.lower().startswith("lfn"):
                self.dbSlice = os.path.basename(self.dbSlice)
            self.db_dump_name = self.dbSlice
        else:
            self.db_dump_name = "CLICMokkaDB.sql"

        res = getEnvironmentScript(self.platform, "mokka",
                                   self.applicationVersion, self.getEnvScript)
        self.log.notice("Got the environment script: %s" % res)
        if not res['OK']:
            self.log.error("Error getting the env script: ", res['Message'])
            return res
        env_script_path = res['Value']

        ####Setup MySQL instance
        mysqlBasePath = '%s/mysqltmp/MokkaDBRoot-%s' % (
            os.getcwd(), generateRandomString(8))
        self.log.notice("Placing mysql files in %s" % mysqlBasePath)

        ###steering file that will be used to run
        mokkasteer = "mokka.steer"
        if os.path.exists("mokka.steer"):
            try:
                os.rename("mokka.steer", "mymokka.steer")
            except EnvironmentError as err:
                self.log.error("Failed renaming the steering file: ", str(err))
            self.SteeringFile = "mymokka.steer"

        ###prepare steering file
        #first, I need to take the stdhep file, find its path (possible LFN)
        if len(self.InputFile) > 0:
            #self.InputFile = os.path.basename(self.InputFile)
            res = resolveIFpaths(self.InputFile)
            if not res['OK']:
                self.log.error("Generator file not found")
                return res
            self.InputFile = res['Value']
        if len(self.macFile) > 0:
            self.macFile = os.path.basename(self.macFile)
        ##idem for steering file

        self.SteeringFile = os.path.basename(self.SteeringFile)
        if not os.path.exists(self.SteeringFile):
            self.log.verbose("Steering file %s not found locally" %
                             self.SteeringFile)
            res = getSteeringFileDirName(self.platform, "mokka",
                                         self.applicationVersion)
            if not res['OK']:
                self.log.error("Missing Steering file directory:",
                               res['Message'])
                return res
            steeringfiledirname = res['Value']
            if os.path.exists(
                    os.path.join(steeringfiledirname, self.SteeringFile)):
                try:
                    shutil.copy(
                        os.path.join(steeringfiledirname, self.SteeringFile),
                        "./" + self.SteeringFile)
                except EnvironmentError as err:
                    self.log.error("Failed copying file", self.SteeringFile)
                    return S_ERROR('Failed to access file %s: %s' %
                                   (self.SteeringFile, str(err)))
                    #self.steeringFile = os.path.join(mySoftwareRoot,"steeringfiles",self.steeringFile)
        if not os.path.exists(self.SteeringFile):
            self.log.error("Missing steering file, should not happen!")
            return S_ERROR("Could not find steering file")
        else:
            self.log.verbose("Found local copy of %s" % self.SteeringFile)
        ### The following is because if someone uses particle gun, there is no InputFile
        if not len(self.InputFile):
            self.InputFile = ['']
        steerok = prepareSteeringFile(
            self.SteeringFile, mokkasteer, self.detectorModel,
            self.InputFile[0], self.macFile, self.NumberOfEvents,
            self.startFrom, self.RandomSeed, self.mcRunNumber, self.ProcessID,
            self.debug, self.OutputFile, self.inputdataMeta)
        if not steerok['OK']:
            self.log.error('Failed to create MOKKA steering file')
            return S_ERROR('Failed to create MOKKA steering file')

        ###Extra option depending on mokka version
        mokkaextraoption = ""
        if self.applicationVersion not in [
                "v07-02", "v07-02fw", "v07-02fwhp", "MokkaRevision42",
                "MokkaRevision43", "MokkaRevision44", "Revision45"
        ]:
            mokkaextraoption = "-U"

        scriptName = 'Mokka_%s_Run_%s.sh' % (self.applicationVersion,
                                             self.STEP_NUMBER)

        if os.path.exists(scriptName):
            os.remove(scriptName)
        script = open(scriptName, 'w')
        script.write('#!/bin/bash \n')
        script.write(
            '#####################################################################\n'
        )
        script.write(
            '# Dynamically generated script to run a production or analysis job. #\n'
        )
        script.write(
            '#####################################################################\n'
        )
        script.write("source %s\n" % env_script_path)
        if os.path.exists("./lib"):
            script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n')
        script.write('echo =============================\n')
        script.write('echo Content of mokka.steer:\n')
        script.write('cat mokka.steer\n')
        if self.macFile:
            script.write('echo =============================\n')
            script.write('echo Content of %s\n' %
                         os.path.basename(self.macFile))
            script.write("cat %s\n" % os.path.basename(self.macFile))
        elif os.path.exists("./mokkamac.mac"):
            script.write("""
echo =============================
echo Content of mokkamac.mac:
cat mokkamac.mac
""")
        script.write('echo =============================\n')
        script.write('echo LD_LIBRARY_PATH is\n')
        script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n')
        script.write('echo =============================\n')
        script.write('echo PATH is\n')
        script.write('echo $PATH | tr ":" "\n"\n')
        if self.debug:
            script.write('echo ldd of Mokka is\n')
            script.write('ldd `which Mokka` \n')
            script.write('echo =============================\n')
            script.write('echo ldd of mysql is\n')
            script.write('ldd `which mysql` \n')
            script.write('echo =============================\n')
        script.write('env | sort >> localEnv.log\n')
        script.write('echo =============================\n')
        script.write("""if [ ! -e ./%(DBDUMP)s ]; then
    echo "Missing DB dump!" >&2
    exit 1
fi    
    """ % {'DBDUMP': self.db_dump_name})
        ##Now start the MySQL server and configure.
        script.write("declare -x MOKKADBROOT=%s\n" % mysqlBasePath)
        script.write("declare -x WORKDIR=%s\n" % os.getcwd())
        script.write("declare -x LOGDIR=$WORKDIR/mysqllogs/\n")
        script.write("mkdir -p $LOGDIR\n")

        #mysql socket must not be longer than 107 characters!
        script.write("declare -x SOCKETBASE=/tmp/mokka-%s\n" %
                     generateRandomString(8))
        script.write("mkdir -p $SOCKETBASE\n")
        script.write("declare -x SOCKETPATH=$SOCKETBASE/mysql.sock\n")
        script.write("mkdir -p $MOKKADBROOT\n")
        script.write("declare -x MYSQLDATA=$MOKKADBROOT/data\n")
        script.write("rm -rf $MYSQLDATA\n")
        script.write("mkdir -p $MYSQLDATA\n")
        script.write("date\n")
        script.write("""echo "*** Installing MySQL"
echo "mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log"
mysql_install_db --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log
install_st=$?
if [ $install_st -ne 0 ]
then
      exit $install_st
fi
date
echo "*** Running mysqld-safe"
cd $MYSQL
bin/mysqld_safe --no-defaults --skip-networking --socket=$SOCKETPATH --datadir=$MYSQLDATA --basedir=$MYSQL --pid-file=$MOKKADBROOT/mysql.pid --log-error=$LOGDIR/mysql.err --log=$LOGDIR/mysql.log &
COUNT=0
while [ -z "$socket_grep" ] ; do
    socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH")
    echo -n .
    sleep 1
    if [ $COUNT -eq 100 ]; then
        echo "Failed to find socket"
        break
    fi
    COUNT=$(( $COUNT + 1 ))
done
cd -
echo "*** Configuring MySQL"
mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot password 'rootpass'
declare -x admin_st=$?
if [ $admin_st -ne 0 ]
then
  echo "*** mysqladmin failed, cannot proceed" >&2
  mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown
  exit $admin_st
fi

mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO root;'
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'GRANT ALL PRIVILEGES ON *.* TO consult IDENTIFIED BY \"consult\";'
mysql --no-defaults -uroot -hlocalhost --socket=$SOCKETPATH -prootpass <<< 'DELETE FROM mysql.user WHERE User = \"\"; FLUSH PRIVILEGES;'
echo "*** Installing Mokka DB"
date
echo "mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s"
time mysql --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass < ./%(DBDUMP)s
sleep 5
date\n""" % {'DBDUMP': self.db_dump_name})
        #Now take care of the particle tables.
        script.write("""
if [ -e "./particle.tbl" ]
then
   declare -x PARTICLETBL=./particle.tbl
fi
if [ -d "${MOKKA}" ]
#As the local env does not set MOKKA, it must come from CVMFS
then
   declare -x PARTICLETBL=$MOKKA/particle.tbl
fi
if [ -d "${MOKKATARBALL}" ]
#As the local env does set MOKKATARBALL, it must be there
then
   declare -x PARTICLETBL=$MOKKATARBALL/ConfigFiles/particle.tbl
fi
echo "/Mokka/init/PDGFile $PARTICLETBL" >> %s
\n""" % mokkasteer)

        ##Now run Mokka
        comm = 'Mokka %s -h localhost:$SOCKETPATH %s %s\n' % (
            mokkaextraoption, mokkasteer, self.extraCLIarguments)
        self.log.info("Command : %s" % (comm))
        script.write(comm)
        script.write('declare -x appstatus=$?\n')

        #Now shutdown the MySQL server
        script.write(
            "mysqladmin --no-defaults -hlocalhost --socket=$SOCKETPATH -uroot -prootpass shutdown\n"
        )
        script.write("""
while [ -n "$socket_grep" ] ; do
    socket_grep=$(netstat -ln 2>/dev/null | grep "$SOCKETPATH")
    echo -n .
    sleep 1
done 
""")
        script.write("rm -f %s\n" % self.db_dump_name)  #remove db file
        script.write('rm -rf $MYSQLDATA\n')  #cleanup
        script.write('rm -rf $MOKKADBROOT\n')  #cleanup
        #script.write('quit\n')
        #script.write('EOF\n')

        script.write('exit $appstatus\n')

        script.close()
        if os.path.exists(self.applicationLog):
            os.remove(self.applicationLog)

        os.chmod(scriptName, 0755)
        comm = 'sh -c "./%s"' % scriptName
        self.setApplicationStatus('Mokka %s step %s' %
                                  (self.applicationVersion, self.STEP_NUMBER))
        self.stdError = ''
        self.result = shellCall(0,
                                comm,
                                callbackFunction=self.redirectLogOutput,
                                bufferLimit=20971520)
        #self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
        resultTuple = self.result['Value']

        status = resultTuple[0]
        # stdOutput = resultTuple[1]
        # stdError = resultTuple[2]
        self.log.info("Status after Mokka execution is %s" % str(status))
        if not os.path.exists(self.applicationLog):
            self.log.error(
                "Something went terribly wrong, the log file is not present")
            self.setApplicationStatus('%s failed terribly, you are doomed!' %
                                      (self.applicationName))
            if not self.ignoreapperrors:
                self.log.error("Missing log file")
                return S_ERROR('%s did not produce the expected log' %
                               (self.applicationName))
        ###Now change the name of Mokka output to the specified filename
        if os.path.exists("out.slcio"):
            if len(self.OutputFile) > 0:
                os.rename("out.slcio", self.OutputFile)

        failed = False
        if status not in [0, 106, 9]:
            self.log.error("Mokka execution completed with errors:")
            failed = True
        elif status in [106, 9]:
            self.log.info(
                "Mokka execution reached end of input generator file")
        else:
            self.log.info("Mokka execution finished successfully")

        message = 'Mokka %s Successful' % (self.applicationVersion)
        if failed is True:
            self.log.error("==================================\n StdError:\n")
            self.log.error(self.stdError)
            #self.setApplicationStatus('%s Exited With Status %s' %(self.applicationName,status))
            self.log.error('Mokka Exited With Status %s' % (status))
            message = 'Mokka Exited With Status %s' % (status)
            self.setApplicationStatus(message)
            if not self.ignoreapperrors:
                self.log.error("Application exists with error:", message)
                return S_ERROR(message)
        else:
            if status in [106, 9]:
                message = 'Mokka %s reached end of input generator file' % (
                    self.applicationVersion)
            self.setApplicationStatus(message)
        return S_OK({'OutputFile': self.OutputFile})
Exemplo n.º 9
0
    def execute(self):
        """ Run the module
    """
        result = self.resolveInputVariables()
        if not result['OK']:
            return result
        self.result = S_OK()

        if not self.applicationLog:
            self.result = S_ERROR('No Log file provided')

        if not self.result['OK']:
            return self.result

        if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
            self.log.verbose(
                'Workflow status = %s, step status = %s' %
                (self.workflowStatus['OK'], self.stepStatus['OK']))
            return S_OK(
                '%s should not proceed as previous step did not end properly' %
                self.applicationName)

        ### Now remove the files in the FC
        lfns = self.InputFile

        ##Check that all the files are here:
        res = resolveIFpaths(lfns)
        if not res['OK']:
            self.log.error(res['Message'])
            return S_ERROR("Failed to find a file locally")

        #All files are here and available
        paths = res['Value']
        localpaths = []
        for inputfile in paths:
            basename = os.path.basename(inputfile)
            locname = os.path.join(os.getcwd(), basename)
            if not locname == inputfile:
                try:
                    shutil.copy(inputfile, locname)
                except shutil.Error:
                    self.log.error(
                        "Failed to copy file locally, will have to stop")
                    return S_ERROR("Failed copy to local directory")
            localpaths.append(locname)
            try:
                os.unlink(inputfile)
            except OSError:
                self.log.warn("Failed to remove initial file, increased \
        disk space usage")

        #all the files are in the run directory

        #get all metadata, ancestor/daughter relations, etc. for all the files

        #Update the listoutput
        if self.listoutput:
            outputlist = []
            for localFile in localpaths:
                item = {}
                item['outputFile'] = localFile
                item['outputPath'] = self.listoutput['outputPath']
                item['outputDataSE'] = self.listoutput['outputDataSE']
                outputlist.append(item)
            if self.enable:
                self.step_commons['listoutput'] = outputlist
            else:
                self.log.info("listoutput would have been ", outputlist)

        ## Make sure the path contains / at the end as we are going to
        ## concatenate final path and local files
        if not self.outputpath[-1] == '/':
            self.outputpath += "/"

        if 'ProductionOutputData' in self.workflow_commons:
            file_list = ";".join([
                self.outputpath + name
                for name in [os.path.basename(fin) for fin in localpaths]
            ])
            if self.enable:
                self.workflow_commons['ProductionOutputData'] = file_list
            else:
                self.log.info("ProductionOutputData would have been",
                              file_list)

        #Now remove them
        if self.enable:
            res = self.repMan.removeFile(lfns, force=True)
            if not res['OK']:
                self.log.error("Failed to remove the files")
                self.setApplicationStatus("Failed to remove the file")
                #return S_ERROR("Failed to remove the files")
        else:
            self.log.info("Would have removed: ", "%s" % str(lfns))

        ## Now the files are not on the storage anymore, they exist only locally. We can hope
        ## that the job will not be killed between now and the time the UploadOutputData module
        ## is called

        return self.finalStatusReport(0)
Exemplo n.º 10
0
    def runIt(self):
        """
    Called by JobAgent

    Execute the following:
      - get the environment variables that should have been set during installation
      - find the detector model xml, using CS query to obtain the path
      - prepare the steering file and command line parameters
      - run DDSim on this steering file and catch the exit status

    :rtype: S_OK, S_ERROR
    """
        self.result = S_OK()
        if not self.platform:
            self.result = S_ERROR('No ILC platform selected')
        elif not self.applicationLog:
            self.result = S_ERROR('No Log file provided')
        if not self.result['OK']:
            self.log.error("Failed to resolve input parameters:",
                           self.result['Message'])
            return self.result

        if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
            self.log.verbose(
                'Workflow status = %s, step status = %s' %
                (self.workflowStatus['OK'], self.stepStatus['OK']))
            return S_OK(
                'DDSim should not proceed as previous step did not end properly'
            )

        ##TODO: Setup LD_LIBRARY_PATH for extensions
        res = getEnvironmentScript(self.platform, self.applicationName,
                                   self.applicationVersion, self.getEnvScript)
        if not res['OK']:
            self.log.error("Could not obtain the environment script: ",
                           res["Message"])
            return res
        envScriptPath = res["Value"]

        #get the path to the detector model, either local or from the software
        resXML = self._getDetectorXML()
        if not resXML['OK']:
            self.log.error("Could not obtain the detector XML file: ",
                           resXML["Message"])
            return resXML
        compactFile = resXML['Value']

        if len(self.InputFile):
            res = resolveIFpaths(self.InputFile)
            if not res['OK']:
                self.log.error("Generator file not found")
                return res
            self.InputFile = res['Value']

        ## if steering file is set try to find it
        if len(self.SteeringFile) > 0:
            self.SteeringFile = os.path.basename(self.SteeringFile)
            if not os.path.exists(self.SteeringFile):
                res = getSteeringFileDirName(self.platform,
                                             self.applicationName,
                                             self.applicationVersion)
                if not res['OK']:
                    self.log.error(
                        "Could not find where the steering files are")
                    return res
                steeringfiledirname = res['Value']
                if os.path.exists(
                        os.path.join(steeringfiledirname, self.SteeringFile)):
                    self.SteeringFile = os.path.join(steeringfiledirname,
                                                     self.SteeringFile)
            if not os.path.exists(self.SteeringFile):
                self.log.error("Missing steering file")
                return S_ERROR("Could not find steering file")
            self.extraCLIarguments += " --steeringFile %s " % self.SteeringFile

        if self.startFrom:
            self.extraCLIarguments += " --skipNEvents %s " % self.startFrom

        if self.debug:
            self.extraCLIarguments += " --printLevel DEBUG "

        ##Same as for mokka: using ParticleGun does not imply InputFile
        if self.InputFile:
            self.InputFile = [self.InputFile] if isinstance(
                self.InputFile, basestring) else self.InputFile
            self.extraCLIarguments += " --inputFile %s " % self.InputFile[0]

        if self.NumberOfEvents:
            self.extraCLIarguments += " --numberOfEvents %s " % self.NumberOfEvents

        self.extraCLIarguments += " --random.seed %s " % self.randomSeed

        if self.OutputFile:
            self.extraCLIarguments += " --outputFile %s " % self.OutputFile

        scriptName = 'DDSim_%s_Run_%s.sh' % (self.applicationVersion,
                                             self.STEP_NUMBER)
        if os.path.exists(scriptName):
            os.remove(scriptName)
        script = []
        script.append('#!/bin/bash')
        script.append(
            '#####################################################################'
        )
        script.append(
            '# Dynamically generated script to run a production or analysis job. #'
        )
        script.append(
            '#####################################################################'
        )
        script.append('source %s' % envScriptPath)
        script.append('echo =========')

        ## for user provided libraries
        if os.path.exists("lib"):
            script.append("export LD_LIBRARY_PATH=$PWD/lib:$LD_LIBRARY_PATH")

        script.append('env | sort >> localEnv.log')
        script.append('echo ddsim:`which ddsim`')
        script.append('echo =========')
        comm = 'ddsim --compactFile %(compactFile)s %(extraArgs)s' % dict(
            compactFile=compactFile, extraArgs=self.extraCLIarguments)
        self.log.info("Command:", comm)
        script.append(comm)
        script.append('declare -x appstatus=$?')
        script.append('exit $appstatus')

        with open(scriptName, 'w') as scriptFile:
            scriptFile.write("\n".join(script))

        if os.path.exists(self.applicationLog):
            os.remove(self.applicationLog)

        os.chmod(scriptName, 0755)
        comm = 'bash "./%s"' % scriptName
        self.setApplicationStatus('DDSim %s step %s' %
                                  (self.applicationVersion, self.STEP_NUMBER))
        self.stdError = ''
        self.result = shellCall(0,
                                comm,
                                callbackFunction=self.redirectLogOutput,
                                bufferLimit=20971520)
        resultTuple = self.result['Value']
        if not os.path.exists(self.applicationLog):
            self.log.error(
                "Something went terribly wrong, the log file is not present")
            self.setApplicationStatus('%s failed to produce log file' %
                                      (self.applicationName))
            if not self.ignoreapperrors:
                return S_ERROR('%s did not produce the expected log %s' %
                               (self.applicationName, self.applicationLog))
        status = resultTuple[0]

        self.log.info("Status after the application execution is %s" % status)

        return self.finalStatusReport(status)
Exemplo n.º 11
0
    def execute(self):
        """ Execute the module, called by JobAgent
    """
        # Checks
        resultRIV = self.resolveInputVariables()
        if not resultRIV['OK']:
            self.log.error("Failed to resolve input variables",
                           resultRIV['Message'])
            return resultRIV

        if not self.platform:
            return S_ERROR('No ILC platform selected')

        if "LCIO" not in os.environ:
            self.log.error(
                "Environment variable LCIO was not defined, cannot do anything"
            )
            return S_ERROR(
                "Environment variable LCIO was not defined, cannot do anything"
            )

        if len(self.InputFile):
            res = resolveIFpaths(self.InputFile)
            if not res['OK']:
                self.log.error("Missing slcio file!")
                self.setApplicationStatus(
                    'LCIOSplit: missing input slcio file')
                return S_ERROR('Missing slcio file!')
            runonslcio = res['Value'][0]
        else:
            return S_OK("No files found to process")

        removeLibc(os.path.join(os.environ["LCIO"], "lib"))

        # Setting up script

        LD_LIBRARY_PATH = os.path.join("$LCIO", "lib")
        if 'LD_LIBRARY_PATH' in os.environ:
            LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH']

        PATH = "$LCIO/bin"
        if 'PATH' in os.environ:
            PATH += ":" + os.environ['PATH']

        scriptContent = """
#!/bin/sh

################################################################################
# Dynamically generated script by LCIOConcatenate module                       #
################################################################################

declare -x LD_LIBRARY_PATH=%s
declare -x PATH=%s

lcio split -i %s -n %s

exit $?

""" % (LD_LIBRARY_PATH, PATH, runonslcio, self.nbEventsPerSlice)

        # Write script to file

        scriptPath = 'LCIOSplit_%s_Run_%s.tcl' % (self.applicationVersion,
                                                  self.STEP_NUMBER)

        if os.path.exists(scriptPath):
            os.remove(scriptPath)

        script = open(scriptPath, 'w')
        script.write(scriptContent)
        script.close()

        # Setup log file for application stdout

        if os.path.exists(self.applicationLog):
            os.remove(self.applicationLog)

        # Run code

        os.chmod(scriptPath, 0755)

        command = '"./%s"' % (scriptPath)

        self.setApplicationStatus('LCIOSplit %s step %s' %
                                  (self.applicationVersion, self.STEP_NUMBER))
        self.stdError = ''

        result = shellCall(0,
                           command,
                           callbackFunction=self.redirectLogOutput,
                           bufferLimit=20971520)

        # Check results
        status = result['Value'][0]

        if not os.path.exists(self.applicationLog):
            self.log.error("Cannot access log file, cannot proceed")
            return S_ERROR("Failed reading the log file")

        baseinputfilename = os.path.basename(runonslcio).split(".slcio")[0]
        output_file_base_name = ''
        if self.OutputFile:
            output_file_base_name = self.OutputFile.split('.slcio')[0]
        self.log.info("Will rename all files using '%s' as base." %
                      output_file_base_name)
        numberofeventsdict = {}
        fname = ''
        with open(self.applicationLog, "r") as logf:
            for line in logf:
                line = line.rstrip()
                if line.count(baseinputfilename):
                    #First, we need to rename those guys
                    current_file = os.path.basename(line).replace(".slcio", "")
                    current_file_extension = current_file.replace(
                        baseinputfilename, "")
                    newfile = output_file_base_name + current_file_extension + ".slcio"
                    os.rename(line, newfile)
                    fname = newfile
                    numberofeventsdict[fname] = 0
                elif line.count("events"):
                    numberofeventsdict[fname] = int(line.split()[0])

        self.log.verbose("Number of eventsdict dict: %s" % numberofeventsdict)

        ##Now update the workflow_commons dict with the relation between filename and number of events: needed for
        #the registerOutputData
        self.workflow_commons[
            'file_number_of_event_relation'] = numberofeventsdict
        if self.listoutput:
            outputlist = []
            for fileName in numberofeventsdict:
                item = {}
                item['outputFile'] = fileName
                item['outputPath'] = self.listoutput['outputPath']
                item['outputDataSE'] = self.listoutput['outputDataSE']
                outputlist.append(item)
            self.step_commons['listoutput'] = outputlist

        #Not only the step_commons must be updated
        if 'ProductionOutputData' in self.workflow_commons:
            proddata = self.workflow_commons['ProductionOutputData'].split(";")
            finalproddata = []
            this_split_data = ''
            for item in proddata:
                if not item.count(output_file_base_name):
                    finalproddata.append(item)
                else:
                    this_split_data = item
            path = os.path.dirname(this_split_data)
            for fileName in numberofeventsdict:
                finalproddata.append(os.path.join(path, fileName))
            self.workflow_commons['ProductionOutputData'] = ";".join(
                finalproddata)

        self.log.info("Status after the application execution is %s" %
                      str(status))
        self.listDir()
        return self.finalStatusReport(status)
Exemplo n.º 12
0
  def execute(self):
    """ Called by Agent
      
      Executes the following:
        - read the application parameters that where defined in ILCJob, and stored in the job definition
        - setup the SQL server and run it in the background, via a call to L{SQLWrapper}
        - prepare the steering fie using L{PrepareSteeringFile}
        - run Mokka and catch its return status
      @return: S_OK(), S_ERROR()
      
    """
    result = self.resolveInputVariables()
    if not result['OK']:
      return result
    #if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
    #  self.log.info('Skip this module, failure detected in a previous step :')
    #  self.log.info('Workflow status : %s' %(self.workflowStatus))
    #  self.log.info('Step Status %s' %(self.stepStatus))
    #  return S_OK()

    self.result = S_OK()
       
    if not self.systemConfig:
      self.result = S_ERROR( 'No ILC platform selected' )
    elif not self.applicationLog:
      self.result = S_ERROR( 'No Log file provided' )

    if not self.result['OK']:
      return self.result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('Mokka should not proceed as previous step did not end properly')

    cwd = os.getcwd()
    root = gConfig.getValue('/LocalSite/Root', cwd)
    self.log.info( "Executing Mokka %s" % ( self.applicationVersion ))
    self.log.info("Platform for job is %s" % ( self.systemConfig ) )
    self.log.info("Root directory for job is %s" % ( root ) )

    mokkaDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig, "mokka", 
                                                                          self.applicationVersion), '')
    if not mokkaDir:
      self.log.error('Could not get Tar ball name')
      return S_ERROR('Failed finding software directory')
    mokkaDir = mokkaDir.replace(".tgz", "").replace(".tar.gz", "")
    #mokkaDir = 'lddLib' ###Temporary while mokka tar ball are not redone.
    mySoftwareRoot = ''
    localArea = LocalArea()
    sharedArea = SharedArea()
    if os.path.exists('%s%s%s' % (localArea, os.sep, mokkaDir)):
      mySoftwareRoot = localArea
    elif os.path.exists('%s%s%s' % (sharedArea, os.sep, mokkaDir)):
      mySoftwareRoot = sharedArea
    else:
      self.log.error("Mokka: could not find installation directory!")
      return S_ERROR("Mokka installation could not be found")  
    ##This is a duplication of what is above, but the above cannot easily be removed...
    res = getSoftwareFolder(mokkaDir)
    if not res['OK']:
      self.log.error("Mokka: could not find installation directory!")
      return res
    myMokkaDir = res['Value']
      
    if not mySoftwareRoot:
      self.log.error('Directory %s was not found in either the local area %s or shared area %s' % (mokkaDir, localArea, 
                                                                                                   sharedArea))
      return S_ERROR('Failed to discover software')


    ####Setup MySQL instance      
    MokkaDBrandomName =  '/tmp/MokkaDBRoot-' + GenRandString(8)
      
    #sqlwrapper = SQLWrapper(self.dbslice,mySoftwareRoot,"/tmp/MokkaDBRoot")#mySoftwareRoot)
    sqlwrapper = SQLWrapper(mySoftwareRoot, MokkaDBrandomName)#mySoftwareRoot)
    res = sqlwrapper.setDBpath(myMokkaDir, self.dbSlice)
    if not res['OK']:
      self.log.error("Failed to find the DB slice")
      return res
    result = sqlwrapper.makedirs()
    if not result['OK']:
      self.setApplicationStatus('MySQL setup failed to create directories.')
      return result
    result = sqlwrapper.mysqlSetup()
    if not result['OK']:
      self.setApplicationStatus('MySQL setup failed.')
      return result

    ##Need to fetch the new LD_LIBRARY_PATH
    new_ld_lib_path = GetNewLDLibs(self.systemConfig, "mokka", self.applicationVersion)

    ##Remove libc
    removeLibc(myMokkaDir)

    ##Get the particle.tbl, if any
    path_to_particle_tbl = ''
    dir_to_particletbl = os.path.join(myMokkaDir, 'ConfigFiles')
    if os.path.exists(dir_to_particletbl):
      configdir = os.listdir(dir_to_particletbl)
      if 'particle.tbl' in configdir:
        path_to_particle_tbl = os.path.join(dir_to_particletbl, 'particle.tbl')
    if os.path.exists('./particle.tbl'):
      path_to_particle_tbl = "./particle.tbl"    
    ###steering file that will be used to run
    mokkasteer = "mokka.steer"
    if os.path.exists("mokka.steer"):
      try:
        shutil.move("mokka.steer", "mymokka.steer")
      except:
        self.log.error("Failed renaming the steering file")
      self.SteeringFile = "mymokka.steer"
        
    ###prepare steering file
    #first, I need to take the stdhep file, find its path (possible LFN)      
    if len(self.InputFile) > 0:
      #self.InputFile = os.path.basename(self.InputFile)
      res = resolveIFpaths(self.InputFile)
      if not res['OK']:
        self.log.error("Generator file not found")
        result = sqlwrapper.mysqlCleanUp()
        return res
      self.InputFile = res['Value']
    if len(self.macFile) > 0:
      self.macFile = os.path.basename(self.macFile)
    ##idem for steering file
      
    self.SteeringFile = os.path.basename(self.SteeringFile)
    if not os.path.exists(self.SteeringFile):
      res =  getSteeringFileDirName(self.systemConfig, "mokka", self.applicationVersion)
      if not res['OK']:
        result = sqlwrapper.mysqlCleanUp()
        return res
      steeringfiledirname = res['Value']
      if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
        try:
          shutil.copy(os.path.join(steeringfiledirname, self.SteeringFile), "./" + self.SteeringFile )
        except Exception, x:
          result = sqlwrapper.mysqlCleanUp()
          return S_ERROR('Failed to access file %s: %s' % (self.SteeringFile, str(x)))  
Exemplo n.º 13
0
    def runIt(self):
        """
    Called by JobAgent
    
    Execute the following:
      - get the environment variables that should have been set during installation
      - download the detector model, using CS query to fetch the address
      - prepare the mac file using :any:`prepareMacFile`
      - run SLIC on this mac File and catch the exit status

    :return: S_OK(), S_ERROR()
    """
        self.result = S_OK()
        if not self.platform:
            self.result = S_ERROR('No ILC platform selected')
        elif not self.applicationLog:
            self.result = S_ERROR('No Log file provided')
        if not self.result['OK']:
            self.log.error("Failed to resolve input parameters:",
                           self.result['Message'])
            return self.result

        if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
            self.log.verbose(
                'Workflow status = %s, step status = %s' %
                (self.workflowStatus['OK'], self.stepStatus['OK']))
            return S_OK(
                'SLIC should not proceed as previous step did not end properly'
            )

        res = getEnvironmentScript(self.platform, self.applicationName,
                                   self.applicationVersion, self.getEnvScript)
        if not res['OK']:
            self.log.error("Could not obtain the environment script: ",
                           res["Message"])
            return res
        env_script_path = res["Value"]

        retMod = self.getDetectorModel()
        if not retMod:
            return retMod

        slicmac = 'slicmac.mac'
        if len(self.InputFile):
            res = resolveIFpaths(self.InputFile)
            if not res['OK']:
                self.log.error("Generator file not found")
                return res
            self.InputFile = res['Value']

        if len(self.SteeringFile) > 0:
            self.SteeringFile = os.path.basename(self.SteeringFile)
            if not os.path.exists(self.SteeringFile):
                res = getSteeringFileDirName(self.platform,
                                             self.applicationName,
                                             self.applicationVersion)
                if not res['OK']:
                    self.log.error(
                        "Could not find where the steering files are")
                    return res
                steeringfiledirname = res['Value']
                if os.path.exists(
                        os.path.join(steeringfiledirname, self.SteeringFile)):
                    self.SteeringFile = os.path.join(steeringfiledirname,
                                                     self.SteeringFile)
            if not os.path.exists(self.SteeringFile):
                self.log.error("Missing steering file")
                return S_ERROR("Could not find mac file")
        ##Same as for mokka: using ParticleGun does not imply InputFile
        if not len(self.InputFile):
            self.InputFile = ['']
        macok = prepareMacFile(self.SteeringFile, slicmac, self.InputFile[0],
                               self.NumberOfEvents, self.startFrom,
                               self.detectorModel, self.RandomSeed,
                               self.OutputFile, self.debug)
        if not macok['OK']:
            self.log.error('Failed to create SLIC mac file')
            return S_ERROR('Error when creating SLIC mac file')

        scriptName = 'SLIC_%s_Run_%s.sh' % (self.applicationVersion,
                                            self.STEP_NUMBER)
        if os.path.exists(scriptName):
            os.remove(scriptName)
        script = open(scriptName, 'w')
        script.write('#!/bin/bash \n')
        script.write(
            '#####################################################################\n'
        )
        script.write(
            '# Dynamically generated script to run a production or analysis job. #\n'
        )
        script.write(
            '#####################################################################\n'
        )
        script.write("source %s\n" % (env_script_path))

        preloadDir = "%s/preload" % os.getcwd()
        if os.path.exists(preloadDir):
            preadloadLibs = set(os.listdir(preloadDir))
            preloadString = ":".join(lib for lib in preadloadLibs
                                     if ".so" in lib)
            script.write('declare -x LD_LIBRARY_PATH=%s:${LD_LIBRARY_PATH}\n' %
                         preloadDir)
            script.write('declare -x LD_PRELOAD=%s\n' % preloadString)

        script.write('echo =========\n')
        script.write('env | sort >> localEnv.log\n')
        script.write('echo SLIC:\n')
        script.write("which slic\n")
        script.write('echo =========\n')
        comm = 'slic -P $PARTICLE_TBL -m %s %s\n' % (slicmac,
                                                     self.extraCLIarguments)
        self.log.info("Command:", comm)
        script.write(comm)
        script.write('declare -x appstatus=$?\n')
        script.write('exit $appstatus\n')
        script.close()
        if os.path.exists(self.applicationLog):
            os.remove(self.applicationLog)

        os.chmod(scriptName, 0755)
        comm = 'sh -c "./%s"' % scriptName
        self.setApplicationStatus('SLIC %s step %s' %
                                  (self.applicationVersion, self.STEP_NUMBER))
        self.stdError = ''
        self.result = shellCall(0,
                                comm,
                                callbackFunction=self.redirectLogOutput,
                                bufferLimit=20971520)
        #self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
        resultTuple = self.result['Value']
        if not os.path.exists(self.applicationLog):
            self.log.error(
                "Something went terribly wrong, the log file is not present")
            self.setApplicationStatus('%s failed terribly, you are doomed!' %
                                      (self.applicationName))
            if not self.ignoreapperrors:
                return S_ERROR('%s did not produce the expected log' %
                               (self.applicationName))
        status = resultTuple[0]
        # stdOutput = resultTuple[1]
        # stdError = resultTuple[2]
        self.log.info("Status after the application execution is %s" %
                      str(status))

        return self.finalStatusReport(status)
Exemplo n.º 14
0
    def runIt(self):
        """ Called from ModuleBase
    """
        self.result = S_OK()
        if not self.platform:
            self.result = S_ERROR('No ILC platform selected')
        elif not self.applicationLog:
            self.result = S_ERROR('No Log file provided')
        if not self.result['OK']:
            self.log.error("Failed to resolve input parameters:",
                           self.result["Message"])
            return self.result

        if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
            self.log.verbose(
                'Workflow status = %s, step status = %s' %
                (self.workflowStatus['OK'], self.stepStatus['OK']))
            return S_OK(
                'SLIC Pandora should not proceed as previous step did not end properly'
            )

        ###Get the env script
        res = getEnvironmentScript(self.platform, self.applicationName,
                                   self.applicationVersion, self.getEnvScript)
        if not res['OK']:
            self.log.error("Failed to get the environment script:",
                           res["Message"])
            return res
        env_script_path = res["Value"]

        res = resolveIFpaths(self.InputFile)
        if not res['OK']:
            self.log.error('Could not find input files')
            self.setApplicationStatus('SLICPandora: missing slcio file')
            return S_ERROR('Missing slcio file!')
        runonslcio = res['Value'][0]

        if not self.detectorxml.count(".xml") or not os.path.exists(
                os.path.basename(self.detectorxml)):
            detmodel = self.detectorxml.replace("_pandora.xml", "")
            if os.path.exists(detmodel + ".zip"):
                try:
                    unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd())
                except (RuntimeError, OSError) as err:
                    self.log.error(
                        "Exception when unpacking detectormodel zip file:",
                        str(err))
                    os.unlink(detmodel + ".zip")
            if not os.path.exists(detmodel + ".zip"):
                #retrieve detector model from web
                detector_urls = self.ops.getValue(
                    '/SLICweb/SLICDetectorModels', [''])
                if len(detector_urls[0]) < 1:
                    self.log.error(
                        'Could not find in CS the URL for detector model')
                    return S_ERROR(
                        'Could not find in CS the URL for detector model')

                for detector_url in detector_urls:
                    try:
                        urllib.urlretrieve(
                            "%s%s" % (detector_url, detmodel + ".zip"),
                            detmodel + ".zip")
                    except IOError:
                        self.log.error("Download of detector model failed")
                        continue
                    try:
                        unzip_file_into_dir(open(detmodel + ".zip"),
                                            os.getcwd())
                        break
                    except (RuntimeError, OSError) as err:
                        self.log.error("Exception for zip file obtained from ",
                                       detector_url)
                        self.log.error("Exception:", str(err))
                        os.unlink(detmodel + ".zip")
                        continue
            #if os.path.exists(detmodel): #and os.path.isdir(detmodel):
            self.detectorxml = os.path.join(os.getcwd(), self.detectorxml)
            if not self.detectorxml.endswith('xml'):
                self.detectorxml = self.detectorxml + "_pandora.xml"

        if not os.path.exists(self.detectorxml):
            self.log.error('Detector model xml %s was not found, exiting' %
                           self.detectorxml)
            return S_ERROR('Detector model xml was not found, exiting')

        oldversion = False
        if self.applicationVersion in [
                'CLIC_CDR', 'CDR1', 'CDR2', 'CDR0', 'V2', 'V3', 'V4'
        ]:
            oldversion = True

        scriptName = 'SLICPandora_%s_Run_%s.sh' % (self.applicationVersion,
                                                   self.STEP_NUMBER)
        if os.path.exists(scriptName):
            os.remove(scriptName)
        script = open(scriptName, 'w')
        script.write('#!/bin/bash \n')
        script.write(
            '#####################################################################\n'
        )
        script.write(
            '# Dynamically generated script to run a production or analysis job. #\n'
        )
        script.write(
            '#####################################################################\n'
        )
        script.write("source %s\n" % env_script_path)
        script.write("declare -x file=./Settings/%s\n" % self.pandorasettings)
        script.write("""
if [ -e "${file}" ]
then
   declare -x PANDORASETTINGS=$file
else
  if [ -d "${PANDORASETTINGSDIR}" ]
  then
    cp $PANDORASETTINGSDIR/*.xml .
    declare -x PANDORASETTINGS=%s
  fi
fi
if [ ! -e "${PANDORASETTINGS}" ]
then
  echo "Missing PandoraSettings file"
  exit 1
fi  
""" % self.pandorasettings)
        script.write('echo =============================\n')
        script.write('echo PATH is \n')
        script.write('echo $PATH | tr ":" "\n"  \n')
        script.write('echo ==============\n')
        if os.path.exists("./lib"):
            script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n')
        script.write('echo =============================\n')
        script.write('echo LD_LIBRARY_PATH is \n')
        script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n')
        script.write('echo ============================= \n')
        script.write('env | sort >> localEnv.log\n')
        #Now run it
        if oldversion:
            comm = 'PandoraFrontend %s $PANDORASETTINGS %s %s %s' % (
                self.detectorxml, runonslcio, self.OutputFile,
                str(self.NumberOfEvents))
        else:
            comm = 'PandoraFrontend -g %s -c $PANDORASETTINGS -i %s -o %s -r %s' % (
                self.detectorxml, runonslcio, self.OutputFile,
                str(self.NumberOfEvents))
        comm = "%s %s\n" % (comm, self.extraCLIarguments)
        self.log.info("Will run %s" % comm)
        script.write(comm)
        script.write('declare -x appstatus=$?\n')
        #script.write('where\n')
        #script.write('quit\n')
        #script.write('EOF\n')
        script.write('exit $appstatus\n')

        script.close()
        if os.path.exists(self.applicationLog):
            os.remove(self.applicationLog)

        os.chmod(scriptName, 0755)
        comm = 'sh -c "./%s"' % (scriptName)
        self.setApplicationStatus('SLICPandora %s step %s' %
                                  (self.applicationVersion, self.STEP_NUMBER))
        self.stdError = ''
        self.result = shellCall(0,
                                comm,
                                callbackFunction=self.redirectLogOutput,
                                bufferLimit=20971520)
        #self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
        resultTuple = self.result['Value']
        if resultTuple[0]:
            self.log.error("There was an error during the execution")

        if not os.path.exists(self.applicationLog):
            self.log.error(
                "Something went terribly wrong, the log file is not present")
            self.setApplicationStatus('%s failed terribly, you are doomed!' %
                                      (self.applicationName))
            if not self.ignoreapperrors:
                return S_ERROR('%s did not produce the expected log' %
                               (self.applicationName))

        logf = open(self.applicationLog, 'r')
        if "Missing PandoraSettings file" in logf.readlines()[-1]:
            self.log.error("Issue with the Pandora Settings file.")
        logf.close()

        status = resultTuple[0]
        # stdOutput = resultTuple[1]
        # stdError = resultTuple[2]
        self.log.info("Status after the application execution is %s" %
                      str(status))

        return self.finalStatusReport(status)
Exemplo n.º 15
0
  def execute(self):
    """
    Called by JobAgent
    
    Execute the following:
      - get the environment variables that should have been set during installation
      - download the detector model, using CS query to fetch the address
      - prepare the mac file using L{PrepareMacFile}
      - run SLIC on this mac File and catch the exit status
    @return: S_OK(), S_ERROR()
    """
    self.result = self.resolveInputVariables()
    if not self.systemConfig:
      self.result = S_ERROR( 'No ILC platform selected' )
    elif not self.applicationLog:
      self.result = S_ERROR( 'No Log file provided' )
    if not self.result['OK']:
      return self.result
    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('SLIC should not proceed as previous step did not end properly')
    
    if not os.environ.has_key('SLIC_DIR'):
      self.log.error('SLIC_DIR not found, probably the software installation failed')
      return S_ERROR('SLIC_DIR not found, probably the software installation failed')
    if not os.environ.has_key('SLIC_VERSION'):
      self.log.error('SLIC_VERSION not found, probably the software installation failed')
      return S_ERROR('SLIC_VERSION not found, probably the software installation failed')
    if not os.environ.has_key('LCDD_VERSION'):
      self.log.error('LCDD_VERSION not found, probably the software installation failed')
      return S_ERROR('LCDD_VERSION not found, probably the software installation failed')
    #if not os.environ.has_key('XERCES_VERSION'):
    #  self.log.error('XERCES_VERSION not found, probably the software installation failed')
    #  return S_ERROR('XERCES_VERSION not found, probably the software installation failed')


    slicDir = os.environ['SLIC_DIR']
    res = getSoftwareFolder(slicDir)
    if not res['OK']:
      self.log.error('Directory %s was not found in either the local area or shared area' % (slicDir))
      return res
    mySoftwareRoot = res['Value']
    ##Need to fetch the new LD_LIBRARY_PATH
    new_ld_lib_path = GetNewLDLibs(self.systemConfig, "slic", self.applicationVersion)

    #retrieve detector model from web
    detector_urls = self.ops.getValue('/SLICweb/SLICDetectorModels', [''])
    if len(detector_urls[0]) < 1:
      self.log.error('Could not find in CS the URL for detector model')
      return S_ERROR('Could not find in CS the URL for detector model')

    if not os.path.exists(self.detectorModel + ".zip"):
      for detector_url in detector_urls:
        try:
          urllib.urlretrieve("%s%s" % (detector_url, self.detectorModel + ".zip"), 
                                                 self.detectorModel + ".zip")
        except:
          self.log.error("Download of detector model failed")
          continue

    if not os.path.exists(self.detectorModel + ".zip"):
      self.log.error('Detector model %s was not found neither locally nor on the web, exiting' % self.detectorModel)
      return S_ERROR('Detector model %s was not found neither locally nor on the web, exiting' % self.detectorModel)
    try:
      unzip_file_into_dir(open(self.detectorModel + ".zip"), os.getcwd())
    except:
      os.unlink(self.detectorModel + ".zip")
      self.log.error('Failed to unzip detector model')
      return S_ERROR('Failed to unzip detector model')
    #unzip detector model
    #self.unzip_file_into_dir(open(self.detectorModel+".zip"),os.getcwd())
    
    slicmac = 'slicmac.mac'
    if len(self.InputFile):
      res = resolveIFpaths(self.InputFile)
      if not res['OK']:
        self.log.error("Generator file not found")
        return res
      self.InputFile = res['Value']
    
    if len(self.SteeringFile) > 0:
      self.SteeringFile = os.path.basename(self.SteeringFile)
      if not os.path.exists(self.SteeringFile):
        res = getSteeringFileDirName(self.systemConfig, "slic", self.applicationVersion)     
        if not res['OK']:
          self.log.error("Could not find where the steering files are")
        steeringfiledirname = res['Value']
        if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
          self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile)
      if not os.path.exists(self.SteeringFile):
        return S_ERROR("Could not find mac file")    
    ##Same as for mokka: using ParticleGun does not imply InputFile
    if not len(self.InputFile):
      self.InputFile = ['']    
    macok = PrepareMacFile(self.SteeringFile, slicmac, self.InputFile[0],
                           self.NumberOfEvents, self.startFrom, self.detectorModel,
                           self.randomseed, self.OutputFile, self.debug)
    if not macok['OK']:
      self.log.error('Failed to create SLIC mac file')
      return S_ERROR('Error when creating SLIC mac file')
    
    scriptName = 'SLIC_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
    if os.path.exists(scriptName): 
      os.remove(scriptName)
    script = open(scriptName, 'w')
    script.write('#!/bin/sh \n')
    script.write('#####################################################################\n')
    script.write('# Dynamically generated script to run a production or analysis job. #\n')
    script.write('#####################################################################\n')
    if os.environ.has_key('XERCES_VERSION'):
      script.write('declare -x XERCES_LIB_DIR=%s/packages/xerces/%s/lib\n' % (mySoftwareRoot, 
                                                                              os.environ['XERCES_VERSION']))
      if new_ld_lib_path:
        script.write('declare -x LD_LIBRARY_PATH=$XERCES_LIB_DIR:%s\n' % new_ld_lib_path)
      else:
        script.write('declare -x LD_LIBRARY_PATH=$XERCES_LIB_DIR\n')
      
    script.write('declare -x GEANT4_DATA_ROOT=%s/packages/geant4/data\n' % mySoftwareRoot)
    script.write('declare -x G4LEVELGAMMADATA=$(ls -d $GEANT4_DATA_ROOT/PhotonEvaporation*)\n')
    script.write('declare -x G4RADIOACTIVEDATA=$(ls -d $GEANT4_DATA_ROOT/RadioactiveDecay*)\n')
    script.write('declare -x G4LEDATA=$(ls -d $GEANT4_DATA_ROOT/G4EMLOW*)\n')
    script.write('declare -x G4NEUTRONHPDATA=$(ls -d $GEANT4_DATA_ROOT/G4NDL*)\n')
    script.write('declare -x GDML_SCHEMA_DIR=%s/packages/lcdd/%s\n' % (mySoftwareRoot, os.environ['LCDD_VERSION']))
    script.write('declare -x PARTICLE_TBL=%s/packages/slic/%s/data/particle.tbl\n' % (mySoftwareRoot, 
                                                                                      os.environ['SLIC_VERSION']))
    script.write('declare -x MALLOC_CHECK_=0\n')
    if os.path.exists("%s/lib" % (mySoftwareRoot)):
      script.write('declare -x LD_LIBRARY_PATH=%s/lib:$LD_LIBRARY_PATH\n' % (mySoftwareRoot))
    script.write('echo =========\n')
    script.write('env | sort >> localEnv.log\n')
    script.write('echo =========\n')
    comm = '%s/packages/slic/%s/bin/Linux-g++/slic -P $PARTICLE_TBL -m %s\n' % (mySoftwareRoot, 
                                                                                os.environ['SLIC_VERSION'], 
                                                                                slicmac)
    print comm
    script.write(comm)
    script.write('declare -x appstatus=$?\n')
    script.write('exit $appstatus\n')
    script.close()
    if os.path.exists(self.applicationLog): 
      os.remove(self.applicationLog)

    os.chmod(scriptName, 0755)
    comm = 'sh -c "./%s"' % scriptName
    self.setApplicationStatus('SLIC %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
    self.stdError = ''
    self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520)
    #self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
    resultTuple = self.result['Value']
    if not os.path.exists(self.applicationLog):
      self.log.error("Something went terribly wrong, the log file is not present")
      self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
      if not self.ignoreapperrors:
        return S_ERROR('%s did not produce the expected log' % (self.applicationName))
    status = resultTuple[0]
    # stdOutput = resultTuple[1]
    # stdError = resultTuple[2]
    self.log.info( "Status after the application execution is %s" % str( status ) )

    return self.finalStatusReport(status)
Exemplo n.º 16
0
  def execute(self):
    """ Execute the module, called by JobAgent
    """
    # Get input variables

    self.result = self.resolveInputVariables()
    # Checks

    if not self.platform:
      self.result = S_ERROR( 'No ILC platform selected' )

    if not self.result['OK']:
      self.log.error("Failed to resolve input parameters:", self.result["Message"])
      return self.result

    
    if len(self.InputFile):
      res = resolveIFpaths(self.InputFile)
      if not res['OK']:
        self.log.error("Cannot find input file")
        self.setApplicationStatus('StdHepSplit: missing input stdhep file')
        return S_ERROR('Missing stdhep file!')
      runonstdhep = res['Value'][0]
    else:
      self.log.warn("No files found to split")
      return S_OK("No files found to process")
    # removeLibc

    #removeLibc( os.path.join( os.environ["LCIO"], "lib" ) )

    prefix = ''
    if self.OutputFile:
      prefix = self.OutputFile.split('.stdhep')[0]
      
    else:
      prefix = "this_split"
    #because we need to make sure the files end up in the base directory at the end
    self.OutputFile = prefix
    
    self.log.info("Will rename all files using '%s' as base." % prefix)

    # Setting up script
    res = getSoftwareFolder(self.platform, "stdhepsplit", self.applicationVersion)
    if not res['OK']:
      self.log.error("Failed to find the software")
      self.setApplicationStatus('StdHepSplit: Could not find neither local area not shared area install')
      return res
    
    mysplitDir = res['Value']
    new_ld_lib = getNewLDLibs(self.platform, "stdhepsplit", self.applicationVersion)
    LD_LIBRARY_PATH = os.path.join(mysplitDir, "lib") + ":" + new_ld_lib

    

    scriptContent = """
#!/bin/sh

################################################################################
# Dynamically generated script by LCIOConcatenate module                       #
################################################################################

declare -x LD_LIBRARY_PATH=%s

%s/hepsplit --infile %s --nw_per_file %s --outpref %s

exit $?

""" % (
    LD_LIBRARY_PATH,
    mysplitDir,
    runonstdhep,
    self.nbEventsPerSlice,
    prefix
)

    # Write script to file

    scriptPath = 'StdHepSplit_%s_Run_%s.tcl' % ( self.applicationVersion, self.STEP_NUMBER )

    if os.path.exists(scriptPath):
      os.remove(scriptPath)

    script = open( scriptPath, 'w' )
    script.write( scriptContent )
    script.close()

    # Setup log file for application stdout

    if os.path.exists(self.applicationLog):
      os.remove(self.applicationLog)

    # Run code

    os.chmod( scriptPath, 0755 )

    command = '"./%s"' % ( scriptPath )

    self.setApplicationStatus( 'StdHepSplit %s step %s' % ( self.applicationVersion, self.STEP_NUMBER ) )
    self.stdError = ''

    self.result = shellCall( 0,
                             command,
                             callbackFunction = self.redirectLogOutput,
                             bufferLimit = 20971520
                           )

        # Check results

    resultTuple = self.result['Value']
    status      = resultTuple[0]

    if not os.path.exists(self.applicationLog):
      self.log.error("Cannot access log file, cannot proceed")
      return S_ERROR("Failed reading the log file")

    with open(self.applicationLog, "r") as logf:
      numberofeventsdict = {}
      fname = ''
      for line in logf:
        line = line.rstrip()
        if line.count('Open output file'):
          fname = line.split()[-1].rstrip().rstrip("\0")
          numberofeventsdict[fname] = 0
        elif line.count("Record") and not line.count('Output Begin Run') :
          #print line
          val = line.split("=")[1].rstrip().lstrip()
          if val != '0':
            numberofeventsdict[fname] = int(val)

    self.log.verbose("numberofeventsdict dict: %s" % numberofeventsdict)   

    ##Now update the workflow_commons dict with the relation between filename and number of events: 
    #needed for the registerOutputData
    self.workflow_commons['file_number_of_event_relation'] = numberofeventsdict
    if self.listoutput:
      outputlist = []
      for of in numberofeventsdict.keys():
        item = {}
        item['outputFile'] = of
        item['outputPath'] = self.listoutput['outputPath']
        item['outputDataSE'] = self.listoutput['outputDataSE']
        outputlist.append(item)
      self.step_commons['listoutput'] = outputlist
      
    #Not only the step_commons must be updated  
    if self.workflow_commons.has_key('ProductionOutputData'):
      proddata = self.workflow_commons['ProductionOutputData'].split(";")
      finalproddata = []
      this_split_data = ''
      for item in proddata:
        if not item.count(prefix):
          finalproddata.append(item)
        else:
          this_split_data = item
      path = os.path.dirname(this_split_data)
      for of in numberofeventsdict.keys():
        finalproddata.append(os.path.join(path, of))
      self.workflow_commons['ProductionOutputData'] = ";".join(finalproddata)  
    
    self.log.info( "Status after the application execution is %s" % str( status ) )
    if status == 2:
      self.log.info("Reached end of input file")
      status = 0
      
    self.listDir()  
    return self.finalStatusReport(status)
Exemplo n.º 17
0
  def runIt(self):
    """
    Called by JobAgent
    
    Execute the following:
      - get the environment variables that should have been set during installation
      - download the detector model, using CS query to fetch the address
      - prepare the mac file using :any:`prepareMacFile`
      - run SLIC on this mac File and catch the exit status

    :return: S_OK(), S_ERROR()
    """
    self.result = S_OK()
    if not self.platform:
      self.result = S_ERROR( 'No ILC platform selected' )
    elif not self.applicationLog:
      self.result = S_ERROR( 'No Log file provided' )
    if not self.result['OK']:
      LOG.error("Failed to resolve input parameters:", self.result['Message'])
      return self.result
    
    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('SLIC should not proceed as previous step did not end properly')
    
    res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript)
    if not res['OK']:
      LOG.error("Could not obtain the environment script: ", res["Message"])
      return res
    env_script_path = res["Value"]
    
    retMod = self.getDetectorModel()
    if not retMod:
      return retMod

    slicmac = 'slicmac.mac'
    if len(self.InputFile):
      res = resolveIFpaths(self.InputFile)
      if not res['OK']:
        LOG.error("Generator file not found")
        return res
      self.InputFile = res['Value']
    
    if len(self.SteeringFile) > 0:
      self.SteeringFile = os.path.basename(self.SteeringFile)
      if not os.path.exists(self.SteeringFile):
        res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion)
        if not res['OK']:
          LOG.error("Could not find where the steering files are")
          return res
        steeringfiledirname = res['Value']
        if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
          self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile)
      if not os.path.exists(self.SteeringFile):
        LOG.error("Missing steering file")
        return S_ERROR("Could not find mac file")    
    ##Same as for mokka: using ParticleGun does not imply InputFile
    if not len(self.InputFile):
      self.InputFile = ['']    
    macok = prepareMacFile(self.SteeringFile, slicmac, self.InputFile[0],
                           self.NumberOfEvents, self.startFrom, self.detectorModel,
                           self.RandomSeed, self.OutputFile, self.debug)
    if not macok['OK']:
      LOG.error('Failed to create SLIC mac file')
      return S_ERROR('Error when creating SLIC mac file')
    
    scriptName = 'SLIC_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
    if os.path.exists(scriptName): 
      os.remove(scriptName)
    script = open(scriptName, 'w')
    script.write('#!/bin/bash \n')
    script.write('#####################################################################\n')
    script.write('# Dynamically generated script to run a production or analysis job. #\n')
    script.write('#####################################################################\n')
    script.write("source %s\n" % (env_script_path))

    preloadDir = "%s/preload" % os.getcwd()
    if os.path.exists( preloadDir ):
      preadloadLibs = set( os.listdir( preloadDir ) )
      preloadString =":".join( lib for lib in preadloadLibs if ".so" in lib )
      script.write( 'declare -x LD_LIBRARY_PATH=%s:${LD_LIBRARY_PATH}\n' % preloadDir )
      script.write( 'declare -x LD_PRELOAD=%s\n' % preloadString )

    script.write('echo =========\n')
    script.write('env | sort >> localEnv.log\n')
    script.write('echo SLIC:\n')
    script.write("which slic\n")
    script.write('echo =========\n')
    comm = 'slic -P $PARTICLE_TBL -m %s %s\n' % (slicmac, self.extraCLIarguments)
    LOG.info("Command:", comm)
    script.write(comm)
    script.write('declare -x appstatus=$?\n')
    script.write('exit $appstatus\n')
    script.close()
    if os.path.exists(self.applicationLog): 
      os.remove(self.applicationLog)

    os.chmod(scriptName, 0o755)
    comm = 'sh -c "./%s"' % scriptName
    self.setApplicationStatus('SLIC %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
    self.stdError = ''
    self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520)
    #self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
    resultTuple = self.result['Value']
    if not os.path.exists(self.applicationLog):
      LOG.error("Something went terribly wrong, the log file is not present")
      self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
      if not self.ignoreapperrors:
        return S_ERROR('%s did not produce the expected log' % (self.applicationName))
    status = resultTuple[0]
    # stdOutput = resultTuple[1]
    # stdError = resultTuple[2]
    LOG.info("Status after the application execution is %s" % str(status))

    return self.finalStatusReport(status)
Exemplo n.º 18
0
    def execute(self):
        """ Execute the module, called by JobAgent
    """
        # Get input variables

        self.result = self.resolveInputVariables()
        # Checks

        if not self.platform:
            self.result = S_ERROR('No ILC platform selected')

        if not self.result['OK']:
            self.log.error("Failed to resolve input parameters:",
                           self.result["Message"])
            return self.result

        if len(self.InputFile):
            res = resolveIFpaths(self.InputFile)
            if not res['OK']:
                self.log.error("Cannot find input file")
                self.setApplicationStatus(
                    'StdHepSplit: missing input stdhep file')
                return S_ERROR('Missing stdhep file!')
            runonstdhep = res['Value'][0]
        else:
            self.log.warn("No files found to split")
            return S_OK("No files found to process")

        prefix = ''
        if self.OutputFile:
            prefix = self.OutputFile.split('.stdhep')[0]

        else:
            prefix = "this_split"
        #because we need to make sure the files end up in the base directory at the end
        self.OutputFile = prefix

        self.log.info("Will rename all files using '%s' as base." % prefix)

        # Setting up script
        res = getSoftwareFolder(self.platform, "stdhepsplit",
                                self.applicationVersion)
        if not res['OK']:
            self.log.error("Failed to find the software")
            self.setApplicationStatus(
                'StdHepSplit: Could not find neither local area not shared area install'
            )
            return res

        mysplitDir = res['Value']
        new_ld_lib = getNewLDLibs(self.platform, "stdhepsplit",
                                  self.applicationVersion)
        LD_LIBRARY_PATH = os.path.join(mysplitDir, "lib") + ":" + new_ld_lib

        maxReadString = ''
        ## hepsplit has a off-by-one error so we add 1 to max read to actually read maxRead events
        if self.maxRead:
            self.maxRead += 1
            maxReadString = " --maxread %s" % self.maxRead

        scriptContent = """
#!/bin/sh

################################################################################
# Dynamically generated script by LCIOConcatenate module                       #
################################################################################

declare -x LD_LIBRARY_PATH=%s

%s/hepsplit --infile %s --nw_per_file %s --outpref %s%s

exit $?

""" % (
            LD_LIBRARY_PATH,
            mysplitDir,
            runonstdhep,
            self.nbEventsPerSlice,
            prefix,
            maxReadString,
        )

        # Write script to file

        scriptPath = 'StdHepSplit_%s_Run_%s.tcl' % (self.applicationVersion,
                                                    self.STEP_NUMBER)

        if os.path.exists(scriptPath):
            os.remove(scriptPath)

        script = open(scriptPath, 'w')
        script.write(scriptContent)
        script.close()

        # Setup log file for application stdout

        if os.path.exists(self.applicationLog):
            os.remove(self.applicationLog)

        # Run code

        os.chmod(scriptPath, 0755)

        command = '"./%s"' % (scriptPath)

        self.setApplicationStatus('StdHepSplit %s step %s' %
                                  (self.applicationVersion, self.STEP_NUMBER))
        self.stdError = ''

        self.result = shellCall(0,
                                command,
                                callbackFunction=self.redirectLogOutput,
                                bufferLimit=20971520)

        resultTuple = self.result['Value']
        status = resultTuple[0]

        if not os.path.exists(self.applicationLog):
            self.log.error("Cannot access log file, cannot proceed")
            return S_ERROR("Failed reading the log file")

        with open(self.applicationLog, "r") as logf:
            numberofeventsdict = {}
            fname = ''
            for line in logf:
                line = line.rstrip()
                if line.count('Open output file'):
                    fname = line.split()[-1].rstrip().rstrip("\0")
                    numberofeventsdict[fname] = 0
                elif line.count(
                        "Record") and not line.count('Output Begin Run'):
                    #print line
                    val = line.split("=")[1].rstrip().lstrip()
                    if val != '0':
                        numberofeventsdict[fname] = int(val)

        self.log.verbose("numberofeventsdict dict: %s" % numberofeventsdict)

        ##Now update the workflow_commons dict with the relation between filename and number of events:
        #needed for the registerOutputData
        self.workflow_commons[
            'file_number_of_event_relation'] = numberofeventsdict
        if self.listoutput:
            outputlist = []
            for of in numberofeventsdict:
                item = {}
                item['outputFile'] = of
                item['outputPath'] = self.listoutput['outputPath']
                item['outputDataSE'] = self.listoutput['outputDataSE']
                outputlist.append(item)
            self.step_commons['listoutput'] = outputlist

        #Not only the step_commons must be updated
        if 'ProductionOutputData' in self.workflow_commons:
            proddata = self.workflow_commons['ProductionOutputData'].split(";")
            finalproddata = []
            this_split_data = ''
            for item in proddata:
                if not item.count(prefix):
                    finalproddata.append(item)
                else:
                    this_split_data = item
            path = os.path.dirname(this_split_data)
            for of in numberofeventsdict:
                finalproddata.append(os.path.join(path, of))
            self.workflow_commons['ProductionOutputData'] = ";".join(
                finalproddata)

        self.log.info("Status after the application execution is %s" %
                      str(status))
        if status == 2:
            self.log.info("Reached end of input file")
            status = 0

        self.listDir()
        return self.finalStatusReport(status)
Exemplo n.º 19
0
  def runIt(self):
    """ Called from ModuleBase
    """
    self.result = S_OK()
    if not self.platform:
      self.result = S_ERROR( 'No ILC platform selected' )
    elif not self.applicationLog:
      self.result = S_ERROR( 'No Log file provided' )
    if not self.result['OK']:
      self.log.error("Failed to resolve input parameters:", self.result["Message"])
      return self.result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('SLIC Pandora should not proceed as previous step did not end properly')
    
    ###Get the env script
    res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript)
    if not res['OK']:
      self.log.error("Failed to get the environment script:", res["Message"])
      return res
    env_script_path = res["Value"]
    
    res = resolveIFpaths(self.InputFile)
    if not res['OK']:
      self.log.error('Could not find input files')
      self.setApplicationStatus('SLICPandora: missing slcio file')
      return S_ERROR('Missing slcio file!')
    runonslcio = res['Value'][0]
    
    if not self.detectorxml.count(".xml") or not os.path.exists(os.path.basename(self.detectorxml)):
      detmodel = self.detectorxml.replace("_pandora.xml", "")
      if os.path.exists(detmodel + ".zip"):
        try:
          unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd())
        except (RuntimeError, OSError) as err:
          self.log.error("Exception when unpacking detectormodel zip file:", str(err))
          os.unlink(detmodel + ".zip") 
      if not os.path.exists(detmodel + ".zip"):  
        #retrieve detector model from web
        detector_urls = self.ops.getValue('/SLICweb/SLICDetectorModels', [''])
        if len(detector_urls[0]) < 1:
          self.log.error('Could not find in CS the URL for detector model')
          return S_ERROR('Could not find in CS the URL for detector model')

        for detector_url in detector_urls:
          try:
            urllib.urlretrieve("%s%s"%(detector_url, detmodel + ".zip"), detmodel + ".zip")
          except IOError:
            self.log.error("Download of detector model failed")
            continue
          try:
            unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd())
            break
          except (RuntimeError, OSError) as err:
            self.log.error("Exception for zip file obtained from ", detector_url)
            self.log.error("Exception:", str(err))
            os.unlink(detmodel + ".zip")
            continue
      #if os.path.exists(detmodel): #and os.path.isdir(detmodel):
      self.detectorxml = os.path.join(os.getcwd(), self.detectorxml)
      self.detectorxml = self.detectorxml + "_pandora.xml"
    
    if not os.path.exists(self.detectorxml):
      self.log.error('Detector model xml %s was not found, exiting' % self.detectorxml)
      return S_ERROR('Detector model xml was not found, exiting')
    
    oldversion = False
    if self.applicationVersion in ['CLIC_CDR', 'CDR1', 'CDR2', 'CDR0', 'V2', 'V3', 'V4']:
      oldversion = True
    
    scriptName = 'SLICPandora_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
    if os.path.exists(scriptName): 
      os.remove(scriptName)
    script = open(scriptName, 'w')
    script.write('#!/bin/bash \n')
    script.write('#####################################################################\n')
    script.write('# Dynamically generated script to run a production or analysis job. #\n')
    script.write('#####################################################################\n')
    script.write("source %s\n" % env_script_path)
    script.write("declare -x file=./Settings/%s\n" % self.pandorasettings)
    script.write("""
if [ -e "${file}" ]
then
   declare -x PANDORASETTINGS=$file
else
  if [ -d "${PANDORASETTINGSDIR}" ]
  then
    cp $PANDORASETTINGSDIR/*.xml .
    declare -x PANDORASETTINGS=%s
  fi
fi
if [ ! -e "${PANDORASETTINGS}" ]
then
  echo "Missing PandoraSettings file"
  exit 1
fi  
""" % self.pandorasettings )
    script.write('echo =============================\n')
    script.write('echo PATH is \n')
    script.write('echo $PATH | tr ":" "\n"  \n')
    script.write('echo ==============\n')
    if os.path.exists("./lib"):
      script.write('declare -x LD_LIBRARY_PATH=./lib:$LD_LIBRARY_PATH\n')
    script.write('echo =============================\n')
    script.write('echo LD_LIBRARY_PATH is \n')
    script.write('echo $LD_LIBRARY_PATH | tr ":" "\n"\n')
    script.write('echo ============================= \n')
    script.write('env | sort >> localEnv.log\n')
    #Now run it
    if oldversion:
      comm = 'PandoraFrontend %s $PANDORASETTINGS %s %s %s' % (self.detectorxml, runonslcio, 
                                                               self.OutputFile, str(self.NumberOfEvents))
    else:
      comm = 'PandoraFrontend -g %s -c $PANDORASETTINGS -i %s -o %s -r %s' % (self.detectorxml, 
                                                                              runonslcio, self.OutputFile, 
                                                                              str(self.NumberOfEvents))
    comm = "%s %s\n" % (comm, self.extraCLIarguments)
    self.log.info("Will run %s" % comm)
    script.write(comm)
    script.write('declare -x appstatus=$?\n')
    #script.write('where\n')
    #script.write('quit\n')
    #script.write('EOF\n')
    script.write('exit $appstatus\n')

    script.close()
    if os.path.exists(self.applicationLog): 
      os.remove(self.applicationLog)

    os.chmod(scriptName, 0755)
    comm = 'sh -c "./%s"' % (scriptName)
    self.setApplicationStatus('SLICPandora %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
    self.stdError = ''
    self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput,
                            bufferLimit = 20971520)
    #self.result = {'OK':True,'Value':(0,'Disabled Execution','')}
    resultTuple = self.result['Value']
    if resultTuple[0]:
      self.log.error("There was an error during the execution")
      
    if not os.path.exists(self.applicationLog):
      self.log.error("Something went terribly wrong, the log file is not present")
      self.setApplicationStatus('%s failed terribly, you are doomed!' % (self.applicationName))
      if not self.ignoreapperrors:
        return S_ERROR('%s did not produce the expected log' % (self.applicationName))
      
    logf = open(self.applicationLog,'r')
    if "Missing PandoraSettings file" in logf.readlines()[-1]:
      self.log.error("Issue with the Pandora Settings file.")
    logf.close()
    
    status = resultTuple[0]
    # stdOutput = resultTuple[1]
    # stdError = resultTuple[2]
    self.log.info( "Status after the application execution is %s" % str( status ) )

    return self.finalStatusReport(status)
Exemplo n.º 20
0
    def runIt(self):
        """
    Called by JobAgent
    
    Execute the following:
      - prepend in the LD_LIBRARY_PATH any lib directory of any dependency (e.g. root)
      - prepare the list of files to run on
      - set the cacheDirectory and put in there the alias.properties
      - set the lcsim file using :any:`prepareLCSIMFile`
      - run java and catch the exit code

    :return: S_OK(), S_ERROR()
    """
        self.result = S_OK()
        if not self.platform:
            self.result = S_ERROR('No ILC platform selected')
        elif not self.applicationLog:
            self.result = S_ERROR('No Log file provided')
        if not self.result['OK']:
            self.log.error("Failed to resolve input parameters:",
                           self.result["Message"])
            return self.result

        if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
            self.log.verbose(
                'Workflow status = %s, step status = %s' %
                (self.workflowStatus['OK'], self.stepStatus['OK']))
            return S_OK(
                'LCSIM should not proceed as previous step did not end properly'
            )

        res = getSoftwareFolder(self.platform, self.applicationName,
                                self.applicationVersion)
        if not res['OK']:
            self.log.error(
                'LCSIM was not found in either the local area or shared area:',
                res['Message'])
            return res
        lcsim_name = res['Value']
        ##Need to fetch the new LD_LIBRARY_PATH
        new_ld_lib_path = getNewLDLibs(self.platform, self.applicationName,
                                       self.applicationVersion)

        runonslcio = []
        if len(self.InputFile):
            res = resolveIFpaths(self.InputFile)
            if not res['OK']:
                self.setApplicationStatus('LCSIM: missing input slcio file')
                return S_ERROR('Missing slcio file!')
            runonslcio = res['Value']
        #for inputfile in inputfilelist:
        #  self.log.verbose("Will try using %s"%(os.path.basename(inputfile)))
        #  runonslcio.append(os.path.join(os.getcwd(),os.path.basename(inputfile)))

        retMod = self.downloadDetectorZip()
        if not retMod:
            return retMod

        ##Collect jar files to put in classspath
        jars = []
        if os.path.exists("lib"):
            for libs in os.listdir("lib"):
                if os.path.basename(libs).find(".jar") > 0:
                    jars.append(os.path.abspath(os.path.join("lib", libs)))
            new_ld_lib_path = "./lib:%s" % new_ld_lib_path
            #Remove any libc remaining in .lib
            removeLibc("./lib")

        ###Define cache directory as local folder
        aliasproperties = os.path.basename(self.aliasproperties)
        cachedir = os.getcwd()
        if not os.path.isdir(os.path.join(cachedir, ".lcsim")):
            try:
                os.mkdir(os.path.join(cachedir, ".lcsim"))
            except OSError, x:
                self.log.error("Could not create .lcsim folder !", str(x))
Exemplo n.º 21
0
  def execute(self):
    """ Run the module
    """
    result = self.resolveInputVariables()
    if not result['OK']:
      return result
    self.result = S_OK()

    if not self.applicationLog:
      self.result = S_ERROR( 'No Log file provided' )

    if not self.result['OK']:
      return self.result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      LOG.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('%s should not proceed as previous step did not end properly'% self.applicationName)

    ### Now remove the files in the FC
    lfns = self.InputFile
    
    ##Check that all the files are here:
    res = resolveIFpaths(lfns)
    if not res['OK']:
      LOG.error(res['Message'])
      return S_ERROR("Failed to find a file locally")
    
    #All files are here and available
    paths = res['Value']
    localpaths = []
    for inputfile in paths:
      basename = os.path.basename(inputfile)
      locname = os.path.join(os.getcwd(),basename)
      if not locname == inputfile:
        try:
          shutil.copy(inputfile, locname)
        except shutil.Error:
          LOG.error("Failed to copy file locally, will have to stop")
          return S_ERROR("Failed copy to local directory")
      localpaths.append(locname)
      try:
        os.unlink(inputfile)
      except OSError:
        LOG.warn("Failed to remove initial file, increased \
        disk space usage")
        
    #all the files are in the run directory 
    
    #get all metadata, ancestor/daughter relations, etc. for all the files
    
    #Update the listoutput
    if self.listoutput:
      outputlist = []
      for localFile in localpaths:
        item = {}
        item['outputFile'] = localFile
        item['outputPath'] = self.listoutput['outputPath']
        item['outputDataSE'] = self.listoutput['outputDataSE']
        outputlist.append(item)
      if self.enable:
        self.step_commons['listoutput'] = outputlist
      else:
        LOG.info("listoutput would have been ", outputlist)

    ## Make sure the path contains / at the end as we are going to 
    ## concatenate final path and local files
    if not self.outputpath[-1]=='/':
      self.outputpath += "/"

    if 'ProductionOutputData' in self.workflow_commons:
      file_list = ";".join([self.outputpath+name for name in [os.path.basename(fin) for fin in localpaths]])
      if self.enable:
        self.workflow_commons['ProductionOutputData'] = file_list
      else:
        LOG.info("ProductionOutputData would have been", file_list)
      
    #Now remove them
    if self.enable:
      res = self.repMan.removeFile(lfns, force=True)
      if not res['OK']:
        LOG.error("Failed to remove the files")
        self.setApplicationStatus("Failed to remove the file")
        #return S_ERROR("Failed to remove the files")
    else:
      LOG.info("Would have removed: ", "%s" % str(lfns))
    
    ## Now the files are not on the storage anymore, they exist only locally. We can hope 
    ## that the job will not be killed between now and the time the UploadOutputData module
    ## is called
    
    return self.finalStatusReport(0)
Exemplo n.º 22
0
  def execute(self):
    """ Execute the module, called by JobAgent
    """
    # Get input variables

    self.result = self.resolveInputVariables()
    # Checks

    if not self.systemConfig:
      self.result = S_ERROR( 'No ILC platform selected' )

    if not self.result['OK']:
      return self.result

    if not os.environ.has_key("LCIO"):
      self.log.error("Environment variable LCIO was not defined, cannot do anything")
      return S_ERROR("Environment variable LCIO was not defined, cannot do anything")

    if len(self.InputFile):
      res = resolveIFpaths(self.InputFile)
      if not res['OK']:
        self.setApplicationStatus('LCIOSplit: missing input slcio file')
        return S_ERROR('Missing slcio file!')
      runonslcio = res['Value'][0]
    else:
      return S_OK("No files found to process")
    # removeLibc

    removeLibc( os.path.join( os.environ["LCIO"], "lib" ) )

    # Setting up script

    LD_LIBRARY_PATH = os.path.join( "$LCIO", "lib" )
    if os.environ.has_key('LD_LIBRARY_PATH'):
      LD_LIBRARY_PATH += ":" + os.environ['LD_LIBRARY_PATH']

    PATH = "$LCIO/bin"
    if os.environ.has_key('PATH'):
      PATH += ":" + os.environ['PATH']

    scriptContent = """
#!/bin/sh

################################################################################
# Dynamically generated script by LCIOConcatenate module                       #
################################################################################

declare -x LD_LIBRARY_PATH=%s
declare -x PATH=%s

lcio split -i %s -n %s

exit $?

""" % (
    LD_LIBRARY_PATH,
    PATH,
    runonslcio,
    self.nbEventsPerSlice
)

    # Write script to file

    scriptPath = 'LCIOSplit_%s_Run_%s.tcl' % ( self.applicationVersion, self.STEP_NUMBER )

    if os.path.exists(scriptPath):
      os.remove(scriptPath)

    script = open( scriptPath, 'w' )
    script.write( scriptContent )
    script.close()

    # Setup log file for application stdout

    if os.path.exists(self.applicationLog):
      os.remove(self.applicationLog)

    # Run code

    os.chmod( scriptPath, 0755 )

    command = '"./%s"' % ( scriptPath )

    self.setApplicationStatus( 'LCIOSplit %s step %s' % ( self.applicationVersion, self.STEP_NUMBER ) )
    self.stdError = ''

    self.result = shellCall(
                            0,
                            command,
                            callbackFunction = self.redirectLogOutput,
                            bufferLimit = 20971520
                            )

        # Check results

    resultTuple = self.result['Value']
    status      = resultTuple[0]

    if not os.path.exists(self.applicationLog):
      self.log.error("Cannot access log file, cannot proceed")
      return S_ERROR("Failed reading the log file")

    logf = file(self.applicationLog,"r")
    baseinputfilename = os.path.basename(runonslcio).split(".slcio")[0]
    output_file_base_name = ''
    if self.OutputFile:
      output_file_base_name = self.OutputFile.split('.slcio')[0]
    self.log.info("Will rename all files using '%s' as base." % output_file_base_name)
    numberofeventsdict = {}
    fname = ''
    for line in logf:
      line = line.rstrip()
      if line.count(baseinputfilename):
        #First, we need to rename those guys
        current_file = os.path.basename(line).replace(".slcio", "")
        current_file_extension = current_file.replace(baseinputfilename, "")
        newfile = output_file_base_name + current_file_extension + ".slcio"
        os.rename(line, newfile)
        fname = newfile
        numberofeventsdict[fname] = 0
      elif line.count("events"):
        numberofeventsdict[fname] = int(line.split()[0])
    
    self.log.verbose("numberofeventsdict dict: %s" % numberofeventsdict)   

    ##Now update the workflow_commons dict with the relation between filename and number of events: needed for 
    #the registerOutputData
    self.workflow_commons['file_number_of_event_relation'] = numberofeventsdict
    if self.listoutput:
      outputlist = []
      for f in numberofeventsdict.keys():
        item = {}
        item['outputFile'] = f
        item['outputPath'] = self.listoutput['outputPath']
        item['outputDataSE'] = self.listoutput['outputDataSE']
        outputlist.append(item)
      self.step_commons['listoutput'] = outputlist
      
    #Not only the step_commons must be updated  
    if self.workflow_commons.has_key('ProductionOutputData'):
      proddata = self.workflow_commons['ProductionOutputData'].split(";")
      finalproddata = []
      this_split_data = ''
      for item in proddata:
        if not item.count(output_file_base_name):
          finalproddata.append(item)
        else:
          this_split_data = item
      path = os.path.dirname(this_split_data)
      for f in numberofeventsdict.keys():
        finalproddata.append(os.path.join(path, f))
      self.workflow_commons['ProductionOutputData'] = ";".join(finalproddata)  
    
    self.log.info( "Status after the application execution is %s" % str( status ) )

    return self.finalStatusReport(status)
Exemplo n.º 23
0
  def runIt(self):
    """
    Called by JobAgent

    Execute the following:
      - get the environment variables that should have been set during installation
      - find the detector model xml, using CS query to obtain the path
      - prepare the steering file and command line parameters
      - run DDSim on this steering file and catch the exit status

    :return: S_OK(), S_ERROR()
    """
    self.result = S_OK()
    if not self.platform:
      self.result = S_ERROR( 'No ILC platform selected' )
    elif not self.applicationLog:
      self.result = S_ERROR( 'No Log file provided' )
    if not self.result['OK']:
      self.log.error("Failed to resolve input parameters:", self.result['Message'])
      return self.result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('DDSim should not proceed as previous step did not end properly')

    ##TODO: Setup LD_LIBRARY_PATH for extensions
    res = getEnvironmentScript(self.platform, self.applicationName, self.applicationVersion, self.getEnvScript)
    if not res['OK']:
      self.log.error("Could not obtain the environment script: ", res["Message"])
      return res
    envScriptPath = res["Value"]

    #get the path to the detector model, either local or from the software
    resXML = self._getDetectorXML()
    if not resXML['OK']:
      self.log.error("Could not obtain the detector XML file: ", resXML["Message"])
      return resXML
    compactFile = resXML['Value']

    if len(self.InputFile):
      res = resolveIFpaths(self.InputFile)
      if not res['OK']:
        self.log.error("Generator file not found")
        return res
      self.InputFile = res['Value']

    ## if steering file is set try to find it
    if len(self.SteeringFile) > 0:
      self.SteeringFile = os.path.basename(self.SteeringFile)
      if not os.path.exists(self.SteeringFile):
        res = getSteeringFileDirName(self.platform, self.applicationName, self.applicationVersion)
        if not res['OK']:
          self.log.error("Could not find where the steering files are")
          return res
        steeringfiledirname = res['Value']
        if os.path.exists(os.path.join(steeringfiledirname, self.SteeringFile)):
          self.SteeringFile = os.path.join(steeringfiledirname, self.SteeringFile)
      if not os.path.exists(self.SteeringFile):
        self.log.error("Missing steering file")
        return S_ERROR("Could not find steering file")
      self.extraCLIarguments += " --steeringFile %s " % self.SteeringFile

    if self.startFrom:
      self.extraCLIarguments += " --skipNEvents %s " % self.startFrom

    if self.debug:
      self.extraCLIarguments += " --printLevel DEBUG "

    ##Same as for mokka: using ParticleGun does not imply InputFile
    if self.InputFile:
      self.InputFile = [self.InputFile] if isinstance(self.InputFile, basestring) else self.InputFile
      self.extraCLIarguments += " --inputFile %s " % self.InputFile[0]

    if self.NumberOfEvents:
      self.extraCLIarguments += " --numberOfEvents %s " % self.NumberOfEvents

    self.extraCLIarguments += " --random.seed %s " % self.randomSeed

    if self.OutputFile:
      self.extraCLIarguments += " --outputFile %s " % self.OutputFile

    scriptName = 'DDSim_%s_Run_%s.sh' % (self.applicationVersion, self.STEP_NUMBER)
    if os.path.exists(scriptName):
      os.remove(scriptName)
    script = []
    script.append('#!/bin/bash')
    script.append('#####################################################################')
    script.append('# Dynamically generated script to run a production or analysis job. #')
    script.append('#####################################################################')
    script.append('source %s' % envScriptPath)
    script.append('echo =========')
    script.append('env | sort >> localEnv.log')
    script.append('echo ddsim:`which ddsim`')
    script.append('echo =========')
    comm = 'ddsim --compactFile %(compactFile)s %(extraArgs)s' % dict(compactFile=compactFile,
                                                                      extraArgs=self.extraCLIarguments)
    self.log.info("Command:", comm)
    script.append(comm)
    script.append('declare -x appstatus=$?')
    script.append('exit $appstatus')

    with open(scriptName, 'w') as scriptFile:
      scriptFile.write( "\n".join(script) )

    if os.path.exists(self.applicationLog):
      os.remove(self.applicationLog)

    os.chmod(scriptName, 0755)
    comm = 'bash "./%s"' % scriptName
    self.setApplicationStatus('DDSim %s step %s' % (self.applicationVersion, self.STEP_NUMBER))
    self.stdError = ''
    self.result = shellCall(0, comm, callbackFunction = self.redirectLogOutput, bufferLimit = 20971520)
    resultTuple = self.result['Value']
    if not os.path.exists(self.applicationLog):
      self.log.error("Something went terribly wrong, the log file is not present")
      self.setApplicationStatus('%s failed to produce log file' % (self.applicationName))
      if not self.ignoreapperrors:
        return S_ERROR('%s did not produce the expected log %s' % (self.applicationName, self.applicationLog))
    status = resultTuple[0]

    self.log.info( "Status after the application execution is %s" % status )

    return self.finalStatusReport(status)
Exemplo n.º 24
0
  def execute(self):
    """ Called from Workflow
    """
    self.result = self.resolveInputVariables()
    if not self.systemConfig:
      self.result = S_ERROR( 'No ILC platform selected' )
    elif not self.applicationLog:
      self.result = S_ERROR( 'No Log file provided' )
    if not self.result['OK']:
      return self.result

    if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
      self.log.verbose('Workflow status = %s, step status = %s' %(self.workflowStatus['OK'], self.stepStatus['OK']))
      return S_OK('SLIC Pandora should not proceed as previous step did not end properly')
    
    slicPandoraDir = self.ops.getValue('/AvailableTarBalls/%s/%s/%s/TarBall' % (self.systemConfig, 
                                                                                "slicpandora", 
                                                                                self.applicationVersion), '')
    slicPandoraDir = slicPandoraDir.replace(".tgz", "").replace(".tar.gz", "")
    res = getSoftwareFolder(slicPandoraDir)
    if not res['OK']:
      self.setApplicationStatus('SLICPandora: Could not find neither local area not shared area install')
      return res
    myslicPandoraDir = res['Value']

    ##Remove libc lib
    removeLibc(myslicPandoraDir + "/LDLibs")

    ##Need to fetch the new LD_LIBRARY_PATH
    new_ld_lib_path = GetNewLDLibs(self.systemConfig, "slicpandora", self.applicationVersion)

    new_path = GetNewPATH(self.systemConfig, "slicpandora", self.applicationVersion)

    res = resolveIFpaths(self.InputFile)
    if not res['OK']:
      self.setApplicationStatus('SLICPandora: missing slcio file')
      return S_ERROR('Missing slcio file!')
    runonslcio = res['Value'][0]
    
    if not self.detectorxml.count(".xml") or not os.path.exists(os.path.basename(self.detectorxml)):
      detmodel = self.detectorxml.replace("_pandora.xml", "")
      if os.path.exists(detmodel + ".zip"):
        try:
          unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd())
        except:
          os.unlink(detmodel + ".zip") 
      if not os.path.exists(detmodel + ".zip"):  
        #retrieve detector model from web
        detector_urls = self.ops.getValue('/SLICweb/SLICDetectorModels', [''])
        if len(detector_urls[0]) < 1:
          self.log.error('Could not find in CS the URL for detector model')
          return S_ERROR('Could not find in CS the URL for detector model')

        for detector_url in detector_urls:
          try:
            urllib.urlretrieve("%s%s"%(detector_url, detmodel + ".zip"), detmodel + ".zip")
          except:
            self.log.error("Download of detector model failed")
            continue
          try:
            unzip_file_into_dir(open(detmodel + ".zip"), os.getcwd())
            break
          except:
            os.unlink(detmodel + ".zip")
            continue
      #if os.path.exists(detmodel): #and os.path.isdir(detmodel):
      self.detectorxml = os.path.join(os.getcwd(), self.detectorxml)
      self.detectorxml = self.detectorxml + "_pandora.xml"
    
    if not os.path.exists(self.detectorxml):
      self.log.error('Detector model xml %s was not found, exiting' % self.detectorxml)
      return S_ERROR('Detector model xml %s was not found, exiting' % self.detectorxml)
    
    if not os.path.exists(self.pandorasettings):
      if os.path.exists("./Settings/%s" % self.pandorasettings):
        self.pandorasettings = "./Settings/%s" % self.pandorasettings
      elif os.path.exists(os.path.join(myslicPandoraDir, 'Settings')):
        xmllist = glob.glob(os.path.join(myslicPandoraDir, 'Settings', "*.xml"))
        for f in xmllist:
          try:
            shutil.copy(f, os.path.join(os.getcwd(), os.path.basename(f)))
          except Exception, x:
            self.log.error('Could not copy %s, exception: %s' % (f, str(x)))
            return S_ERROR('Could not copy PandoraSettings file')
      else:
        self.log.error("Failed to find PandoraSettings anywhere, possibly SLICPandora install broken")
        return S_ERROR("Failed to find PandoraSettings anywhere")