コード例 #1
0
  def submitBigDataJobs( self, endpoint, numBigDataJobsAllowed, runningSiteName, NameNode,
                         BigDataSoftware, BigDataSoftwareVersion, HLLName, HLLVersion,
                         PublicIP, Port, jobIds , runningEndPointName, JobName, User, dataset, UsePilot, IsInteractive ):
    """
    Big Data job submission with all the parameters of SITE and Job
    """

    self.log.info( 'Director:submitBigDataJobs:JobSubmisionProcess' )

    if ( numBigDataJobsAllowed <= 0 ):
      return S_ERROR( "Number of slots reached for %s in the NameNode " % runningSiteName, NameNode )
    if NameNode not in self.runningEndPoints[endpoint]['NameNode']:
      return S_ERROR( 'Unknown NameNode: %s' % NameNode )

    newJob = BigDataDB.insertBigDataJob( jobIds, JobName, Time.toString(), NameNode,
                                              runningSiteName, PublicIP, "", "",
                                              "", BigDataSoftware, BigDataSoftwareVersion, HLLName,
                                              HLLVersion, "Submitted" )

    self.log.info( 'Director:submitBigDataJobs:SubmitJob' )
    dictBDJobSubmitted = self._submitBigDataJobs( NameNode, Port, jobIds, PublicIP,
                                                  runningEndPointName, User, JobName, dataset, UsePilot, IsInteractive )

    if not dictBDJobSubmitted[ 'OK' ]:
      return dictBDJobSubmitted
    bdjobID = dictBDJobSubmitted['Value']
    result = BigDataDB.setHadoopID( jobIds, bdjobID )
    if not result[ 'OK' ]:
      S_ERROR( "BigData ID not updated" )

    result = BigDataDB.setIntoJobDBStatus( jobIds, "Submitted", "",
                                           runningSiteName, bdjobID )
    if not result[ 'OK' ]:
      S_ERROR( "JobDB of BigData Soft not updated" )

    self.log.info( 'Director:submitBigDataJobs:JobSubmitted' )

    return S_OK( "OK" )
コード例 #2
0
    def monitoring( self, loop, parentthread, output ):

      self.initialTiming = os.times()
      accountingReport = AccountingJob()
      accountingReport.setStartTime()

      numberJobsFlag = True
      numberJobs = 0
      numberStartedJobsDict = {}
      numberEndingJobsDict = {}

      job_pattern = re.compile( 'Job =.*?,' )
      job_pattern_2 = re.compile( 'Job =.*?\n' )
      jobid = int( re.split( "_", re.split( "/", output )[int( len( re.split( "/", output ) ) - 1 )] )[0] )

      cmd = '/bin/chmod 555 ' + self.getinfo
      returned = self.commandLaunch( cmd )

      while parentthread.isAlive():
        time.sleep( loop )
        if numberJobsFlag:
          cmd = self.getinfo + ' -c step1'
          returned = self.commandLaunch( cmd )
          self.log.info( 'InteractiveJobMonitorThread:step1:numJobs:', returned )
          if returned != None:
            if ( returned['Value'][1] != "" ):
              if re.split( "=", returned['Value'][1] )[1].strip().isdigit():
                numberJobs = int( re.split( "=", returned['Value'][1] )[1] )
            if ( numberJobs != 0 ):
              numberJobsFlag = False
              BigDataDB.setJobStatus( jobid, "Running" )
        else:
          cmd = self.getinfo + ' -c step2'
          returned = self.commandLaunch( cmd )
          self.log.info( 'InteractiveJobMonitorThread:step2:startedJobs:', returned )
          if returned != "":
            if ( returned['Value'][1] != "" ):
              startedJobs = job_pattern.findall( returned['Value'][1] )
              self.log.info( 'step2:startedJobs:', startedJobs )
          cmd = self.getinfo + ' -c step3'
          returned = self.commandLaunch( cmd )
          self.log.info( 'InteractiveJobMonitorThread:step3:endedJobs:', returned )
          if returned != "":
            if ( returned['Value'][1] != "" ):
              finishedJobs = job_pattern_2.findall( returned['Value'][1] )
              self.log.info( 'step3:finishedJobs:', finishedJobs )
              if ( len( finishedJobs ) == numberJobs ):
                BigDataDB.setJobStatus( jobid, "Done" )
                BigDataDB.setHadoopID( jobid, finishedJobs )
                self.__updateSandBox( jobid, output )

                #Update Accounting                
                EXECUTION_RESULT = {}
                EXECUTION_RESULT['CPU'] = []
                finalStat = os.times()
                for i in range( len( finalStat ) ):
                  EXECUTION_RESULT['CPU'].append( finalStat[i] - self.initialTiming[i] )
                utime, stime, cutime, cstime, elapsed = EXECUTION_RESULT['CPU']
                cpuTime = utime + stime + cutime + cstime
                execTime = elapsed
                result = jobDB.getJobAttributes( jobid )
                getting = result['Value']
                acData = {
                        'User' : getting['Owner'],
                        'UserGroup' : getting['OwnerGroup'],
                        'JobGroup' : 'cesga',
                        'JobType' : 'User',
                        'JobClass' : 'unknown',
                        'ProcessingType' : 'unknown',
                        'FinalMajorStatus' : getting['Status'],
                        'FinalMinorStatus' : getting['MinorStatus'],
                        'CPUTime' : cpuTime,
                        'Site' : getting['Site'],
                        # Based on the factor to convert raw CPU to Normalized units (based on the CPU Model)
                        'NormCPUTime' : 0,
                        'ExecTime' : cpuTime,
                        'InputDataSize' : 0,
                        'OutputDataSize' : 0,
                        'InputDataFiles' : 0,
                        'OutputDataFiles' : 0,
                        'DiskSpace' : 0,
                        'InputSandBoxSize' : 0,
                        'OutputSandBoxSize' : 0,
                        'ProcessedEvents' : 0
                        }
                accountingReport.setEndTime()
                accountingReport.setValuesFromDict( acData )
                result = accountingReport.commit()