def submitBigDataJobs( self, endpoint, numBigDataJobsAllowed, runningSiteName, NameNode,
BigDataSoftware, BigDataSoftwareVersion, HLLName, HLLVersion,
PublicIP, Port, jobIds , runningEndPointName, JobName, User, dataset, UsePilot, IsInteractive ):
"""
Big Data job submission with all the parameters of SITE and Job
"""
self.log.info( 'Director:submitBigDataJobs:JobSubmisionProcess' )
if ( numBigDataJobsAllowed <= 0 ):
return S_ERROR( "Number of slots reached for %s in the NameNode " % runningSiteName, NameNode )
if NameNode not in self.runningEndPoints[endpoint]['NameNode']:
return S_ERROR( 'Unknown NameNode: %s' % NameNode )
newJob = BigDataDB.insertBigDataJob( jobIds, JobName, Time.toString(), NameNode,
runningSiteName, PublicIP, "", "",
"", BigDataSoftware, BigDataSoftwareVersion, HLLName,
HLLVersion, "Submitted" )
self.log.info( 'Director:submitBigDataJobs:SubmitJob' )
dictBDJobSubmitted = self._submitBigDataJobs( NameNode, Port, jobIds, PublicIP,
runningEndPointName, User, JobName, dataset, UsePilot, IsInteractive )
if not dictBDJobSubmitted[ 'OK' ]:
return dictBDJobSubmitted
bdjobID = dictBDJobSubmitted['Value']
result = BigDataDB.setHadoopID( jobIds, bdjobID )
if not result[ 'OK' ]:
S_ERROR( "BigData ID not updated" )
result = BigDataDB.setIntoJobDBStatus( jobIds, "Submitted", "",
runningSiteName, bdjobID )
if not result[ 'OK' ]:
S_ERROR( "JobDB of BigData Soft not updated" )
self.log.info( 'Director:submitBigDataJobs:JobSubmitted' )
return S_OK( "OK" )
def monitoring( self, loop, parentthread, output ):
self.initialTiming = os.times()
accountingReport = AccountingJob()
accountingReport.setStartTime()
numberJobsFlag = True
numberJobs = 0
numberStartedJobsDict = {}
numberEndingJobsDict = {}
job_pattern = re.compile( 'Job =.*?,' )
job_pattern_2 = re.compile( 'Job =.*?\n' )
jobid = int( re.split( "_", re.split( "/", output )[int( len( re.split( "/", output ) ) - 1 )] )[0] )
cmd = '/bin/chmod 555 ' + self.getinfo
returned = self.commandLaunch( cmd )
while parentthread.isAlive():
time.sleep( loop )
if numberJobsFlag:
cmd = self.getinfo + ' -c step1'
returned = self.commandLaunch( cmd )
self.log.info( 'InteractiveJobMonitorThread:step1:numJobs:', returned )
if returned != None:
if ( returned['Value'][1] != "" ):
if re.split( "=", returned['Value'][1] )[1].strip().isdigit():
numberJobs = int( re.split( "=", returned['Value'][1] )[1] )
if ( numberJobs != 0 ):
numberJobsFlag = False
BigDataDB.setJobStatus( jobid, "Running" )
else:
cmd = self.getinfo + ' -c step2'
returned = self.commandLaunch( cmd )
self.log.info( 'InteractiveJobMonitorThread:step2:startedJobs:', returned )
if returned != "":
if ( returned['Value'][1] != "" ):
startedJobs = job_pattern.findall( returned['Value'][1] )
self.log.info( 'step2:startedJobs:', startedJobs )
cmd = self.getinfo + ' -c step3'
returned = self.commandLaunch( cmd )
self.log.info( 'InteractiveJobMonitorThread:step3:endedJobs:', returned )
if returned != "":
if ( returned['Value'][1] != "" ):
finishedJobs = job_pattern_2.findall( returned['Value'][1] )
self.log.info( 'step3:finishedJobs:', finishedJobs )
if ( len( finishedJobs ) == numberJobs ):
BigDataDB.setJobStatus( jobid, "Done" )
BigDataDB.setHadoopID( jobid, finishedJobs )
self.__updateSandBox( jobid, output )
#Update Accounting
EXECUTION_RESULT = {}
EXECUTION_RESULT['CPU'] = []
finalStat = os.times()
for i in range( len( finalStat ) ):
EXECUTION_RESULT['CPU'].append( finalStat[i] - self.initialTiming[i] )
utime, stime, cutime, cstime, elapsed = EXECUTION_RESULT['CPU']
cpuTime = utime + stime + cutime + cstime
execTime = elapsed
result = jobDB.getJobAttributes( jobid )
getting = result['Value']
acData = {
'User' : getting['Owner'],
'UserGroup' : getting['OwnerGroup'],
'JobGroup' : 'cesga',
'JobType' : 'User',
'JobClass' : 'unknown',
'ProcessingType' : 'unknown',
'FinalMajorStatus' : getting['Status'],
'FinalMinorStatus' : getting['MinorStatus'],
'CPUTime' : cpuTime,
'Site' : getting['Site'],
# Based on the factor to convert raw CPU to Normalized units (based on the CPU Model)
'NormCPUTime' : 0,
'ExecTime' : cpuTime,
'InputDataSize' : 0,
'OutputDataSize' : 0,
'InputDataFiles' : 0,
'OutputDataFiles' : 0,
'DiskSpace' : 0,
'InputSandBoxSize' : 0,
'OutputSandBoxSize' : 0,
'ProcessedEvents' : 0
}
accountingReport.setEndTime()
accountingReport.setValuesFromDict( acData )
result = accountingReport.commit()