def jobTimes(self,job):
creationt = job.get('creationTime')
startt = job.get('startTime')
jobstat = job.get('jobStatus')
endt = job.get('endTime')
transfert = timedelta(0)
duration = timedelta(0)
try:
if utils.isValid(endt) and utils.isValid(startt) and endt > startt:
duration = endt - startt
elif utils.isValid(endt) and utils.isValid(creationt):
duration = endt - creationt
elif utils.isValid(startt) and not jobstat in ('failed', 'finished', 'cancelled'):
duration = datetime.utcnow() - startt
else:
duration = timedelta(0)
except:
duration = timedelta(0)
if jobstat in ('finished', 'failed','cancelled'):
try:
transfert = job.get('modificationTime') - endt
except:
transfert = timedelta(0)
pass
else:
endt = job.get('modificationTime')
if utils.isValid(creationt) and utils.isValid(startt):
tostart = startt - creationt
elif utils.isValid(creationt) and utils.isValid(endt):
tostart = endt - creationt
elif utils.isValid(creationt) and not jobstat in ('failed', 'finished', 'cancelled'):
tostart = datetime.utcnow() - creationt
else:
tostart = timedelta(0)
return (duration,endt,tostart,transfert)
def jobsetTime(self,job):
# if not utils.isJob(job): return None
job['username'] = self.cleanUserID(job['prodUserID'])
# if job['creationTime'] == None or job['startTime'] == None or job['jobStatus'] == None or job['endTime'] ==None: return job
creationtDict = self.creationtDict()
maxDuration = self.maxDuration()
maxTransfer = self.maxTransfer()
jobsetid = self.setid(job)
crtime = job.get('creationTime')
job['tostart'] = timedelta(0)
job['duration'] = timedelta(0)
job['endt'] = None
job['transfert']= timedelta(0)
if crtime != None:
if not creationtDict.has_key(jobsetid):
creationtDict[jobsetid] = job['creationTime']
elif creationtDict[jobsetid] > job['creationTime']:
creationtDict[jobsetid] = job['creationTime']
(duration,endt,tostart,transfert) = self.jobTimes(job)
try:
if duration > maxDuration and job['jobStatus'] !='cancelled': self.maxDuration(duration)
except:
raise ValueError(" %s %s JOB: %s " % (duration,maxDuration, job) )
if utils.isValid(transfert) and transfert> maxTransfer : self.maxTransfer(transfert)
job['tostart'] = tostart
job['duration'] = duration
job['endt'] = endt
job['transfert']= transfert
return job
def getSiteDDM(self, job,sites,nicks):
""" Get DDM for site from schedconfig for logfiles """
def getDDM(site):
siteDDM=None
if site != None:
siteDDM = sites.get(site)
if siteDDM != None:
siteDDM =siteDDM['ddm']
if siteDDM == None:
siteDDM = nicks.get(site)
if siteDDM != None: siteDDM =siteDDM['ddm']
if siteDDM == None:
for nick in nicks:
if nick.startswith(site):
siteDDM = nicks[nick].get('ddm')
break
return siteDDM
ddmse = job.get('ddm')
if not utils.isFilled(ddmse):
computingElement = job.get('computingElement')
computingSite = job.get('computingSite')
pilotID = job.get('pilotID')
if computingSite != None and utils.isValid(pilotID) and pilotID.startswith('tp_') \
and computingElement != None and computingElement.find('.') < 0:
ddmse = getDDM(computingElement)
else:
ddmse = getDDM(computingSite)
if sites.has_key(ddmse) and sites[ddmse].has_key('dq2Site'): \
ddmse = sites[ddmse]['dq2Site']
job['ddmse'] = ddmse
return job
def jobRetry(self,job):
if self.isRetried(job):
diag = job.get('taskBufferErrorDiag')
if utils.isValid(diag):
pandaid=self._newRetried.search(diag)
if pandaid:
c = pandaid.group(2)
job['retryID'] = c
return job
def getErrors(cls,job,errdict,errorcodes):
""" collect the unique errors to publish """
for errcode in errorcodes.keys():
errval = 0
errval = job[errcode]
if errdict!=None:
ec = errdict.get(errcode)
if ec!=None:
ev = ec.get(errval)
if ev != None: continue
try:
if errval != 0 and errval != '0' and utils.isValid(errval):
errval = int(errval)
## set prompt to True if error code is 1178 ##
if errval == 1178: prompt = bool(1)
errdiag = errcode.replace('ErrorCode','ErrorDiag')
if errcode.find('ErrorCode') > 0:
diagtxt = job[errdiag]
else:
diagtxt = ''
if utils.isValid(diagtxt) and len(diagtxt) > 0:
desc = diagtxt
elif errorcodes[errcode].has_key(errval):
desc = errorcodes[errcode][errval]
else:
desc = "Unknown %s error code %s" % ( errcode, errval )
errname = errcode.replace('ErrorCode','')
errname = errname.replace('ExitCode','')
if errdict ==None:
errdict = {errcode: { errval : ( errname, desc ) }}
else:
errcd = errdict.get(errcode);
if errcd == None: errdict[errcode] = { errval : ( errname, desc ) }
else: errcd[errval]=( errname, desc )
except:
incident = "Wrong data: %s " % utils.reportError("Unexpected value '%s' of the '%s' error code" %(errval,errcode))
# utils.recordIncident(incident,'monalarm')
pass
return errdict
def fillJobsetId(self,job,merge=None):
analysis = utils.isJob(job)
username = job['username']
# jid = "jobsetID=<a href='%s?job=*&jobsetID=%s&user=%s'>%s</a>" % ( utils.monURL, job['jobsetID'],username, job['jobsetID'] )
jobsetid = self.setid(job)
jsetid = '%s:%s' % ( username, jobsetid )
## added to fix sorting bug ##
# if have_tmpuser == bool(0):
# if not str(utils.cleanUserID(job['prodUserID'])) == "":
# tmpuser = '******' % username
# have_tmpuser = bool(1)
# if have_tmpuser == bool(1):
# if not tmpuser == username:
# oneuser = bool(0)
# if not int(job['jobsetID']) in tmpjslist:
# tmpjslist.append(int(job['jobsetID']))
##############################
jsets = self.jobsets()
if not jsets.has_key('jobsets') : jsets['jobsets'] = {}
jobsets = jsets['jobsets']
creationtDict = self.creationtDict()
jobid = job['PandaID']
attempt = job.get('attemptNr',0)
if attempt>0 and job['parentID']==None and job['jobsetID']!=None: attempt = 0 # workaround issue #https://savannah.XXXXX.ch/bugs/index.php?91176#comment6
status = job.get('jobStatus')
if jsetid != None and not jobsets.has_key(jsetid):
jobsets[jsetid] = {}
jobsets[jsetid]['analysis'] = analysis
jobsets[jsetid]['workingGroup'] = job['workingGroup']
jobsets[jsetid]['user'] = username
jobsets[jsetid]['created'] = creationtDict[jobsetid]
jobsets[jsetid]['latest'] = job.get('modificationTime',creationtDict[jobsetid])
jobsets[jsetid]['site'] = job['computingSite']
jobsets[jsetid]['inDS'] = job['prodDBlock']
jobsets[jsetid]['outDS'] = job['destinationDBlock']
jobsets[jsetid]['mergingJobs'] = { }
jobsets[jsetid]['attempt'] = attempt
jobsets[jsetid]['jobs'] = {jobid: self.cleanjob(job) }
jobsets[jsetid]['retried'] = 1 if self.isRetried(job) else 0
else:
jbTime = job.get('modificationTime')
if jbTime != None and jobsets[jsetid]['latest'] < jbTime:
jobsets[jsetid]['latest'] = jbTime
jobsets[jsetid]['jobs'][jobid] = self.cleanjob(job)
if self.isRetried(job): jobsets[jsetid]['retried'] += 1
if jobsets[jsetid]['site'] != job['computingSite']:
jobsets[jsetid]['site'] = self.fgMultiSiteSetLabel;
jobsets[jsetid]['attempt'] += attempt+1
try:
if job['transformation'].find('buildJob') > 0:
jobsets[jsetid]['libDS'] = job['destinationDBlock']
jobsets[jsetid]['build'] = jobid
elif job['transformation'].find('runJob') > 0 or job['transformation'].find('runAthena') > 0:
# set color
pass
except:
pass
if job['prodSourceLabel'] == 'user' and job['processingType'] == 'usermerge':
mj = jobsets[jsetid]['mergingJobs']
mj['all'] = mj.get('all',0) + 1
mj[status] = mj.get(status,0) + 1
elif utils.isValid(merge):
try:
# check whether there is merging status for the job
tbuffer = pmt.checkMergeStatus( job['prodUserID'] ,job['jobDefinitionID'])
if tbuffer != None:
mj = jobsets[jsetid]['mergingJobs']
status = tbuffer['status']
if status in ( 'generated' ):
mj = jobsets[jsetid]['mergingJobs']
mj['all']=mj.get('all',0) + 1
mj[status] = mj.get(status,0) + 1
elif status in ('NA'):
pass
else:
mj[status] = len(tbuffer.get('mergeIDs',[]))
except:
pass