def filterDate( selectOutput, start, end ):
'''
Selects all the downtimes that meet the constraints of 'start' and 'end' dates
'''
downtimes = selectOutput
downtimesFiltered = []
if start is not None:
try:
start = Time.fromString( start )
except:
error( "datetime formt is incorrect, pls try [%Y-%m-%d[ %H:%M:%S]]" )
start = Time.toEpoch( start )
if end is not None:
try:
end = Time.fromString( end )
except:
error( "datetime formt is incorrect, pls try [%Y-%m-%d[ %H:%M:%S]]" )
end = Time.toEpoch( end )
if start is not None and end is not None:
for dt in downtimes:
dtStart = Time.toEpoch( dt[ 'startDate' ] )
dtEnd = Time.toEpoch( dt[ 'endDate' ] )
if ( dtStart >= start ) and ( dtEnd <= end ):
downtimesFiltered.append( dt )
elif start is not None and end is None:
for dt in downtimes:
dtStart = Time.toEpoch( dt[ 'startDate' ] )
if dtStart >= start:
downtimesFiltered.append( dt )
elif start is None and end is not None:
for dt in downtimes:
dtEnd = Time.toEpoch( dt[ 'endDate' ] )
if dtEnd <= end:
downtimesFiltered.append( dt )
else:
downtimesFiltered = downtimes
return downtimesFiltered
def addLoggingRecord(self,
jobID,
status='idem',
minor='idem',
application='idem',
date='',
source='Unknown'):
""" Add a new entry to the JobLoggingDB table. One, two or all the three status
components (major, minor, application) can be specified.
Optionally the time stamp of the status can
be provided in a form of a string in a format '%Y-%m-%d %H:%M:%S' or
as datetime.datetime object. If the time stamp is not provided the current
UTC time is used.
"""
event = 'status/minor/app=%s/%s/%s' % (status, minor, application)
self.gLogger.info("Adding record for job " + str(jobID) + ": '" + event + "' from " + source)
if not date:
# Make the UTC datetime string and float
_date = Time.dateTime()
epoc = time.mktime(_date.timetuple()) + _date.microsecond / 1000000. - MAGIC_EPOC_NUMBER
time_order = round(epoc, 3)
else:
try:
if isinstance(date, basestring):
# The date is provided as a string in UTC
_date = Time.fromString(date)
epoc = time.mktime(_date.timetuple()) + _date.microsecond / 1000000. - MAGIC_EPOC_NUMBER
time_order = round(epoc, 3)
elif isinstance(date, Time._dateTimeType):
_date = date
epoc = time.mktime(_date.timetuple()) + _date.microsecond / 1000000. - \
MAGIC_EPOC_NUMBER # pylint: disable=no-member
time_order = round(epoc, 3)
else:
self.gLogger.error('Incorrect date for the logging record')
_date = Time.dateTime()
epoc = time.mktime(_date.timetuple()) - MAGIC_EPOC_NUMBER
time_order = round(epoc, 3)
except BaseException:
self.gLogger.exception('Exception while date evaluation')
_date = Time.dateTime()
epoc = time.mktime(_date.timetuple()) - MAGIC_EPOC_NUMBER
time_order = round(epoc, 3)
cmd = "INSERT INTO LoggingInfo (JobId, Status, MinorStatus, ApplicationStatus, " + \
"StatusTime, StatusTimeOrder, StatusSource) VALUES (%d,'%s','%s','%s','%s',%f,'%s')" % \
(int(jobID), status, minor, application[:255],
str(_date), time_order, source)
return self._update(cmd)
def accountPilots( self, pilotsToAccount, connection ):
""" account for pilots
"""
accountingFlag = False
pae = self.am_getOption( 'PilotAccountingEnabled', 'yes' )
if pae.lower() == "yes":
accountingFlag = True
if not pilotsToAccount:
self.log.info( 'No pilots to Account' )
return S_OK()
accountingSent = False
if accountingFlag:
retVal = self.pilotDB.getPilotInfo( pilotsToAccount.keys(), conn = connection )
if not retVal['OK']:
self.log.error( 'Fail to retrieve Info for pilots', retVal['Message'] )
return retVal
dbData = retVal[ 'Value' ]
for pref in dbData:
if pref in pilotsToAccount:
if dbData[pref][ 'Status' ] not in self.finalStateList:
dbData[pref][ 'Status' ] = pilotsToAccount[pref][ 'Status' ]
dbData[pref][ 'DestinationSite' ] = pilotsToAccount[pref][ 'DestinationSite' ]
dbData[pref][ 'LastUpdateTime' ] = Time.fromString( pilotsToAccount[pref][ 'StatusDate' ] )
retVal = self.__addPilotsAccountingReport( dbData )
if not retVal['OK']:
self.log.error( 'Fail to retrieve Info for pilots', retVal['Message'] )
return retVal
self.log.info( "Sending accounting records..." )
retVal = gDataStoreClient.commit()
if not retVal[ 'OK' ]:
self.log.error( "Can't send accounting repots", retVal[ 'Message' ] )
else:
self.log.info( "Accounting sent for %s pilots" % len( pilotsToAccount ) )
accountingSent = True
if not accountingFlag or accountingSent:
for pRef in pilotsToAccount:
pDict = pilotsToAccount[pRef]
self.log.verbose( 'Setting Status for %s to %s' % ( pRef, pDict['Status'] ) )
self.pilotDB.setPilotStatus( pRef,
pDict['Status'],
pDict['DestinationSite'],
pDict['StatusDate'],
conn = connection )
return S_OK()
def _checkJobLastUpdateTime( self, joblist , StalledDays ):
timeLimitToConsider = Time.dateTime() - Time.day * StalledDays
ret = False
for JobID in joblist:
result = self.jobDB.getJobAttributes(int(JobID))
if result['OK']:
if result['Value'].has_key('LastUpdateTime'):
LastUpdateTime = result['Value']['LastUpdateTime']
if Time.fromString(LastUpdateTime) > timeLimitToConsider:
ret = True
self.log.debug('Since '+str(JobID)+' updates LastUpdateTime on '+str(LastUpdateTime)+', this does not to need to be deleted.')
break
else:
self.log.error("Error taking job info. from DB:%s" % str( result['Message'] ) )
return ret
def _checkJobLastUpdateTime( self, joblist , StalledDays ):
timeLimitToConsider = Time.dateTime() - Time.day * StalledDays
ret = False
for jobID in joblist:
result = self.jobDB.getJobAttributes( int( jobID ) )
if result['OK']:
if 'LastUpdateTime' in result['Value']:
lastUpdateTime = result['Value']['LastUpdateTime']
if Time.fromString( lastUpdateTime ) > timeLimitToConsider:
ret = True
self.log.debug( 'Since %s updates LastUpdateTime on %s this does not to need to be deleted.' % ( str( jobID ), str( lastUpdateTime ) ) )
break
else:
self.log.error( "Error taking job info from DB", result['Message'] )
return ret
def addLoggingRecord(self, jobID, status="idem", minor="idem", application="idem", date="", source="Unknown"):
""" Add a new entry to the JobLoggingDB table. One, two or all the three status
components can be specified. Optionaly the time stamp of the status can
be provided in a form of a string in a format '%Y-%m-%d %H:%M:%S' or
as datetime.datetime object. If the time stamp is not provided the current
UTC time is used.
"""
event = "status/minor/app=%s/%s/%s" % (status, minor, application)
self.gLogger.info("Adding record for job " + str(jobID) + ": '" + event + "' from " + source)
if not date:
# Make the UTC datetime string and float
_date = Time.dateTime()
epoc = time.mktime(_date.timetuple()) + _date.microsecond / 1000000.0 - MAGIC_EPOC_NUMBER
time_order = round(epoc, 3)
else:
try:
if type(date) in StringTypes:
# The date is provided as a string in UTC
_date = Time.fromString(date)
epoc = time.mktime(_date.timetuple()) + _date.microsecond / 1000000.0 - MAGIC_EPOC_NUMBER
time_order = round(epoc, 3)
elif type(date) == Time._dateTimeType:
_date = date
epoc = time.mktime(_date.timetuple()) + _date.microsecond / 1000000.0 - MAGIC_EPOC_NUMBER
time_order = round(epoc, 3)
else:
self.gLogger.error("Incorrect date for the logging record")
_date = Time.dateTime()
epoc = time.mktime(_date.timetuple()) - MAGIC_EPOC_NUMBER
time_order = round(epoc, 3)
except:
self.gLogger.exception("Exception while date evaluation")
_date = Time.dateTime()
epoc = time.mktime(_date.timetuple()) - MAGIC_EPOC_NUMBER
time_order = round(epoc, 3)
cmd = (
"INSERT INTO LoggingInfo (JobId, Status, MinorStatus, ApplicationStatus, "
+ "StatusTime, StatusTimeOrder, StatusSource) VALUES (%d,'%s','%s','%s','%s',%f,'%s')"
% (int(jobID), status, minor, application, str(_date), time_order, source)
)
return self._update(cmd)
def addFileRecord( self, lfns, status, minor = "Unknown", date = None, source = "Unknown" ):
""" Add a new entry to the DataLoggingDB table.
:warning: Optionally the time stamp of the status can be provided in a form of a string
in a format '%Y-%m-%d %H:%M:%S' or as datetime.datetime object. If the time stamp is not
provided the current UTC time is used.
:param self: self reference
:param list lfns: list of LFNs
:param str status: status
:param str minor: additional information
:param mixed date: date and time
:param str source: source setting the new status
"""
self.gLogger.info( "Entering records for %s lfns: %s/%s from source %s" % ( len( lfns ), status, minor, source ) )
if not date:
_date = Time.dateTime()
else:
if type( date ) in StringTypes:
_date = Time.fromString( date )
else:
_date = date
try:
time_order = Time.to2K( _date ) - NEW_MAGIC_EPOCH_2K
except AttributeError:
gLogger.error( 'Wrong date argument given using current time stamp' )
dateNow = Time.dateTime()
time_order = Time.to2K( dateNow ) - NEW_MAGIC_EPOCH_2K
inDict = { 'Status': status,
'MinorStatus': minor,
'StatusTime': _date,
'StatusTimeOrder': time_order,
'Source': source
}
result = S_OK( 0 )
for lfn in lfns:
inDict['LFN'] = lfn
res = self.insertFields( self.tableName, inDict = inDict )
if not res['OK']:
return res
result['Value'] += res['Value']
result['lastRowId'] = res['lastRowId']
return result
def tupleToMessage(varTuple):
varList = list(varTuple)
varList[2] = Time.fromString(varList[2])
return Message(*varList)
def __parseFormParams(self):
params = self.request.arguments
pD = {}
extraParams = {}
pinDates = False
for name in params:
if name.find("_") != 0:
continue
value = params[name][0]
name = name[1:]
pD[name] = str(value)
print pD
#Personalized title?
if 'plotTitle' in pD:
extraParams['plotTitle'] = pD['plotTitle']
del (pD['plotTitle'])
#Pin dates?
if 'pinDates' in pD:
pinDates = pD['pinDates']
del (pD['pinDates'])
pinDates = pinDates.lower() in ("yes", "y", "true", "1")
#Get plotname
if not 'grouping' in pD:
return S_ERROR("Missing grouping!")
grouping = pD['grouping']
#Get plotname
if not 'typeName' in pD:
return S_ERROR("Missing type name!")
typeName = pD['typeName']
del (pD['typeName'])
#Get plotname
if not 'plotName' in pD:
return S_ERROR("Missing plot name!")
reportName = pD['plotName']
del (pD['plotName'])
#Get times
if not 'timeSelector' in pD:
return S_ERROR("Missing time span!")
#Find the proper time!
pD['timeSelector'] = int(pD['timeSelector'])
if pD['timeSelector'] > 0:
end = Time.dateTime()
start = end - datetime.timedelta(seconds=pD['timeSelector'])
if not pinDates:
extraParams['lastSeconds'] = pD['timeSelector']
else:
if 'endTime' not in pD:
end = False
else:
end = Time.fromString(pD['endTime'])
del (pD['endTime'])
if 'startTime' not in pD:
return S_ERROR("Missing starTime!")
else:
start = Time.fromString(pD['startTime'])
del (pD['startTime'])
del (pD['timeSelector'])
for k in pD:
if k.find("ex_") == 0:
extraParams[k[3:]] = pD[k]
#Listify the rest
for selName in pD:
pD[selName] = List.fromChar(pD[selName], ",")
return S_OK(
(typeName, reportName, start, end, pD, grouping, extraParams))
def _setJobStatusBulk(cls, jobID, statusDict, force=False):
"""Set various status fields for job specified by its jobId.
Set only the last status in the JobDB, updating all the status
logging information in the JobLoggingDB. The statusDict has datetime
as a key and status information dictionary as values
"""
jobID = int(jobID)
log = gLogger.getLocalSubLogger("JobStatusBulk/Job-%d" % jobID)
result = cls.jobDB.getJobAttributes(
jobID, ["Status", "StartExecTime", "EndExecTime"])
if not result["OK"]:
return result
if not result["Value"]:
# if there is no matching Job it returns an empty dictionary
return S_ERROR("No Matching Job")
# If the current status is Stalled and we get an update, it should probably be "Running"
currentStatus = result["Value"]["Status"]
if currentStatus == JobStatus.STALLED:
currentStatus = JobStatus.RUNNING
startTime = result["Value"].get("StartExecTime")
endTime = result["Value"].get("EndExecTime")
# getJobAttributes only returns strings :(
if startTime == "None":
startTime = None
if endTime == "None":
endTime = None
# Remove useless items in order to make it simpler later, although there should not be any
for sDict in statusDict.values():
for item in sorted(sDict):
if not sDict[item]:
sDict.pop(item, None)
# Get the latest time stamps of major status updates
result = cls.jobLoggingDB.getWMSTimeStamps(int(jobID))
if not result["OK"]:
return result
if not result["Value"]:
return S_ERROR("No registered WMS timeStamps")
# This is more precise than "LastTime". timeStamps is a sorted list of tuples...
timeStamps = sorted((float(t), s) for s, t in result["Value"].items()
if s != "LastTime")
lastTime = Time.toString(Time.fromEpoch(timeStamps[-1][0]))
# Get chronological order of new updates
updateTimes = sorted(statusDict)
log.debug(
"*** New call ***", "Last update time %s - Sorted new times %s" %
(lastTime, updateTimes))
# Get the status (if any) at the time of the first update
newStat = ""
firstUpdate = Time.toEpoch(Time.fromString(updateTimes[0]))
for ts, st in timeStamps:
if firstUpdate >= ts:
newStat = st
# Pick up start and end times from all updates
for updTime in updateTimes:
sDict = statusDict[updTime]
newStat = sDict.get("Status", newStat)
if not startTime and newStat == JobStatus.RUNNING:
# Pick up the start date when the job starts running if not existing
startTime = updTime
log.debug("Set job start time", startTime)
elif not endTime and newStat in JobStatus.JOB_FINAL_STATES:
# Pick up the end time when the job is in a final status
endTime = updTime
log.debug("Set job end time", endTime)
# We should only update the status to the last one if its time stamp is more recent than the last update
attrNames = []
attrValues = []
if updateTimes[-1] >= lastTime:
minor = ""
application = ""
# Get the last status values looping on the most recent upupdateTimes in chronological order
for updTime in [dt for dt in updateTimes if dt >= lastTime]:
sDict = statusDict[updTime]
log.debug("\t",
"Time %s - Statuses %s" % (updTime, str(sDict)))
status = sDict.get("Status", currentStatus)
# evaluate the state machine if the status is changing
if not force and status != currentStatus:
res = JobStatus.JobsStateMachine(
currentStatus).getNextState(status)
if not res["OK"]:
return res
newStat = res["Value"]
# If the JobsStateMachine does not accept the candidate, don't update
if newStat != status:
# keeping the same status
log.error(
"Job Status Error",
"%s can't move from %s to %s: using %s" %
(jobID, currentStatus, status, newStat),
)
status = newStat
sDict["Status"] = newStat
# Change the source to indicate this is not what was requested
source = sDict.get("Source", "")
sDict["Source"] = source + "(SM)"
# at this stage status == newStat. Set currentStatus to this new status
currentStatus = newStat
minor = sDict.get("MinorStatus", minor)
application = sDict.get("ApplicationStatus", application)
log.debug(
"Final statuses:",
"status '%s', minor '%s', application '%s'" %
(status, minor, application))
if status:
attrNames.append("Status")
attrValues.append(status)
if minor:
attrNames.append("MinorStatus")
attrValues.append(minor)
if application:
attrNames.append("ApplicationStatus")
attrValues.append(application)
# Here we are forcing the update as it's always updating to the last status
result = cls.jobDB.setJobAttributes(jobID,
attrNames,
attrValues,
update=True,
force=True)
if not result["OK"]:
return result
# Update start and end time if needed
if endTime:
result = cls.jobDB.setEndExecTime(jobID, endTime)
if not result["OK"]:
return result
if startTime:
result = cls.jobDB.setStartExecTime(jobID, startTime)
if not result["OK"]:
return result
# Update the JobLoggingDB records
heartBeatTime = None
for updTime in updateTimes:
sDict = statusDict[updTime]
status = sDict.get("Status", "idem")
minor = sDict.get("MinorStatus", "idem")
application = sDict.get("ApplicationStatus", "idem")
source = sDict.get("Source", "Unknown")
result = cls.jobLoggingDB.addLoggingRecord(
jobID,
status=status,
minorStatus=minor,
applicationStatus=application,
date=updTime,
source=source)
if not result["OK"]:
return result
# If the update comes from a job, update the heart beat time stamp with this item's stamp
if source.startswith("Job"):
heartBeatTime = updTime
if heartBeatTime is not None:
result = cls.jobDB.setHeartBeatData(
jobID, {"HeartBeatTime": heartBeatTime})
if not result["OK"]:
return result
return S_OK((attrNames, attrValues))
def export_getJobPageSummaryWeb(self, selectDict, sortList, startItem, maxItems, selectJobs=True):
""" Get the summary of the job information for a given page in the
job monitor in a generic format
"""
resultDict = {}
startDate = selectDict.get('FromDate', None)
if startDate:
del selectDict['FromDate']
# For backward compatibility
if startDate is None:
startDate = selectDict.get('LastUpdate', None)
if startDate:
del selectDict['LastUpdate']
endDate = selectDict.get('ToDate', None)
if endDate:
del selectDict['ToDate']
result = self.jobPolicy.getControlledUsers(RIGHT_GET_INFO)
if not result['OK']:
return S_ERROR('Failed to evaluate user rights')
if result['Value'] != 'ALL':
selectDict[('Owner', 'OwnerGroup')] = result['Value']
# Sorting instructions. Only one for the moment.
if sortList:
orderAttribute = sortList[0][0] + ":" + sortList[0][1]
else:
orderAttribute = None
statusDict = {}
result = gJobDB.getCounters('Jobs', ['Status'], selectDict,
newer=startDate,
older=endDate,
timeStamp='LastUpdateTime')
nJobs = 0
if result['OK']:
for stDict, count in result['Value']:
nJobs += count
statusDict[stDict['Status']] = count
resultDict['TotalRecords'] = nJobs
if nJobs == 0:
return S_OK(resultDict)
resultDict['Extras'] = statusDict
if selectJobs:
iniJob = startItem
if iniJob >= nJobs:
return S_ERROR('Item number out of range')
result = gJobDB.selectJobs(selectDict, orderAttribute=orderAttribute,
newer=startDate, older=endDate, limit=(maxItems, iniJob))
if not result['OK']:
return S_ERROR('Failed to select jobs: ' + result['Message'])
summaryJobList = result['Value']
if not self.globalJobsInfo:
validJobs, _invalidJobs, _nonauthJobs, _ownJobs = self.jobPolicy.evaluateJobRights(summaryJobList,
RIGHT_GET_INFO)
summaryJobList = validJobs
result = gJobDB.getAttributesForJobList(summaryJobList, SUMMARY)
if not result['OK']:
return S_ERROR('Failed to get job summary: ' + result['Message'])
summaryDict = result['Value']
# Evaluate last sign of life time
for jobID, jobDict in summaryDict.items():
if jobDict['HeartBeatTime'] == 'None':
jobDict['LastSignOfLife'] = jobDict['LastUpdateTime']
else:
lastTime = Time.fromString(jobDict['LastUpdateTime'])
hbTime = Time.fromString(jobDict['HeartBeatTime'])
# There is no way to express a timedelta of 0 ;-)
# Not only Stalled jobs but also Failed jobs because Stalled
if ((hbTime - lastTime) > (lastTime - lastTime) or
jobDict['Status'] == "Stalled" or
jobDict['MinorStatus'].startswith('Job stalled') or
jobDict['MinorStatus'].startswith('Stalling')):
jobDict['LastSignOfLife'] = jobDict['HeartBeatTime']
else:
jobDict['LastSignOfLife'] = jobDict['LastUpdateTime']
tqDict = {}
result = gTaskQueueDB.getTaskQueueForJobs(summaryJobList)
if result['OK']:
tqDict = result['Value']
# If no jobs can be selected after the properties check
if not summaryDict.keys():
return S_OK(resultDict)
# prepare the standard structure now
key = summaryDict.keys()[0]
paramNames = summaryDict[key].keys()
records = []
for jobID, jobDict in summaryDict.items():
jParList = []
for pname in paramNames:
jParList.append(jobDict[pname])
jParList.append(tqDict.get(jobID, 0))
records.append(jParList)
resultDict['ParameterNames'] = paramNames + ['TaskQueueID']
resultDict['Records'] = records
return S_OK(resultDict)
def __parseFormParams(self):
params = self.request.arguments
pD = {}
extraParams = {}
pinDates = False
for name in params:
if name.find("_") != 0:
continue
value = params[name][0]
name = name[1:]
pD[name] = str(value)
# Personalized title?
if 'plotTitle' in pD:
extraParams['plotTitle'] = pD['plotTitle']
del(pD['plotTitle'])
# Pin dates?
if 'pinDates' in pD:
pinDates = pD['pinDates']
del(pD['pinDates'])
pinDates = pinDates.lower() in ("yes", "y", "true", "1")
# Get plotname
if 'grouping' not in pD:
return S_ERROR("Missing grouping!")
grouping = pD['grouping']
# Get plotname
if 'typeName' not in pD:
return S_ERROR("Missing type name!")
typeName = pD['typeName']
del(pD['typeName'])
# Get plotname
if 'plotName' not in pD:
return S_ERROR("Missing plot name!")
reportName = pD['plotName']
del(pD['plotName'])
# Get times
if 'timeSelector' not in pD:
return S_ERROR("Missing time span!")
# Find the proper time!
pD['timeSelector'] = int(pD['timeSelector'])
if pD['timeSelector'] > 0:
end = Time.dateTime()
start = end - datetime.timedelta(seconds=pD['timeSelector'])
if not pinDates:
extraParams['lastSeconds'] = pD['timeSelector']
else:
if 'endTime' not in pD:
end = False
else:
end = Time.fromString(pD['endTime'])
del(pD['endTime'])
if 'startTime' not in pD:
return S_ERROR("Missing starTime!")
else:
start = Time.fromString(pD['startTime'])
del(pD['startTime'])
del(pD['timeSelector'])
for k in pD:
if k.find("ex_") == 0:
extraParams[k[3:]] = pD[k]
# Listify the rest
for selName in pD:
if selName == 'grouping':
pD[selName] = [pD[selName]]
else:
try:
pD[selName] = json.loads(pD[selName])
except ValueError:
pD[selName] = List.fromChar(pD[selName], ",")
return S_OK((typeName, reportName, start, end, pD, grouping, extraParams))
def __parseJobStatus( self, job, gridType ):
""" Parse output of grid pilot status command
"""
statusRE = 'Current Status:\s*(\w*)'
destinationRE = 'Destination:\s*([\w\.-]*)'
statusDateLCGRE = 'reached on:\s*....(.*)'
submittedDateRE = 'Submitted:\s*....(.*)'
statusFailedRE = 'Current Status:.*\(Failed\)'
status = None
destination = 'Unknown'
statusDate = None
submittedDate = None
try:
status = re.search( statusRE, job ).group( 1 )
if status == 'Done' and re.search( statusFailedRE, job ):
status = 'Failed'
if re.search( destinationRE, job ):
destination = re.search( destinationRE, job ).group( 1 )
if gridType == 'LCG' and re.search( statusDateLCGRE, job ):
statusDate = re.search( statusDateLCGRE, job ).group( 1 )
statusDate = time.strftime( '%Y-%m-%d %H:%M:%S', time.strptime( statusDate, '%b %d %H:%M:%S %Y' ) )
if gridType == 'gLite' and re.search( submittedDateRE, job ):
submittedDate = re.search( submittedDateRE, job ).group( 1 )
submittedDate = time.strftime( '%Y-%m-%d %H:%M:%S', time.strptime( submittedDate, '%b %d %H:%M:%S %Y %Z' ) )
except:
self.log.exception( 'Error parsing %s Job Status output:\n' % gridType, job )
isParent = False
if re.search( 'Nodes information', job ):
isParent = True
isChild = False
if re.search( 'Parent Job', job ):
isChild = True
if status == "Running":
# Pilots can be in Running state for too long, due to bugs in the WMS
if statusDate:
statusTime = Time.fromString( statusDate )
delta = Time.dateTime() - statusTime
if delta > 4 * Time.day:
self.log.info( 'Setting pilot status to Deleted after 4 days in Running' )
status = "Deleted"
statusDate = statusTime + 4 * Time.day
elif submittedDate:
statusTime = Time.fromString( submittedDate )
delta = Time.dateTime() - statusTime
if delta > 7 * Time.day:
self.log.info( 'Setting pilot status to Deleted more than 7 days after submission still in Running' )
status = "Deleted"
statusDate = statusTime + 7 * Time.day
childRefs = []
childDicts = {}
if isParent:
for subjob in List.fromChar( job, ' Status info for the Job :' )[1:]:
chRef = List.fromChar( subjob, '\n' )[0].strip()
childDict = self.__parseJobStatus( subjob, gridType )
childRefs.append( chRef )
childDicts[chRef] = childDict
return { 'Status': status,
'DestinationSite': destination,
'StatusDate': statusDate,
'isChild': isChild,
'isParent': isParent,
'ParentRef': False,
'FinalStatus' : status in self.finalStateList,
'ChildRefs' : childRefs,
'ChildDicts' : childDicts }
def tupleToMessage( varTuple ): varList = list( varTuple ) varList[ 2 ] = Time.fromString( varList[ 2 ] ) return Message( *varList )
def export_getJobPageSummaryWeb(self,
selectDict,
sortList,
startItem,
maxItems,
selectJobs=True):
""" Get the summary of the job information for a given page in the
job monitor in a generic format
"""
resultDict = {}
startDate = selectDict.get('FromDate', None)
if startDate:
del selectDict['FromDate']
# For backward compatibility
if startDate is None:
startDate = selectDict.get('LastUpdate', None)
if startDate:
del selectDict['LastUpdate']
endDate = selectDict.get('ToDate', None)
if endDate:
del selectDict['ToDate']
# Sorting instructions. Only one for the moment.
if sortList:
orderAttribute = sortList[0][0] + ":" + sortList[0][1]
else:
orderAttribute = None
if selectJobs:
result = jobDB.selectJobs(selectDict,
orderAttribute=orderAttribute,
newer=startDate,
older=endDate)
if not result['OK']:
return S_ERROR('Failed to select jobs: ' + result['Message'])
jobList = result['Value']
# A.T. This needs optimization
#validJobList, invalidJobList, nonauthJobList, ownerJobList = self.jobPolicy.evaluateJobRights( jobList,
# RIGHT_GET_INFO )
#jobList = validJobList
nJobs = len(jobList)
resultDict['TotalRecords'] = nJobs
if nJobs == 0:
return S_OK(resultDict)
iniJob = startItem
lastJob = iniJob + maxItems
if iniJob >= nJobs:
return S_ERROR('Item number out of range')
if lastJob > nJobs:
lastJob = nJobs
summaryJobList = jobList[iniJob:lastJob]
result = jobDB.getAttributesForJobList(summaryJobList, SUMMARY)
if not result['OK']:
return S_ERROR('Failed to get job summary: ' +
result['Message'])
summaryDict = result['Value']
# Evaluate last sign of life time
for jobID, jobDict in summaryDict.items():
if jobDict['HeartBeatTime'] == 'None':
jobDict['LastSignOfLife'] = jobDict['LastUpdateTime']
else:
lastTime = Time.fromString(jobDict['LastUpdateTime'])
hbTime = Time.fromString(jobDict['HeartBeatTime'])
if (hbTime - lastTime) > (
lastTime -
lastTime) or jobDict['Status'] == "Stalled":
jobDict['LastSignOfLife'] = jobDict['HeartBeatTime']
else:
jobDict['LastSignOfLife'] = jobDict['LastUpdateTime']
tqDict = {}
result = taskQueueDB.getTaskQueueForJobs(summaryJobList)
if result['OK']:
tqDict = result['Value']
# prepare the standard structure now
key = summaryDict.keys()[0]
paramNames = summaryDict[key].keys()
records = []
for jobID, jobDict in summaryDict.items():
jParList = []
for pname in paramNames:
jParList.append(jobDict[pname])
if tqDict and tqDict.has_key(jobID):
jParList.append(tqDict[jobID])
else:
jParList.append(0)
records.append(jParList)
resultDict['ParameterNames'] = paramNames + ['TaskQueueID']
resultDict['Records'] = records
statusDict = {}
result = jobDB.getCounters('Jobs', ['Status'],
selectDict,
newer=startDate,
older=endDate,
timeStamp='LastUpdateTime')
if result['OK']:
for stDict, count in result['Value']:
statusDict[stDict['Status']] = count
resultDict['Extras'] = statusDict
return S_OK(resultDict)
def export_getJobPageSummaryWeb(self,
selectDict,
sortList,
startItem,
maxItems,
selectJobs=True):
""" Get the summary of the job information for a given page in the
job monitor in a generic format
"""
resultDict = {}
startDate = selectDict.get('FromDate', None)
if startDate:
del selectDict['FromDate']
# For backward compatibility
if startDate is None:
startDate = selectDict.get('LastUpdate', None)
if startDate:
del selectDict['LastUpdate']
endDate = selectDict.get('ToDate', None)
if endDate:
del selectDict['ToDate']
# Provide JobID bound to a specific PilotJobReference
# There is no reason to have both PilotJobReference and JobID in selectDict
# If that occurs, use the JobID instead of the PilotJobReference
pilotJobRefs = selectDict.get('PilotJobReference')
if pilotJobRefs:
del selectDict['PilotJobReference']
if 'JobID' not in selectDict or not selectDict['JobID']:
if not isinstance(pilotJobRefs, list):
pilotJobRefs = [pilotJobRefs]
selectDict['JobID'] = []
for pilotJobRef in pilotJobRefs:
res = PilotManagerClient().getPilotInfo(pilotJobRef)
if res['OK'] and 'Jobs' in res['Value'][pilotJobRef]:
selectDict['JobID'].extend(
res['Value'][pilotJobRef]['Jobs'])
result = self.jobPolicy.getControlledUsers(RIGHT_GET_INFO)
if not result['OK']:
return S_ERROR('Failed to evaluate user rights')
if result['Value'] != 'ALL':
selectDict[('Owner', 'OwnerGroup')] = result['Value']
# Sorting instructions. Only one for the moment.
if sortList:
orderAttribute = sortList[0][0] + ":" + sortList[0][1]
else:
orderAttribute = None
statusDict = {}
result = self.gJobDB.getCounters('Jobs', ['Status'],
selectDict,
newer=startDate,
older=endDate,
timeStamp='LastUpdateTime')
nJobs = 0
if result['OK']:
for stDict, count in result['Value']:
nJobs += count
statusDict[stDict['Status']] = count
resultDict['TotalRecords'] = nJobs
if nJobs == 0:
return S_OK(resultDict)
resultDict['Extras'] = statusDict
if selectJobs:
iniJob = startItem
if iniJob >= nJobs:
return S_ERROR('Item number out of range')
result = self.gJobDB.selectJobs(selectDict,
orderAttribute=orderAttribute,
newer=startDate,
older=endDate,
limit=(maxItems, iniJob))
if not result['OK']:
return S_ERROR('Failed to select jobs: ' + result['Message'])
summaryJobList = result['Value']
if not self.globalJobsInfo:
validJobs, _invalidJobs, _nonauthJobs, _ownJobs = self.jobPolicy.evaluateJobRights(
summaryJobList, RIGHT_GET_INFO)
summaryJobList = validJobs
result = self.getAttributesForJobList(summaryJobList, SUMMARY)
if not result['OK']:
return S_ERROR('Failed to get job summary: ' +
result['Message'])
summaryDict = result['Value']
# Evaluate last sign of life time
for jobID, jobDict in summaryDict.items():
if jobDict['HeartBeatTime'] == 'None':
jobDict['LastSignOfLife'] = jobDict['LastUpdateTime']
else:
lastTime = Time.fromString(jobDict['LastUpdateTime'])
hbTime = Time.fromString(jobDict['HeartBeatTime'])
# Not only Stalled jobs but also Failed jobs because Stalled
if ((hbTime - lastTime) > timedelta(0)
or jobDict['Status'] == "Stalled"
or jobDict['MinorStatus'].startswith('Job stalled')
or jobDict['MinorStatus'].startswith('Stalling')):
jobDict['LastSignOfLife'] = jobDict['HeartBeatTime']
else:
jobDict['LastSignOfLife'] = jobDict['LastUpdateTime']
tqDict = {}
result = self.gTaskQueueDB.getTaskQueueForJobs(summaryJobList)
if result['OK']:
tqDict = result['Value']
# If no jobs can be selected after the properties check
if not summaryDict.keys():
return S_OK(resultDict)
# prepare the standard structure now
key = summaryDict.keys()[0]
paramNames = summaryDict[key].keys()
records = []
for jobID, jobDict in summaryDict.items():
jParList = []
for pname in paramNames:
jParList.append(jobDict[pname])
jParList.append(tqDict.get(jobID, 0))
records.append(jParList)
resultDict['ParameterNames'] = paramNames + ['TaskQueueID']
resultDict['Records'] = records
return S_OK(resultDict)
def export_getJobPageSummaryWeb( self, selectDict, sortList, startItem, maxItems, selectJobs = True ):
""" Get the summary of the job information for a given page in the
job monitor in a generic format
"""
resultDict = {}
startDate = selectDict.get( 'FromDate', None )
if startDate:
del selectDict['FromDate']
# For backward compatibility
if startDate is None:
startDate = selectDict.get( 'LastUpdate', None )
if startDate:
del selectDict['LastUpdate']
endDate = selectDict.get( 'ToDate', None )
if endDate:
del selectDict['ToDate']
# Sorting instructions. Only one for the moment.
if sortList:
orderAttribute = sortList[0][0] + ":" + sortList[0][1]
else:
orderAttribute = None
if selectJobs:
result = jobDB.selectJobs( selectDict, orderAttribute = orderAttribute,
newer = startDate, older = endDate )
if not result['OK']:
return S_ERROR( 'Failed to select jobs: ' + result['Message'] )
jobList = result['Value']
# A.T. This needs optimization
#validJobList, invalidJobList, nonauthJobList, ownerJobList = self.jobPolicy.evaluateJobRights( jobList,
# RIGHT_GET_INFO )
#jobList = validJobList
nJobs = len( jobList )
resultDict['TotalRecords'] = nJobs
if nJobs == 0:
return S_OK( resultDict )
iniJob = startItem
lastJob = iniJob + maxItems
if iniJob >= nJobs:
return S_ERROR( 'Item number out of range' )
if lastJob > nJobs:
lastJob = nJobs
summaryJobList = jobList[iniJob:lastJob]
result = jobDB.getAttributesForJobList( summaryJobList, SUMMARY )
if not result['OK']:
return S_ERROR( 'Failed to get job summary: ' + result['Message'] )
summaryDict = result['Value']
# Evaluate last sign of life time
for jobID, jobDict in summaryDict.items():
if jobDict['HeartBeatTime'] == 'None':
jobDict['LastSignOfLife'] = jobDict['LastUpdateTime']
else:
lastTime = Time.fromString( jobDict['LastUpdateTime'] )
hbTime = Time.fromString( jobDict['HeartBeatTime'] )
if ( hbTime - lastTime ) > ( lastTime - lastTime ) or jobDict['Status'] == "Stalled":
jobDict['LastSignOfLife'] = jobDict['HeartBeatTime']
else:
jobDict['LastSignOfLife'] = jobDict['LastUpdateTime']
tqDict = {}
result = taskQueueDB.getTaskQueueForJobs( summaryJobList )
if result['OK']:
tqDict = result['Value']
# prepare the standard structure now
key = summaryDict.keys()[0]
paramNames = summaryDict[key].keys()
records = []
for jobID, jobDict in summaryDict.items():
jParList = []
for pname in paramNames:
jParList.append( jobDict[pname] )
if tqDict and tqDict.has_key( jobID ):
jParList.append( tqDict[jobID] )
else:
jParList.append( 0 )
records.append( jParList )
resultDict['ParameterNames'] = paramNames + ['TaskQueueID']
resultDict['Records'] = records
statusDict = {}
result = jobDB.getCounters( 'Jobs', ['Status'], selectDict,
newer = startDate,
older = endDate,
timeStamp = 'LastUpdateTime' )
if result['OK']:
for stDict, count in result['Value']:
statusDict[stDict['Status']] = count
resultDict['Extras'] = statusDict
return S_OK( resultDict )
def addLoggingRecord(
self,
jobID,
status=None,
minorStatus=None,
applicationStatus=None,
date=None,
source=None,
minor=None,
application=None,
):
"""Add a new entry to the JobLoggingDB table. One, two or all the three status
components (status, minorStatus, applicationStatus) can be specified.
Optionally the time stamp of the status can
be provided in a form of a string in a format '%Y-%m-%d %H:%M:%S' or
as datetime.datetime object. If the time stamp is not provided the current
UTC time is used.
"""
# Backward compatibility
# FIXME: to remove in next version
if minor:
minorStatus = minor
if application:
applicationStatus = application
status = status or "idem"
minorStatus = minorStatus or "idem"
applicationStatus = applicationStatus or "idem"
source = source or "Unknown"
event = "status/minor/app=%s/%s/%s" % (status, minorStatus,
applicationStatus)
self.log.info("Adding record for job ",
str(jobID) + ": '" + event + "' from " + source)
try:
if not date:
# Make the UTC datetime string and float
_date = Time.dateTime()
elif isinstance(date, six.string_types):
# The date is provided as a string in UTC
_date = Time.fromString(date)
elif isinstance(date, Time._dateTimeType):
_date = date
else:
self.log.error("Incorrect date for the logging record")
_date = Time.dateTime()
except Exception:
self.log.exception("Exception while date evaluation")
_date = Time.dateTime()
epoc = time.mktime(_date.timetuple(
)) + _date.microsecond / 1000000.0 - MAGIC_EPOC_NUMBER
cmd = (
"INSERT INTO LoggingInfo (JobId, Status, MinorStatus, ApplicationStatus, "
+
"StatusTime, StatusTimeOrder, StatusSource) VALUES (%d,'%s','%s','%s','%s',%f,'%s')"
% (int(jobID), status, minorStatus, applicationStatus[:255],
str(_date), epoc, source[:32]))
return self._update(cmd)
def export_getJobPageSummaryWeb( self, selectDict, sortList, startItem, maxItems, selectJobs = True ):
""" Get the summary of the job information for a given page in the
job monitor in a generic format
"""
resultDict = {}
startDate = selectDict.get( 'FromDate', None )
if startDate:
del selectDict['FromDate']
# For backward compatibility
if startDate is None:
startDate = selectDict.get( 'LastUpdate', None )
if startDate:
del selectDict['LastUpdate']
endDate = selectDict.get( 'ToDate', None )
if endDate:
del selectDict['ToDate']
result = self.jobPolicy.getControlledUsers( RIGHT_GET_INFO )
if not result['OK']:
return S_ERROR( 'Failed to evaluate user rights' )
if result['Value'] != 'ALL':
selectDict[ ( 'Owner', 'OwnerGroup' ) ] = result['Value']
# Sorting instructions. Only one for the moment.
if sortList:
orderAttribute = sortList[0][0] + ":" + sortList[0][1]
else:
orderAttribute = None
statusDict = {}
result = gJobDB.getCounters( 'Jobs', ['Status'], selectDict,
newer = startDate,
older = endDate,
timeStamp = 'LastUpdateTime' )
nJobs = 0
if result['OK']:
for stDict, count in result['Value']:
nJobs += count
statusDict[stDict['Status']] = count
resultDict['TotalRecords'] = nJobs
if nJobs == 0:
return S_OK( resultDict )
resultDict['Extras'] = statusDict
if selectJobs:
iniJob = startItem
if iniJob >= nJobs:
return S_ERROR( 'Item number out of range' )
result = gJobDB.selectJobs( selectDict, orderAttribute = orderAttribute,
newer = startDate, older = endDate, limit = ( maxItems, iniJob ) )
if not result['OK']:
return S_ERROR( 'Failed to select jobs: ' + result['Message'] )
summaryJobList = result['Value']
if not self.globalJobsInfo:
validJobs, _invalidJobs, _nonauthJobs, _ownJobs = self.jobPolicy.evaluateJobRights( summaryJobList,
RIGHT_GET_INFO )
summaryJobList = validJobs
result = gJobDB.getAttributesForJobList( summaryJobList, SUMMARY )
if not result['OK']:
return S_ERROR( 'Failed to get job summary: ' + result['Message'] )
summaryDict = result['Value']
# Evaluate last sign of life time
for jobID, jobDict in summaryDict.items():
if jobDict['HeartBeatTime'] == 'None':
jobDict['LastSignOfLife'] = jobDict['LastUpdateTime']
else:
lastTime = Time.fromString( jobDict['LastUpdateTime'] )
hbTime = Time.fromString( jobDict['HeartBeatTime'] )
if ( hbTime - lastTime ) > ( lastTime - lastTime ) or jobDict['Status'] == "Stalled":
jobDict['LastSignOfLife'] = jobDict['HeartBeatTime']
else:
jobDict['LastSignOfLife'] = jobDict['LastUpdateTime']
tqDict = {}
result = gTaskQueueDB.getTaskQueueForJobs( summaryJobList )
if result['OK']:
tqDict = result['Value']
# If no jobs can be selected after the properties check
if not summaryDict.keys():
return S_OK( resultDict )
# prepare the standard structure now
key = summaryDict.keys()[0]
paramNames = summaryDict[key].keys()
records = []
for jobID, jobDict in summaryDict.items():
jParList = []
for pname in paramNames:
jParList.append( jobDict[pname] )
jParList.append( tqDict.get( jobID, 0 ) )
records.append( jParList )
resultDict['ParameterNames'] = paramNames + ['TaskQueueID']
resultDict['Records'] = records
return S_OK( resultDict )
def __parseJobStatus(self, job, gridType):
""" Parse output of grid pilot status command
"""
statusRE = 'Current Status:\s*(\w*)'
destinationRE = 'Destination:\s*([\w\.-]*)'
statusDateLCGRE = 'reached on:\s*....(.*)'
submittedDateRE = 'Submitted:\s*....(.*)'
statusFailedRE = 'Current Status:.*\(Failed\)'
status = None
destination = 'Unknown'
statusDate = None
submittedDate = None
try:
status = re.search(statusRE, job).group(1)
if status == 'Done' and re.search(statusFailedRE, job):
status = 'Failed'
if re.search(destinationRE, job):
destination = re.search(destinationRE, job).group(1)
if gridType == 'LCG' and re.search(statusDateLCGRE, job):
statusDate = re.search(statusDateLCGRE, job).group(1)
statusDate = time.strftime(
'%Y-%m-%d %H:%M:%S',
time.strptime(statusDate, '%b %d %H:%M:%S %Y'))
if gridType == 'gLite' and re.search(submittedDateRE, job):
submittedDate = re.search(submittedDateRE, job).group(1)
submittedDate = time.strftime(
'%Y-%m-%d %H:%M:%S',
time.strptime(submittedDate, '%b %d %H:%M:%S %Y %Z'))
except:
self.log.exception(
'Error parsing %s Job Status output:\n' % gridType, job)
isParent = False
if re.search('Nodes information', job):
isParent = True
isChild = False
if re.search('Parent Job', job):
isChild = True
if status == "Running":
# Pilots can be in Running state for too long, due to bugs in the WMS
if statusDate:
statusTime = Time.fromString(statusDate)
delta = Time.dateTime() - statusTime
if delta > 4 * Time.day:
self.log.info(
'Setting pilot status to Deleted after 4 days in Running'
)
status = "Deleted"
statusDate = statusTime + 4 * Time.day
elif submittedDate:
statusTime = Time.fromString(submittedDate)
delta = Time.dateTime() - statusTime
if delta > 7 * Time.day:
self.log.info(
'Setting pilot status to Deleted more than 7 days after submission still in Running'
)
status = "Deleted"
statusDate = statusTime + 7 * Time.day
childRefs = []
childDicts = {}
if isParent:
for subjob in List.fromChar(job, ' Status info for the Job :')[1:]:
chRef = List.fromChar(subjob, '\n')[0].strip()
childDict = self.__parseJobStatus(subjob, gridType)
childRefs.append(chRef)
childDicts[chRef] = childDict
return {
'Status': status,
'DestinationSite': destination,
'StatusDate': statusDate,
'isChild': isChild,
'isParent': isParent,
'ParentRef': False,
'FinalStatus': status in self.finalStateList,
'ChildRefs': childRefs,
'ChildDicts': childDicts
}
def __parseJobStatus(self, job, gridType):
""" Parse output of grid pilot status command
"""
statusRE = "Current Status:\s*(\w*)"
destinationRE = "Destination:\s*([\w\.-]*)"
statusDateLCGRE = "reached on:\s*....(.*)"
submittedDateRE = "Submitted:\s*....(.*)"
statusFailedRE = "Current Status:.*\(Failed\)"
status = None
destination = "Unknown"
statusDate = None
submittedDate = None
try:
status = re.search(statusRE, job).group(1)
if status == "Done" and re.search(statusFailedRE, job):
status = "Failed"
if re.search(destinationRE, job):
destination = re.search(destinationRE, job).group(1)
if gridType == "LCG" and re.search(statusDateLCGRE, job):
statusDate = re.search(statusDateLCGRE, job).group(1)
statusDate = time.strftime("%Y-%m-%d %H:%M:%S", time.strptime(statusDate, "%b %d %H:%M:%S %Y"))
if gridType == "gLite" and re.search(submittedDateRE, job):
submittedDate = re.search(submittedDateRE, job).group(1)
submittedDate = time.strftime("%Y-%m-%d %H:%M:%S", time.strptime(submittedDate, "%b %d %H:%M:%S %Y %Z"))
except:
self.log.exception("Error parsing %s Job Status output:\n" % gridType, job)
isParent = False
if re.search("Nodes information", job):
isParent = True
isChild = False
if re.search("Parent Job", job):
isChild = True
if status == "Running":
# Pilots can be in Running state for too long, due to bugs in the WMS
if statusDate:
statusTime = Time.fromString(statusDate)
delta = Time.dateTime() - statusTime
if delta > 4 * Time.day:
self.log.info("Setting pilot status to Deleted after 4 days in Running")
status = "Deleted"
statusDate = statusTime + 4 * Time.day
elif submittedDate:
statusTime = Time.fromString(submittedDate)
delta = Time.dateTime() - statusTime
if delta > 7 * Time.day:
self.log.info("Setting pilot status to Deleted more than 7 days after submission still in Running")
status = "Deleted"
statusDate = statusTime + 7 * Time.day
childRefs = []
childDicts = {}
if isParent:
for subjob in List.fromChar(job, " Status info for the Job :")[1:]:
chRef = List.fromChar(subjob, "\n")[0].strip()
childDict = self.__parseJobStatus(subjob, gridType)
childRefs.append(chRef)
childDicts[chRef] = childDict
return {
"Status": status,
"DestinationSite": destination,
"StatusDate": statusDate,
"isChild": isChild,
"isParent": isParent,
"ParentRef": False,
"FinalStatus": status in self.finalStateList,
"ChildRefs": childRefs,
"ChildDicts": childDicts,
}
def __dateToSecs(self, timeVar):
dt = Time.fromString(timeVar)
return int(Time.toEpoch(dt))
def __dateToSecs( self, timeVar ): dt = Time.fromString( timeVar ) return int( Time.toEpoch( dt ) )
def export_getJobPageSummaryWeb(self,
selectDict,
sortList,
startItem,
maxItems,
selectJobs=True):
"""Get the summary of the job information for a given page in the
job monitor in a generic format
"""
resultDict = {}
startDate, endDate, selectDict = self.parseSelectors(selectDict)
# initialize jobPolicy
credDict = self.getRemoteCredentials()
ownerDN = credDict["DN"]
ownerGroup = credDict["group"]
operations = Operations(group=ownerGroup)
globalJobsInfo = operations.getValue(
"/Services/JobMonitoring/GlobalJobsInfo", True)
jobPolicy = JobPolicy(ownerDN, ownerGroup, globalJobsInfo)
jobPolicy.jobDB = self.jobDB
result = jobPolicy.getControlledUsers(RIGHT_GET_INFO)
if not result["OK"]:
return result
if not result["Value"]:
return S_ERROR(
"User and group combination has no job rights (%r, %r)" %
(ownerDN, ownerGroup))
if result["Value"] != "ALL":
selectDict[("Owner", "OwnerGroup")] = result["Value"]
# Sorting instructions. Only one for the moment.
if sortList:
orderAttribute = sortList[0][0] + ":" + sortList[0][1]
else:
orderAttribute = None
result = self.jobDB.getCounters("Jobs", ["Status"],
selectDict,
newer=startDate,
older=endDate,
timeStamp="LastUpdateTime")
if not result["OK"]:
return result
statusDict = {}
nJobs = 0
for stDict, count in result["Value"]:
nJobs += count
statusDict[stDict["Status"]] = count
resultDict["TotalRecords"] = nJobs
if nJobs == 0:
return S_OK(resultDict)
resultDict["Extras"] = statusDict
if selectJobs:
iniJob = startItem
if iniJob >= nJobs:
return S_ERROR("Item number out of range")
result = self.jobDB.selectJobs(selectDict,
orderAttribute=orderAttribute,
newer=startDate,
older=endDate,
limit=(maxItems, iniJob))
if not result["OK"]:
return result
summaryJobList = result["Value"]
if not globalJobsInfo:
validJobs, _invalidJobs, _nonauthJobs, _ownJobs = jobPolicy.evaluateJobRights(
summaryJobList, RIGHT_GET_INFO)
summaryJobList = validJobs
result = self.getJobsAttributes(summaryJobList, SUMMARY)
if not result["OK"]:
return result
summaryDict = result["Value"]
# If no jobs can be selected after the properties check
if not summaryDict:
return S_OK(resultDict)
# Evaluate last sign of life time
for jobDict in summaryDict.values():
if not jobDict.get(
"HeartBeatTime") or jobDict["HeartBeatTime"] == "None":
jobDict["LastSignOfLife"] = jobDict["LastUpdateTime"]
elif False:
# Code kept in case this is not working, but if we update the HeartBeatTime
# at each status change from the jobs it should not be needed
# Items are always strings
lastTime = Time.fromString(jobDict["LastUpdateTime"])
hbTime = Time.fromString(jobDict["HeartBeatTime"])
# Try and identify statuses not set by the job itself as too expensive to get logging info
# Not only Stalled jobs but also Failed jobs because Stalled
if (hbTime > lastTime
or jobDict["Status"] == JobStatus.STALLED
or jobDict["MinorStatus"] in (
JobMinorStatus.REQUESTS_DONE,
JobMinorStatus.STALLED_PILOT_NOT_RUNNING,
)
or jobDict["MinorStatus"].startswith("Stalling")):
jobDict["LastSignOfLife"] = jobDict["HeartBeatTime"]
else:
jobDict["LastSignOfLife"] = jobDict["LastUpdateTime"]
else:
jobDict["LastSignOfLife"] = jobDict["HeartBeatTime"]
# prepare the standard structure now
# This should be faster than making a list of values()
for jobDict in summaryDict.values():
paramNames = list(jobDict)
break
records = [
list(jobDict.values()) for jobDict in summaryDict.values()
]
resultDict["ParameterNames"] = paramNames
resultDict["Records"] = records
return S_OK(resultDict)