def filterDate(selectOutput, start, end):
"""
Selects all the downtimes that meet the constraints of 'start' and 'end' dates
"""
downtimes = selectOutput
downtimesFiltered = []
if start is not None:
try:
start = TimeUtilities.fromString(start)
except Exception:
error("datetime formt is incorrect, pls try [%Y-%m-%d[ %H:%M:%S]]")
start = TimeUtilities.toEpoch(start)
if end is not None:
try:
end = TimeUtilities.fromString(end)
except Exception:
error("datetime formt is incorrect, pls try [%Y-%m-%d[ %H:%M:%S]]")
end = TimeUtilities.toEpoch(end)
if start is not None and end is not None:
for dt in downtimes:
dtStart = TimeUtilities.toEpoch(dt["startDate"])
dtEnd = TimeUtilities.toEpoch(dt["endDate"])
if (dtStart >= start) and (dtEnd <= end):
downtimesFiltered.append(dt)
elif start is not None and end is None:
for dt in downtimes:
dtStart = TimeUtilities.toEpoch(dt["startDate"])
if dtStart >= start:
downtimesFiltered.append(dt)
elif start is None and end is not None:
for dt in downtimes:
dtEnd = TimeUtilities.toEpoch(dt["endDate"])
if dtEnd <= end:
downtimesFiltered.append(dt)
else:
downtimesFiltered = downtimes
return downtimesFiltered
def addLoggingRecord(
self,
jobID,
status="idem",
minorStatus="idem",
applicationStatus="idem",
date=None,
source="Unknown",
minor=None,
application=None,
):
"""Add a new entry to the JobLoggingDB table. One, two or all the three status
components (status, minorStatus, applicationStatus) can be specified.
Optionally the time stamp of the status can
be provided in a form of a string in a format '%Y-%m-%d %H:%M:%S' or
as datetime.datetime object. If the time stamp is not provided the current
UTC time is used.
"""
# Backward compatibility
# FIXME: to remove in next version
if minor:
minorStatus = minor
if application:
applicationStatus = application
event = "status/minor/app=%s/%s/%s" % (status, minorStatus,
applicationStatus)
self.log.info("Adding record for job ",
str(jobID) + ": '" + event + "' from " + source)
try:
if not date:
# Make the UTC datetime string and float
_date = datetime.datetime.utcnow()
elif isinstance(date, str):
# The date is provided as a string in UTC
_date = TimeUtilities.fromString(date)
elif isinstance(date, datetime.datetime):
_date = date
else:
self.log.error("Incorrect date for the logging record")
_date = datetime.datetime.utcnow()
except Exception:
self.log.exception("Exception while date evaluation")
_date = datetime.datetime.utcnow()
epoc = time.mktime(_date.timetuple(
)) + _date.microsecond / 1000000.0 - MAGIC_EPOC_NUMBER
cmd = (
"INSERT INTO LoggingInfo (JobId, Status, MinorStatus, ApplicationStatus, "
+
"StatusTime, StatusTimeOrder, StatusSource) VALUES (%d,'%s','%s','%s','%s',%f,'%s')"
% (int(jobID), status, minorStatus, applicationStatus[:255],
str(_date), epoc, source[:32]))
return self._update(cmd)
def getTypedValue(value, mtype):
if mtype[0:3].lower() == "int":
return int(value)
elif mtype[0:5].lower() == "float":
return float(value)
elif mtype[0:4].lower() == "date":
return TimeUtilities.fromString(value)
else:
return value
def _checkJobLastUpdateTime(self, joblist, StalledDays):
timeLimitToConsider = datetime.datetime.utcnow(
) - TimeUtilities.day * StalledDays
ret = False
for jobID in joblist:
result = self.jobDB.getJobAttributes(int(jobID))
if result["OK"]:
if "LastUpdateTime" in result["Value"]:
lastUpdateTime = result["Value"]["LastUpdateTime"]
if TimeUtilities.fromString(
lastUpdateTime) > timeLimitToConsider:
ret = True
self.log.debug(
"Since %s updates LastUpdateTime on %s this does not to need to be deleted."
% (str(jobID), str(lastUpdateTime)))
break
else:
self.log.error("Error taking job info from DB",
result["Message"])
return ret
def export_getJobPageSummaryWeb(self,
selectDict,
sortList,
startItem,
maxItems,
selectJobs=True):
"""Get the summary of the job information for a given page in the
job monitor in a generic format
"""
resultDict = {}
startDate, endDate, selectDict = self.parseSelectors(selectDict)
# initialize jobPolicy
credDict = self.getRemoteCredentials()
ownerDN = credDict["DN"]
ownerGroup = credDict["group"]
operations = Operations(group=ownerGroup)
globalJobsInfo = operations.getValue(
"/Services/JobMonitoring/GlobalJobsInfo", True)
jobPolicy = JobPolicy(ownerDN, ownerGroup, globalJobsInfo)
jobPolicy.jobDB = self.jobDB
result = jobPolicy.getControlledUsers(RIGHT_GET_INFO)
if not result["OK"]:
return result
if not result["Value"]:
return S_ERROR(
"User and group combination has no job rights (%r, %r)" %
(ownerDN, ownerGroup))
if result["Value"] != "ALL":
selectDict[("Owner", "OwnerGroup")] = result["Value"]
# Sorting instructions. Only one for the moment.
if sortList:
orderAttribute = sortList[0][0] + ":" + sortList[0][1]
else:
orderAttribute = None
result = self.jobDB.getCounters("Jobs", ["Status"],
selectDict,
newer=startDate,
older=endDate,
timeStamp="LastUpdateTime")
if not result["OK"]:
return result
statusDict = {}
nJobs = 0
for stDict, count in result["Value"]:
nJobs += count
statusDict[stDict["Status"]] = count
resultDict["TotalRecords"] = nJobs
if nJobs == 0:
return S_OK(resultDict)
resultDict["Extras"] = statusDict
if selectJobs:
iniJob = startItem
if iniJob >= nJobs:
return S_ERROR("Item number out of range")
result = self.jobDB.selectJobs(selectDict,
orderAttribute=orderAttribute,
newer=startDate,
older=endDate,
limit=(maxItems, iniJob))
if not result["OK"]:
return result
summaryJobList = result["Value"]
if not globalJobsInfo:
validJobs, _invalidJobs, _nonauthJobs, _ownJobs = jobPolicy.evaluateJobRights(
summaryJobList, RIGHT_GET_INFO)
summaryJobList = validJobs
result = self.getJobsAttributes(summaryJobList, SUMMARY)
if not result["OK"]:
return result
summaryDict = result["Value"]
# If no jobs can be selected after the properties check
if not summaryDict:
return S_OK(resultDict)
# Evaluate last sign of life time
for jobDict in summaryDict.values():
if not jobDict.get(
"HeartBeatTime") or jobDict["HeartBeatTime"] == "None":
jobDict["LastSignOfLife"] = jobDict["LastUpdateTime"]
elif False:
# Code kept in case this is not working, but if we update the HeartBeatTime
# at each status change from the jobs it should not be needed
# Items are always strings
lastTime = TimeUtilities.fromString(
jobDict["LastUpdateTime"])
hbTime = TimeUtilities.fromString(jobDict["HeartBeatTime"])
# Try and identify statuses not set by the job itself as too expensive to get logging info
# Not only Stalled jobs but also Failed jobs because Stalled
if (hbTime > lastTime
or jobDict["Status"] == JobStatus.STALLED
or jobDict["MinorStatus"] in (
JobMinorStatus.REQUESTS_DONE,
JobMinorStatus.STALLED_PILOT_NOT_RUNNING,
)
or jobDict["MinorStatus"].startswith("Stalling")):
jobDict["LastSignOfLife"] = jobDict["HeartBeatTime"]
else:
jobDict["LastSignOfLife"] = jobDict["LastUpdateTime"]
else:
jobDict["LastSignOfLife"] = jobDict["HeartBeatTime"]
# prepare the standard structure now
# This should be faster than making a list of values()
for jobDict in summaryDict.values():
paramNames = list(jobDict)
break
records = [
list(jobDict.values()) for jobDict in summaryDict.values()
]
resultDict["ParameterNames"] = paramNames
resultDict["Records"] = records
return S_OK(resultDict)
def _setJobStatusBulk(cls, jobID, statusDict, force=False):
"""Set various status fields for job specified by its jobId.
Set only the last status in the JobDB, updating all the status
logging information in the JobLoggingDB. The statusDict has datetime
as a key and status information dictionary as values
"""
jobID = int(jobID)
log = cls.log.getLocalSubLogger("JobStatusBulk/Job-%d" % jobID)
result = cls.jobDB.getJobAttributes(jobID, ["Status", "StartExecTime", "EndExecTime"])
if not result["OK"]:
return result
if not result["Value"]:
# if there is no matching Job it returns an empty dictionary
return S_ERROR("No Matching Job")
# If the current status is Stalled and we get an update, it should probably be "Running"
currentStatus = result["Value"]["Status"]
if currentStatus == JobStatus.STALLED:
currentStatus = JobStatus.RUNNING
startTime = result["Value"].get("StartExecTime")
endTime = result["Value"].get("EndExecTime")
# getJobAttributes only returns strings :(
if startTime == "None":
startTime = None
if endTime == "None":
endTime = None
# Remove useless items in order to make it simpler later, although there should not be any
for sDict in statusDict.values():
for item in sorted(sDict):
if not sDict[item]:
sDict.pop(item, None)
# Get the latest time stamps of major status updates
result = cls.jobLoggingDB.getWMSTimeStamps(int(jobID))
if not result["OK"]:
return result
if not result["Value"]:
return S_ERROR("No registered WMS timeStamps")
# This is more precise than "LastTime". timeStamps is a sorted list of tuples...
timeStamps = sorted((float(t), s) for s, t in result["Value"].items() if s != "LastTime")
lastTime = TimeUtilities.toString(TimeUtilities.fromEpoch(timeStamps[-1][0]))
# Get chronological order of new updates
updateTimes = sorted(statusDict)
log.debug("*** New call ***", "Last update time %s - Sorted new times %s" % (lastTime, updateTimes))
# Get the status (if any) at the time of the first update
newStat = ""
firstUpdate = TimeUtilities.toEpoch(TimeUtilities.fromString(updateTimes[0]))
for ts, st in timeStamps:
if firstUpdate >= ts:
newStat = st
# Pick up start and end times from all updates
for updTime in updateTimes:
sDict = statusDict[updTime]
newStat = sDict.get("Status", newStat)
if not startTime and newStat == JobStatus.RUNNING:
# Pick up the start date when the job starts running if not existing
startTime = updTime
log.debug("Set job start time", startTime)
elif not endTime and newStat in JobStatus.JOB_FINAL_STATES:
# Pick up the end time when the job is in a final status
endTime = updTime
log.debug("Set job end time", endTime)
# We should only update the status to the last one if its time stamp is more recent than the last update
attrNames = []
attrValues = []
if updateTimes[-1] >= lastTime:
minor = ""
application = ""
# Get the last status values looping on the most recent upupdateTimes in chronological order
for updTime in [dt for dt in updateTimes if dt >= lastTime]:
sDict = statusDict[updTime]
log.debug("\t", "Time %s - Statuses %s" % (updTime, str(sDict)))
status = sDict.get("Status", currentStatus)
# evaluate the state machine if the status is changing
if not force and status != currentStatus:
res = JobStatus.JobsStateMachine(currentStatus).getNextState(status)
if not res["OK"]:
return res
newStat = res["Value"]
# If the JobsStateMachine does not accept the candidate, don't update
if newStat != status:
# keeping the same status
log.error(
"Job Status Error",
"%s can't move from %s to %s: using %s" % (jobID, currentStatus, status, newStat),
)
status = newStat
sDict["Status"] = newStat
# Change the source to indicate this is not what was requested
source = sDict.get("Source", "")
sDict["Source"] = source + "(SM)"
# at this stage status == newStat. Set currentStatus to this new status
currentStatus = newStat
minor = sDict.get("MinorStatus", minor)
application = sDict.get("ApplicationStatus", application)
log.debug("Final statuses:", "status '%s', minor '%s', application '%s'" % (status, minor, application))
if status:
attrNames.append("Status")
attrValues.append(status)
if minor:
attrNames.append("MinorStatus")
attrValues.append(minor)
if application:
attrNames.append("ApplicationStatus")
attrValues.append(application)
# Here we are forcing the update as it's always updating to the last status
result = cls.jobDB.setJobAttributes(jobID, attrNames, attrValues, update=True, force=True)
if not result["OK"]:
return result
# Update start and end time if needed
if endTime:
result = cls.jobDB.setEndExecTime(jobID, endTime)
if not result["OK"]:
return result
if startTime:
result = cls.jobDB.setStartExecTime(jobID, startTime)
if not result["OK"]:
return result
# Update the JobLoggingDB records
heartBeatTime = None
for updTime in updateTimes:
sDict = statusDict[updTime]
status = sDict.get("Status", "idem")
minor = sDict.get("MinorStatus", "idem")
application = sDict.get("ApplicationStatus", "idem")
source = sDict.get("Source", "Unknown")
result = cls.jobLoggingDB.addLoggingRecord(
jobID, status=status, minorStatus=minor, applicationStatus=application, date=updTime, source=source
)
if not result["OK"]:
return result
# If the update comes from a job, update the heart beat time stamp with this item's stamp
if source.startswith("Job"):
heartBeatTime = updTime
if heartBeatTime is not None:
result = cls.jobDB.setHeartBeatData(jobID, {"HeartBeatTime": heartBeatTime})
if not result["OK"]:
return result
return S_OK((attrNames, attrValues))