def main():
params = Params()
params.registerCLISwitches()
Script.parseCommandLine(ignoreErrors=True)
result = gProxyManager.getDBContents()
if not result["OK"]:
print("Can't retrieve list of users: %s" % result["Message"])
DIRAC.exit(1)
keys = result["Value"]["ParameterNames"]
records = result["Value"]["Records"]
dataDict = {}
now = datetime.datetime.utcnow()
for record in records:
expirationDate = record[3]
dt = expirationDate - now
secsLeft = dt.days * 86400 + dt.seconds
if secsLeft > params.proxyLifeTime:
userName, userDN, userGroup, _, persistent = record
if userName not in dataDict:
dataDict[userName] = []
dataDict[userName].append((userDN, userGroup, expirationDate, persistent))
for userName in dataDict:
print("* %s" % userName)
for iP in range(len(dataDict[userName])):
data = dataDict[userName][iP]
print(" DN : %s" % data[0])
print(" group : %s" % data[1])
print(" not after : %s" % TimeUtilities.toString(data[2]))
print(" persistent : %s" % data[3])
if iP < len(dataDict[userName]) - 1:
print(" -")
DIRAC.exit(0)
def handleOldPilots(self, connection):
"""
select all pilots that have not been updated in the last N days and declared them
Deleted, accounting for them.
"""
pilotsToAccount = {}
timeLimitToConsider = TimeUtilities.toString(
datetime.datetime.utcnow() -
TimeUtilities.day * self.pilotStalledDays)
result = self.pilotDB.selectPilots(
{"Status": PilotStatus.PILOT_TRANSIENT_STATES},
older=timeLimitToConsider,
timeStamp="LastUpdateTime")
if not result["OK"]:
self.log.error("Failed to get the Pilot Agents")
return result
if not result["Value"]:
return S_OK()
refList = result["Value"]
result = self.pilotDB.getPilotInfo(refList)
if not result["OK"]:
self.log.error("Failed to get Info for Pilot Agents")
return result
pilotsDict = result["Value"]
for pRef in pilotsDict:
if pilotsDict[pRef].get("Jobs") and self._checkJobLastUpdateTime(
pilotsDict[pRef]["Jobs"], self.pilotStalledDays):
self.log.debug(
"%s should not be deleted since one job of %s is running."
% (str(pRef), str(pilotsDict[pRef]["Jobs"])))
continue
deletedJobDict = pilotsDict[pRef]
deletedJobDict["Status"] = PilotStatus.DELETED
deletedJobDict["StatusDate"] = datetime.datetime.utcnow()
pilotsToAccount[pRef] = deletedJobDict
if len(pilotsToAccount) > 100:
self.accountPilots(pilotsToAccount, connection)
self._killPilots(pilotsToAccount)
pilotsToAccount = {}
self.accountPilots(pilotsToAccount, connection)
self._killPilots(pilotsToAccount)
return S_OK()
def tabularPrint(table):
columns_names = list(table[0])
records = []
for row in table:
record = []
for _k, v in row.items():
if isinstance(v, datetime.datetime):
record.append(TimeUtilities.toString(v))
elif v is None:
record.append("")
else:
record.append(v)
records.append(record)
output = printTable(columns_names,
records,
numbering=False,
columnSeparator=" | ",
printOut=False)
subLogger.notice(output)
def _setJobStatusBulk(cls, jobID, statusDict, force=False):
"""Set various status fields for job specified by its jobId.
Set only the last status in the JobDB, updating all the status
logging information in the JobLoggingDB. The statusDict has datetime
as a key and status information dictionary as values
"""
jobID = int(jobID)
log = cls.log.getLocalSubLogger("JobStatusBulk/Job-%d" % jobID)
result = cls.jobDB.getJobAttributes(jobID, ["Status", "StartExecTime", "EndExecTime"])
if not result["OK"]:
return result
if not result["Value"]:
# if there is no matching Job it returns an empty dictionary
return S_ERROR("No Matching Job")
# If the current status is Stalled and we get an update, it should probably be "Running"
currentStatus = result["Value"]["Status"]
if currentStatus == JobStatus.STALLED:
currentStatus = JobStatus.RUNNING
startTime = result["Value"].get("StartExecTime")
endTime = result["Value"].get("EndExecTime")
# getJobAttributes only returns strings :(
if startTime == "None":
startTime = None
if endTime == "None":
endTime = None
# Remove useless items in order to make it simpler later, although there should not be any
for sDict in statusDict.values():
for item in sorted(sDict):
if not sDict[item]:
sDict.pop(item, None)
# Get the latest time stamps of major status updates
result = cls.jobLoggingDB.getWMSTimeStamps(int(jobID))
if not result["OK"]:
return result
if not result["Value"]:
return S_ERROR("No registered WMS timeStamps")
# This is more precise than "LastTime". timeStamps is a sorted list of tuples...
timeStamps = sorted((float(t), s) for s, t in result["Value"].items() if s != "LastTime")
lastTime = TimeUtilities.toString(TimeUtilities.fromEpoch(timeStamps[-1][0]))
# Get chronological order of new updates
updateTimes = sorted(statusDict)
log.debug("*** New call ***", "Last update time %s - Sorted new times %s" % (lastTime, updateTimes))
# Get the status (if any) at the time of the first update
newStat = ""
firstUpdate = TimeUtilities.toEpoch(TimeUtilities.fromString(updateTimes[0]))
for ts, st in timeStamps:
if firstUpdate >= ts:
newStat = st
# Pick up start and end times from all updates
for updTime in updateTimes:
sDict = statusDict[updTime]
newStat = sDict.get("Status", newStat)
if not startTime and newStat == JobStatus.RUNNING:
# Pick up the start date when the job starts running if not existing
startTime = updTime
log.debug("Set job start time", startTime)
elif not endTime and newStat in JobStatus.JOB_FINAL_STATES:
# Pick up the end time when the job is in a final status
endTime = updTime
log.debug("Set job end time", endTime)
# We should only update the status to the last one if its time stamp is more recent than the last update
attrNames = []
attrValues = []
if updateTimes[-1] >= lastTime:
minor = ""
application = ""
# Get the last status values looping on the most recent upupdateTimes in chronological order
for updTime in [dt for dt in updateTimes if dt >= lastTime]:
sDict = statusDict[updTime]
log.debug("\t", "Time %s - Statuses %s" % (updTime, str(sDict)))
status = sDict.get("Status", currentStatus)
# evaluate the state machine if the status is changing
if not force and status != currentStatus:
res = JobStatus.JobsStateMachine(currentStatus).getNextState(status)
if not res["OK"]:
return res
newStat = res["Value"]
# If the JobsStateMachine does not accept the candidate, don't update
if newStat != status:
# keeping the same status
log.error(
"Job Status Error",
"%s can't move from %s to %s: using %s" % (jobID, currentStatus, status, newStat),
)
status = newStat
sDict["Status"] = newStat
# Change the source to indicate this is not what was requested
source = sDict.get("Source", "")
sDict["Source"] = source + "(SM)"
# at this stage status == newStat. Set currentStatus to this new status
currentStatus = newStat
minor = sDict.get("MinorStatus", minor)
application = sDict.get("ApplicationStatus", application)
log.debug("Final statuses:", "status '%s', minor '%s', application '%s'" % (status, minor, application))
if status:
attrNames.append("Status")
attrValues.append(status)
if minor:
attrNames.append("MinorStatus")
attrValues.append(minor)
if application:
attrNames.append("ApplicationStatus")
attrValues.append(application)
# Here we are forcing the update as it's always updating to the last status
result = cls.jobDB.setJobAttributes(jobID, attrNames, attrValues, update=True, force=True)
if not result["OK"]:
return result
# Update start and end time if needed
if endTime:
result = cls.jobDB.setEndExecTime(jobID, endTime)
if not result["OK"]:
return result
if startTime:
result = cls.jobDB.setStartExecTime(jobID, startTime)
if not result["OK"]:
return result
# Update the JobLoggingDB records
heartBeatTime = None
for updTime in updateTimes:
sDict = statusDict[updTime]
status = sDict.get("Status", "idem")
minor = sDict.get("MinorStatus", "idem")
application = sDict.get("ApplicationStatus", "idem")
source = sDict.get("Source", "Unknown")
result = cls.jobLoggingDB.addLoggingRecord(
jobID, status=status, minorStatus=minor, applicationStatus=application, date=updTime, source=source
)
if not result["OK"]:
return result
# If the update comes from a job, update the heart beat time stamp with this item's stamp
if source.startswith("Job"):
heartBeatTime = updTime
if heartBeatTime is not None:
result = cls.jobDB.setHeartBeatData(jobID, {"HeartBeatTime": heartBeatTime})
if not result["OK"]:
return result
return S_OK((attrNames, attrValues))