def __graphTimeComment(self, fromEpoch, toEpoch):
comStr = " 'COMMENT:Generated on %s UTC'" % Time.toString().replace(
":", r"\:").split(".")[0]
comStr += " 'COMMENT:%s'" % (
"From %s to %s" %
(Time.fromEpoch(fromEpoch), Time.fromEpoch(toEpoch))).replace(
":", r"\:")
return comStr
def export_setJobStatusBulk(self, jobID, statusDict):
""" Set various status fields for job specified by its JobId.
Set only the last status in the JobDB, updating all the status
logging information in the JobLoggingDB. The statusDict has datetime
as a key and status information dictionary as values
"""
status = ''
minor = ''
application = ''
appCounter = ''
jobID = int(jobID)
result = jobDB.getJobAttributes(jobID, ['Status', 'StartExecTime', 'EndExecTime'])
if not result['OK']:
return result
if not result['Value']:
# if there is no matching Job it returns an empty dictionary
return S_ERROR('No Matching Job')
# If the current status is Stalled and we get an update, it should probably be "Running"
if result['Value']['Status'] == JobStatus.STALLED:
status = JobStatus.RUNNING
startTime = result['Value'].get('StartExecTime', '')
endTime = result['Value'].get('EndExecTime', '')
# Get the latest WN time stamps of status updates
result = logDB.getWMSTimeStamps(int(jobID))
if not result['OK']:
return result
lastTime = max([float(t) for s, t in result['Value'].items() if s != 'LastTime'])
lastTime = Time.toString(Time.fromEpoch(lastTime))
dates = sorted(statusDict)
# Pick up start and end times from all updates, if they don't exist
for date in dates:
sDict = statusDict[date]
status = sDict.get('Status', status)
if status in JobStatus.JOB_FINAL_STATES and not endTime:
endTime = date
minor = sDict.get('MinorStatus', minor)
# Pick up the start date
if minor == "Application" and status == JobStatus.RUNNING and not startTime:
startTime = date
# We should only update the status if its time stamp is more recent than the last update
if dates[-1] >= lastTime:
# Get the last status values
for date in [date for date in dates if date >= lastTime]:
sDict = statusDict[date]
status = sDict.get('Status', status)
minor = sDict.get('MinorStatus', minor)
application = sDict.get('ApplicationStatus', application)
appCounter = sDict.get('ApplicationCounter', appCounter)
attrNames = []
attrValues = []
if status:
attrNames.append('Status')
attrValues.append(status)
if minor:
attrNames.append('MinorStatus')
attrValues.append(minor)
if application:
attrNames.append('ApplicationStatus')
attrValues.append(application)
if appCounter:
attrNames.append('ApplicationCounter')
attrValues.append(appCounter)
result = jobDB.setJobAttributes(jobID, attrNames, attrValues, update=True)
if not result['OK']:
return result
# Update start and end time if needed
if endTime:
result = jobDB.setEndExecTime(jobID, endTime)
if startTime:
result = jobDB.setStartExecTime(jobID, startTime)
# Update the JobLoggingDB records
for date in dates:
sDict = statusDict[date]
status = sDict['Status'] if sDict['Status'] else 'idem'
minor = sDict['MinorStatus'] if sDict['MinorStatus'] else 'idem'
application = sDict['ApplicationStatus'] if sDict['ApplicationStatus'] else 'idem'
source = sDict['Source']
result = logDB.addLoggingRecord(jobID, status, minor, application, date, source)
if not result['OK']:
return result
return S_OK()
def export_setJobStatusBulk(self, jobID, statusDict):
""" Set various status fields for job specified by its JobId.
Set only the last status in the JobDB, updating all the status
logging information in the JobLoggingDB. The statusDict has datetime
as a key and status information dictionary as values
"""
status = ""
minor = ""
application = ""
appCounter = ""
endDate = ''
startDate = ''
startFlag = ''
jobID = int(jobID)
result = jobDB.getJobAttributes(jobID, ['Status'])
if not result['OK']:
return result
if not result['Value']:
# if there is no matching Job it returns an empty dictionary
return S_ERROR('No Matching Job')
new_status = result['Value']['Status']
if new_status == "Stalled":
status = 'Running'
# Get the latest WN time stamps of status updates
result = logDB.getWMSTimeStamps(int(jobID))
if not result['OK']:
return result
lastTime = max(
[float(t) for s, t in result['Value'].items() if s != 'LastTime'])
lastTime = Time.toString(Time.fromEpoch(lastTime))
# Get the last status values
dates = sorted(statusDict)
# We should only update the status if its time stamp is more recent than the last update
for date in [date for date in dates if date >= lastTime]:
sDict = statusDict[date]
if sDict['Status']:
status = sDict['Status']
if status in JOB_FINAL_STATES:
endDate = date
if status == "Running":
startFlag = 'Running'
if sDict['MinorStatus']:
minor = sDict['MinorStatus']
if minor == "Application" and startFlag == 'Running':
startDate = date
if sDict['ApplicationStatus']:
application = sDict['ApplicationStatus']
counter = sDict.get('ApplicationCounter')
if counter:
appCounter = counter
attrNames = []
attrValues = []
if status:
attrNames.append('Status')
attrValues.append(status)
if minor:
attrNames.append('MinorStatus')
attrValues.append(minor)
if application:
attrNames.append('ApplicationStatus')
attrValues.append(application)
if appCounter:
attrNames.append('ApplicationCounter')
attrValues.append(appCounter)
result = jobDB.setJobAttributes(jobID,
attrNames,
attrValues,
update=True)
if not result['OK']:
return result
if endDate:
result = jobDB.setEndExecTime(jobID, endDate)
if startDate:
result = jobDB.setStartExecTime(jobID, startDate)
# Update the JobLoggingDB records
for date in dates:
sDict = statusDict[date]
status = sDict['Status']
if not status:
status = 'idem'
minor = sDict['MinorStatus']
if not minor:
minor = 'idem'
application = sDict['ApplicationStatus']
if not application:
application = 'idem'
source = sDict['Source']
result = logDB.addLoggingRecord(jobID, status, minor, application,
date, source)
if not result['OK']:
return result
return S_OK()
def export_setJobStatusBulk( self, jobID, statusDict ):
""" Set various status fields for job specified by its JobId.
Set only the last status in the JobDB, updating all the status
logging information in the JobLoggingDB. The statusDict has datetime
as a key and status information dictionary as values
"""
status = ""
minor = ""
application = ""
appCounter = ""
endDate = ''
startDate = ''
startFlag = ''
jobID = int( jobID )
result = jobDB.getJobAttributes( jobID, ['Status'] )
if not result['OK']:
return result
if not result['Value']:
# if there is no matching Job it returns an empty dictionary
return S_ERROR( 'No Matching Job' )
new_status = result['Value']['Status']
if new_status == "Stalled":
status = 'Running'
# Get the latest WN time stamps of status updates
result = logDB.getWMSTimeStamps( int( jobID ) )
if not result['OK']:
return result
lastTime = max( [float( t ) for s, t in result['Value'].items() if s != 'LastTime'] )
lastTime = Time.toString( Time.fromEpoch( lastTime ) )
# Get the last status values
dates = sorted( statusDict )
# We should only update the status if its time stamp is more recent than the last update
for date in [date for date in dates if date >= lastTime]:
sDict = statusDict[date]
if sDict['Status']:
status = sDict['Status']
if status in JOB_FINAL_STATES:
endDate = date
if status == "Running":
startFlag = 'Running'
if sDict['MinorStatus']:
minor = sDict['MinorStatus']
if minor == "Application" and startFlag == 'Running':
startDate = date
if sDict['ApplicationStatus']:
application = sDict['ApplicationStatus']
counter = sDict.get( 'ApplicationCounter' )
if counter:
appCounter = counter
attrNames = []
attrValues = []
if status:
attrNames.append( 'Status' )
attrValues.append( status )
if minor:
attrNames.append( 'MinorStatus' )
attrValues.append( minor )
if application:
attrNames.append( 'ApplicationStatus' )
attrValues.append( application )
if appCounter:
attrNames.append( 'ApplicationCounter' )
attrValues.append( appCounter )
result = jobDB.setJobAttributes( jobID, attrNames, attrValues, update = True )
if not result['OK']:
return result
if endDate:
result = jobDB.setEndExecTime( jobID, endDate )
if startDate:
result = jobDB.setStartExecTime( jobID, startDate )
# Update the JobLoggingDB records
for date in dates:
sDict = statusDict[date]
status = sDict['Status']
if not status:
status = 'idem'
minor = sDict['MinorStatus']
if not minor:
minor = 'idem'
application = sDict['ApplicationStatus']
if not application:
application = 'idem'
source = sDict['Source']
result = logDB.addLoggingRecord( jobID, status, minor, application, date, source )
if not result['OK']:
return result
return S_OK()
def __graphTimeComment( self, fromEpoch, toEpoch ): comStr = " 'COMMENT:Generated on %s UTC'" % Time.toString().replace( ":", "\:" ).split( "." )[0] comStr += " 'COMMENT:%s'" % ( "From %s to %s" % ( Time.fromEpoch( fromEpoch ), Time.fromEpoch( toEpoch ) ) ).replace( ":", "\:" ) return comStr
def __setJobStatusBulk(cls, jobID, statusDict):
""" Set various status fields for job specified by its JobId.
Set only the last status in the JobDB, updating all the status
logging information in the JobLoggingDB. The statusDict has datetime
as a key and status information dictionary as values
"""
status = ''
minor = ''
application = ''
jobID = int(jobID)
result = cls.jobDB.getJobAttributes(
jobID, ['Status', 'StartExecTime', 'EndExecTime'])
if not result['OK']:
return result
if not result['Value']:
# if there is no matching Job it returns an empty dictionary
return S_ERROR('No Matching Job')
# If the current status is Stalled and we get an update, it should probably be "Running"
currentStatus = result['Value']['Status']
if currentStatus == JobStatus.STALLED:
status = JobStatus.RUNNING
startTime = result['Value'].get('StartExecTime')
endTime = result['Value'].get('EndExecTime')
# getJobAttributes only returns strings :(
if startTime == 'None':
startTime = None
if endTime == 'None':
endTime = None
# Get the latest WN time stamps of status updates
result = cls.jobLoggingDB.getWMSTimeStamps(int(jobID))
if not result['OK']:
return result
lastTime = max(
[float(t) for s, t in result['Value'].items() if s != 'LastTime'])
lastTime = Time.toString(Time.fromEpoch(lastTime))
dates = sorted(statusDict)
# If real updates, start from the current status
if dates[0] >= lastTime and not status:
status = currentStatus
log = gLogger.getSubLogger('JobStatusBulk/Job-%s' % jobID)
log.debug(
"*** New call ***",
"Last update time %s - Sorted new times %s" % (lastTime, dates))
# Remove useless items in order to make it simpler later, although there should not be any
for sDict in statusDict.values():
for item in sorted(sDict):
if not sDict[item]:
sDict.pop(item, None)
# Pick up start and end times from all updates, if they don't exist
newStat = status
for date in dates:
sDict = statusDict[date]
# This is to recover Matched jobs that set the application status: they are running!
if sDict.get('ApplicationStatus') and newStat == JobStatus.MATCHED:
sDict['Status'] = JobStatus.RUNNING
newStat = sDict.get('Status', newStat)
if newStat == JobStatus.RUNNING and not startTime:
# Pick up the start date when the job starts running if not existing
startTime = date
log.debug("Set job start time", startTime)
elif newStat in JobStatus.JOB_FINAL_STATES and not endTime:
# Pick up the end time when the job is in a final status
endTime = date
log.debug("Set job end time", endTime)
# We should only update the status if its time stamp is more recent than the last update
if dates[-1] >= lastTime:
# Get the last status values
for date in [dt for dt in dates if dt >= lastTime]:
sDict = statusDict[date]
log.debug("\t", "Time %s - Statuses %s" % (date, str(sDict)))
status = sDict.get('Status', status)
minor = sDict.get('MinorStatus', minor)
application = sDict.get('ApplicationStatus', application)
log.debug(
"Final statuses:",
"status '%s', minor '%s', application '%s'" %
(status, minor, application))
attrNames = []
attrValues = []
if status:
attrNames.append('Status')
attrValues.append(status)
if minor:
attrNames.append('MinorStatus')
attrValues.append(minor)
if application:
attrNames.append('ApplicationStatus')
attrValues.append(application)
result = cls.jobDB.setJobAttributes(jobID,
attrNames,
attrValues,
update=True)
if not result['OK']:
return result
# Update start and end time if needed
if endTime:
result = cls.jobDB.setEndExecTime(jobID, endTime)
if not result['OK']:
return result
if startTime:
result = cls.jobDB.setStartExecTime(jobID, startTime)
if not result['OK']:
return result
# Update the JobLoggingDB records
for date in dates:
sDict = statusDict[date]
status = sDict.get('Status', 'idem')
minor = sDict.get('MinorStatus', 'idem')
application = sDict.get('ApplicationStatus', 'idem')
source = sDict.get('Source', 'Unknown')
result = cls.jobLoggingDB.addLoggingRecord(
jobID,
status=status,
minorStatus=minor,
applicationStatus=application,
date=date,
source=source)
if not result['OK']:
return result
return S_OK()
def _setJobStatusBulk(cls, jobID, statusDict, force=False):
"""Set various status fields for job specified by its jobId.
Set only the last status in the JobDB, updating all the status
logging information in the JobLoggingDB. The statusDict has datetime
as a key and status information dictionary as values
"""
jobID = int(jobID)
log = gLogger.getLocalSubLogger("JobStatusBulk/Job-%d" % jobID)
result = cls.jobDB.getJobAttributes(
jobID, ["Status", "StartExecTime", "EndExecTime"])
if not result["OK"]:
return result
if not result["Value"]:
# if there is no matching Job it returns an empty dictionary
return S_ERROR("No Matching Job")
# If the current status is Stalled and we get an update, it should probably be "Running"
currentStatus = result["Value"]["Status"]
if currentStatus == JobStatus.STALLED:
currentStatus = JobStatus.RUNNING
startTime = result["Value"].get("StartExecTime")
endTime = result["Value"].get("EndExecTime")
# getJobAttributes only returns strings :(
if startTime == "None":
startTime = None
if endTime == "None":
endTime = None
# Remove useless items in order to make it simpler later, although there should not be any
for sDict in statusDict.values():
for item in sorted(sDict):
if not sDict[item]:
sDict.pop(item, None)
# Get the latest time stamps of major status updates
result = cls.jobLoggingDB.getWMSTimeStamps(int(jobID))
if not result["OK"]:
return result
if not result["Value"]:
return S_ERROR("No registered WMS timeStamps")
# This is more precise than "LastTime". timeStamps is a sorted list of tuples...
timeStamps = sorted((float(t), s) for s, t in result["Value"].items()
if s != "LastTime")
lastTime = Time.toString(Time.fromEpoch(timeStamps[-1][0]))
# Get chronological order of new updates
updateTimes = sorted(statusDict)
log.debug(
"*** New call ***", "Last update time %s - Sorted new times %s" %
(lastTime, updateTimes))
# Get the status (if any) at the time of the first update
newStat = ""
firstUpdate = Time.toEpoch(Time.fromString(updateTimes[0]))
for ts, st in timeStamps:
if firstUpdate >= ts:
newStat = st
# Pick up start and end times from all updates
for updTime in updateTimes:
sDict = statusDict[updTime]
newStat = sDict.get("Status", newStat)
if not startTime and newStat == JobStatus.RUNNING:
# Pick up the start date when the job starts running if not existing
startTime = updTime
log.debug("Set job start time", startTime)
elif not endTime and newStat in JobStatus.JOB_FINAL_STATES:
# Pick up the end time when the job is in a final status
endTime = updTime
log.debug("Set job end time", endTime)
# We should only update the status to the last one if its time stamp is more recent than the last update
attrNames = []
attrValues = []
if updateTimes[-1] >= lastTime:
minor = ""
application = ""
# Get the last status values looping on the most recent upupdateTimes in chronological order
for updTime in [dt for dt in updateTimes if dt >= lastTime]:
sDict = statusDict[updTime]
log.debug("\t",
"Time %s - Statuses %s" % (updTime, str(sDict)))
status = sDict.get("Status", currentStatus)
# evaluate the state machine if the status is changing
if not force and status != currentStatus:
res = JobStatus.JobsStateMachine(
currentStatus).getNextState(status)
if not res["OK"]:
return res
newStat = res["Value"]
# If the JobsStateMachine does not accept the candidate, don't update
if newStat != status:
# keeping the same status
log.error(
"Job Status Error",
"%s can't move from %s to %s: using %s" %
(jobID, currentStatus, status, newStat),
)
status = newStat
sDict["Status"] = newStat
# Change the source to indicate this is not what was requested
source = sDict.get("Source", "")
sDict["Source"] = source + "(SM)"
# at this stage status == newStat. Set currentStatus to this new status
currentStatus = newStat
minor = sDict.get("MinorStatus", minor)
application = sDict.get("ApplicationStatus", application)
log.debug(
"Final statuses:",
"status '%s', minor '%s', application '%s'" %
(status, minor, application))
if status:
attrNames.append("Status")
attrValues.append(status)
if minor:
attrNames.append("MinorStatus")
attrValues.append(minor)
if application:
attrNames.append("ApplicationStatus")
attrValues.append(application)
# Here we are forcing the update as it's always updating to the last status
result = cls.jobDB.setJobAttributes(jobID,
attrNames,
attrValues,
update=True,
force=True)
if not result["OK"]:
return result
# Update start and end time if needed
if endTime:
result = cls.jobDB.setEndExecTime(jobID, endTime)
if not result["OK"]:
return result
if startTime:
result = cls.jobDB.setStartExecTime(jobID, startTime)
if not result["OK"]:
return result
# Update the JobLoggingDB records
heartBeatTime = None
for updTime in updateTimes:
sDict = statusDict[updTime]
status = sDict.get("Status", "idem")
minor = sDict.get("MinorStatus", "idem")
application = sDict.get("ApplicationStatus", "idem")
source = sDict.get("Source", "Unknown")
result = cls.jobLoggingDB.addLoggingRecord(
jobID,
status=status,
minorStatus=minor,
applicationStatus=application,
date=updTime,
source=source)
if not result["OK"]:
return result
# If the update comes from a job, update the heart beat time stamp with this item's stamp
if source.startswith("Job"):
heartBeatTime = updTime
if heartBeatTime is not None:
result = cls.jobDB.setHeartBeatData(
jobID, {"HeartBeatTime": heartBeatTime})
if not result["OK"]:
return result
return S_OK((attrNames, attrValues))
