def queueNormalizedCPU( ceUniqueID ):
"""
Report Normalized CPU length of queue
"""
result = getQueueInfo( ceUniqueID )
if not result['OK']:
return result
ceInfoDict = result['Value']
siteCSSEction = ceInfoDict['SiteCSSEction']
queueCSSection = ceInfoDict['QueueCSSection']
benchmarkSI00 = __getQueueNormalization( queueCSSection, siteCSSEction )
maxCPUTime = __getMaxCPUTime( queueCSSection )
if maxCPUTime and benchmarkSI00:
# To get to the Current LHCb
normCPUTime = NORMALIZATIONCONSTANT * maxCPUTime * benchmarkSI00
else:
if not benchmarkSI00:
subClusterUniqueID = ceInfoDict['SubClusterUniqueID']
return S_ERROR( 'benchmarkSI00 info not available for %s' % subClusterUniqueID )
if not maxCPUTime:
return S_ERROR( 'maxCPUTime info not available' )
return S_OK( normCPUTime )
def queueNormalizedCPU(ceUniqueID):
""" Report Normalized CPU length of queue
"""
result = getQueueInfo(ceUniqueID)
if not result['OK']:
return result
ceInfoDict = result['Value']
siteCSSEction = ceInfoDict['SiteCSSEction']
queueCSSection = ceInfoDict['QueueCSSection']
benchmarkSI00 = __getQueueNormalization(queueCSSection, siteCSSEction)
maxCPUTime = __getMaxCPUTime(queueCSSection)
if maxCPUTime and benchmarkSI00:
normCPUTime = NORMALIZATIONCONSTANT * maxCPUTime * benchmarkSI00
else:
if not benchmarkSI00:
subClusterUniqueID = ceInfoDict['SubClusterUniqueID']
return S_ERROR('benchmarkSI00 info not available for %s' %
subClusterUniqueID)
if not maxCPUTime:
return S_ERROR('maxCPUTime info not available')
return S_OK(normCPUTime)
def queueNormalizedCPU( ceUniqueID ):
"""
Report Normalized CPU length of queue
"""
result = getQueueInfo( ceUniqueID )
if not result['OK']:
return result
ceInfoDict = result['Value']
benchmarkSI00 = ceInfoDict['SI00']
maxCPUTime = ceInfoDict['maxCPUTime']
# For some sites there are crazy values in the CS
maxCPUTime = max( maxCPUTime, 0 )
maxCPUTime = min( maxCPUTime, 86400 * 12.5 )
if maxCPUTime and benchmarkSI00:
normCPUTime = NORMALIZATIONCONSTANT * maxCPUTime * benchmarkSI00
else:
if not benchmarkSI00:
subClusterUniqueID = ceInfoDict['SubClusterUniqueID']
return S_ERROR( 'benchmarkSI00 info not available for %s' % subClusterUniqueID )
if not maxCPUTime:
return S_ERROR( 'maxCPUTime info not available' )
return S_OK( normCPUTime )
def queueNormalizedCPU(ceUniqueID):
"""
Report Normalized CPU length of queue
"""
result = getQueueInfo(ceUniqueID)
if not result['OK']:
return result
ceInfoDict = result['Value']
benchmarkSI00 = ceInfoDict['SI00']
maxCPUTime = ceInfoDict['maxCPUTime']
# For some sites there are crazy values in the CS
maxCPUTime = max(maxCPUTime, 0)
maxCPUTime = min(maxCPUTime, 86400 * 12.5)
if maxCPUTime and benchmarkSI00:
normCPUTime = NORMALIZATIONCONSTANT * maxCPUTime * benchmarkSI00
else:
if not benchmarkSI00:
subClusterUniqueID = ceInfoDict['SubClusterUniqueID']
return S_ERROR('benchmarkSI00 info not available for %s' %
subClusterUniqueID)
if not maxCPUTime:
return S_ERROR('maxCPUTime info not available')
return S_OK(normCPUTime)
def getCPUTime(CPUNormalizationFactor):
""" Trying to get CPUTime (in seconds) from the CS. The default is a (low) 10000s.
This is a generic method, independent from the middleware of the resource.
"""
CPUTime = gConfig.getValue('/LocalSite/CPUTimeLeft', 0)
if CPUTime:
# This is in HS06sseconds
# We need to convert in real seconds
CPUTime = CPUTime / int(CPUNormalizationFactor)
else:
# now we know that we have to find the CPUTimeLeft by looking in the CS
gridCE = gConfig.getValue('/LocalSite/GridCE')
CEQueue = gConfig.getValue('/LocalSite/CEQueue')
if not CEQueue:
# we have to look for a CEQueue in the CS
# A bit hacky. We should better profit from something generic
gLogger.warn(
"No CEQueue in local configuration, looking to find one in CS")
siteName = gConfig.getValue('/LocalSite/Site')
queueSection = '/Resources/Sites/%s/%s/CEs/%s/Queues' % (
siteName.split('.')[0], siteName, gridCE)
res = gConfig.getSections(queueSection)
if not res['OK']:
raise RuntimeError(res['Message'])
queues = res['Value']
CPUTimes = []
for queue in queues:
CPUTimes.append(
gConfig.getValue(
queueSection + '/' + queue + '/maxCPUTime', 10000))
cpuTimeInMinutes = min(CPUTimes)
# These are (real, wall clock) minutes - damn BDII!
CPUTime = int(cpuTimeInMinutes) * 60
else:
queueInfo = getQueueInfo('%s/%s' % (gridCE, CEQueue))
CPUTime = 10000
if not queueInfo['OK'] or not queueInfo['Value']:
gLogger.warn(
"Can't find a CE/queue, defaulting CPUTime to %d" %
CPUTime)
else:
queueCSSection = queueInfo['Value']['QueueCSSection']
# These are (real, wall clock) minutes - damn BDII!
cpuTimeInMinutes = gConfig.getValue('%s/maxCPUTime' %
queueCSSection)
if cpuTimeInMinutes:
CPUTime = int(cpuTimeInMinutes) * 60
gLogger.info("CPUTime for %s: %d" %
(queueCSSection, CPUTime))
else:
gLogger.warn(
"Can't find maxCPUTime for %s, defaulting CPUTime to %d"
% (queueCSSection, CPUTime))
return CPUTime
def getCPUTime( cpuNormalizationFactor ):
""" Trying to get CPUTime (in seconds) from the CS or from TimeLeft. The default is a (low) 3600s.
This is a generic method, independent from the middleware of the resource if TimeLeft doesn't return a value
"""
cpuTimeLeft = 0.
cpuWorkLeft = gConfig.getValue( '/LocalSite/CPUTimeLeft', 0 )
if not cpuWorkLeft:
# Try and get the information from the CPU left utility
result = TimeLeft().getTimeLeft()
if result['OK']:
cpuWorkLeft = result['Value']
if cpuWorkLeft:
# This is in HS06sseconds
# We need to convert in real seconds
if not cpuNormalizationFactor: # if cpuNormalizationFactor passed in is 0, try get it from the local cfg
cpuNormalizationFactor = gConfig.getValue( '/LocalSite/CPUNormalizationFactor', 0.0 )
if cpuNormalizationFactor:
cpuTimeLeft = cpuWorkLeft / cpuNormalizationFactor # this is a float
if not cpuTimeLeft:
# now we know that we have to find the CPUTimeLeft by looking in the CS
# this is not granted to be correct as the CS units may not be real seconds
gridCE = gConfig.getValue( '/LocalSite/GridCE' )
ceQueue = gConfig.getValue( '/LocalSite/CEQueue' )
if not ceQueue:
# we have to look for a CEQueue in the CS
# A bit hacky. We should better profit from something generic
gLogger.warn( "No CEQueue in local configuration, looking to find one in CS" )
siteName = gConfig.getValue( '/LocalSite/Site' )
queueSection = '/Resources/Sites/%s/%s/CEs/%s/Queues' % ( siteName.split( '.' )[0], siteName, gridCE )
res = gConfig.getSections( queueSection )
if not res['OK']:
raise RuntimeError( res['Message'] )
queues = res['Value']
cpuTimes = [gConfig.getValue( queueSection + '/' + queue + '/maxCPUTime', 10000. ) for queue in queues]
# These are (real, wall clock) minutes - damn BDII!
cpuTimeLeft = min( cpuTimes ) * 60
else:
queueInfo = getQueueInfo( '%s/%s' % ( gridCE, ceQueue ) )
cpuTimeLeft = 3600.
if not queueInfo['OK'] or not queueInfo['Value']:
gLogger.warn( "Can't find a CE/queue, defaulting CPUTime to %d" % cpuTimeLeft )
else:
queueCSSection = queueInfo['Value']['QueueCSSection']
# These are (real, wall clock) minutes - damn BDII!
cpuTimeInMinutes = gConfig.getValue( '%s/maxCPUTime' % queueCSSection, 0. )
if cpuTimeInMinutes:
cpuTimeLeft = cpuTimeInMinutes * 60.
gLogger.info( "CPUTime for %s: %f" % ( queueCSSection, cpuTimeLeft ) )
else:
gLogger.warn( "Can't find maxCPUTime for %s, defaulting CPUTime to %f" % ( queueCSSection, cpuTimeLeft ) )
return int( cpuTimeLeft )
def getCPUTime( CPUNormalizationFactor ):
""" Trying to get CPUTime (in seconds) from the CS. The default is a (low) 10000s.
This is a generic method, independent from the middleware of the resource.
"""
CPUTime = gConfig.getValue( '/LocalSite/CPUTimeLeft', 0 )
if CPUTime:
# This is in HS06sseconds
# We need to convert in real seconds
if not CPUNormalizationFactor: # if CPUNormalizationFactor passed in is 0, try get it from the local cfg
CPUNormalizationFactor = gConfig.getValue( '/LocalSite/CPUNormalizationFactor', 0.0 )
# if CPUNormalizationFactor is not even in the local cfg, it's a problem, and yes the next line will raise an exception
CPUTime = CPUTime / int( CPUNormalizationFactor )
else:
# now we know that we have to find the CPUTimeLeft by looking in the CS
gridCE = gConfig.getValue( '/LocalSite/GridCE' )
CEQueue = gConfig.getValue( '/LocalSite/CEQueue' )
if not CEQueue:
# we have to look for a CEQueue in the CS
# A bit hacky. We should better profit from something generic
gLogger.warn( "No CEQueue in local configuration, looking to find one in CS" )
siteName = gConfig.getValue( '/LocalSite/Site' )
queueSection = '/Resources/Sites/%s/%s/CEs/%s/Queues' % ( siteName.split( '.' )[0], siteName, gridCE )
res = gConfig.getSections( queueSection )
if not res['OK']:
raise RuntimeError( res['Message'] )
queues = res['Value']
CPUTimes = []
for queue in queues:
CPUTimes.append( gConfig.getValue( queueSection + '/' + queue + '/maxCPUTime', 10000 ) )
cpuTimeInMinutes = min( CPUTimes )
# These are (real, wall clock) minutes - damn BDII!
CPUTime = int( cpuTimeInMinutes ) * 60
else:
queueInfo = getQueueInfo( '%s/%s' % ( gridCE, CEQueue ) )
CPUTime = 10000
if not queueInfo['OK'] or not queueInfo['Value']:
gLogger.warn( "Can't find a CE/queue, defaulting CPUTime to %d" % CPUTime )
else:
queueCSSection = queueInfo['Value']['QueueCSSection']
# These are (real, wall clock) minutes - damn BDII!
cpuTimeInMinutes = gConfig.getValue( '%s/maxCPUTime' % queueCSSection )
if cpuTimeInMinutes:
CPUTime = int( cpuTimeInMinutes ) * 60
gLogger.info( "CPUTime for %s: %d" % ( queueCSSection, CPUTime ) )
else:
gLogger.warn( "Can't find maxCPUTime for %s, defaulting CPUTime to %d" % ( queueCSSection, CPUTime ) )
return CPUTime
def getQueueNormalization( ceUniqueID ):
"""
Report Normalization Factor applied by Site to the given Queue
"""
result = getQueueInfo( ceUniqueID )
if not result['OK']:
return result
ceInfoDict = result['Value']
subClusterUniqueID = ceInfoDict['SubClusterUniqueID']
benchmarkSI00 = ceInfoDict['SI00']
if benchmarkSI00:
return S_OK( benchmarkSI00 )
else:
return S_ERROR( 'benchmarkSI00 info not available for %s' % subClusterUniqueID )
def getQueueNormalization(ceUniqueID):
"""
Report Normalization Factor applied by Site to the given Queue
"""
result = getQueueInfo(ceUniqueID)
if not result['OK']:
return result
ceInfoDict = result['Value']
subClusterUniqueID = ceInfoDict['SubClusterUniqueID']
benchmarkSI00 = ceInfoDict['SI00']
if benchmarkSI00:
return S_OK(benchmarkSI00)
else:
return S_ERROR('benchmarkSI00 info not available for %s' %
subClusterUniqueID)
def getQueueNormalization(ceUniqueID):
"""
Report Normalization Factor applied by Site to the given Queue
"""
result = getQueueInfo(ceUniqueID)
if not result["OK"]:
return result
ceInfoDict = result["Value"]
siteCSSEction = ceInfoDict["SiteCSSEction"]
queueCSSection = ceInfoDict["QueueCSSection"]
subClusterUniqueID = ceInfoDict["SubClusterUniqueID"]
benchmarkSI00 = __getQueueNormalization(queueCSSection, siteCSSEction)
if benchmarkSI00:
return S_OK(benchmarkSI00)
else:
return S_ERROR("benchmarkSI00 info not available for %s" % subClusterUniqueID)
def getQueueNormalization( ceUniqueID ):
"""
Report Normalization Factor applied by Site to the given Queue
"""
result = getQueueInfo( ceUniqueID )
if not result['OK']:
return result
ceInfoDict = result['Value']
siteCSSEction = ceInfoDict['SiteCSSEction']
queueCSSection = ceInfoDict['QueueCSSection']
benchmarkSI00 = __getQueueNormalization( queueCSSection, siteCSSEction )
if benchmarkSI00:
return S_OK( benchmarkSI00 )
else:
return S_ERROR( 'benchmarkSI00 info not available for %s' % subClusterUniqueID )
errorList.append( ( subClusterUniqueID , 'benchmarkSI00 info not available' ) )
exitCode = 3
pass
def getQueueNormalization( ceUniqueID ):
"""
Report Normalization Factor applied by Site to the given Queue
"""
result = getQueueInfo( ceUniqueID )
if not result['OK']:
return result
ceInfoDict = result['Value']
siteCSSEction = ceInfoDict['SiteCSSEction']
queueCSSection = ceInfoDict['QueueCSSection']
benchmarkSI00 = __getQueueNormalization( queueCSSection, siteCSSEction )
if benchmarkSI00:
return S_OK( benchmarkSI00 )
else:
return S_ERROR( 'benchmarkSI00 info not available for %s' % subClusterUniqueID )
errorList.append( ( subClusterUniqueID , 'benchmarkSI00 info not available' ) )
exitCode = 3
pass
def getCPUTime(cpuNormalizationFactor):
""" Trying to get CPUTime (in seconds) from the CS or from TimeLeft. The default is a (low) 3600s.
This is a generic method, independent from the middleware of the resource if TimeLeft doesn't return a value
"""
cpuTimeLeft = 0.
cpuWorkLeft = gConfig.getValue('/LocalSite/CPUTimeLeft', 0)
if not cpuWorkLeft:
# Try and get the information from the CPU left utility
result = TimeLeft().getTimeLeft()
if result['OK']:
cpuWorkLeft = result['Value']
if cpuWorkLeft:
# This is in HS06sseconds
# We need to convert in real seconds
if not cpuNormalizationFactor: # if cpuNormalizationFactor passed in is 0, try get it from the local cfg
cpuNormalizationFactor = gConfig.getValue(
'/LocalSite/CPUNormalizationFactor', 0.0)
if cpuNormalizationFactor:
cpuTimeLeft = cpuWorkLeft / cpuNormalizationFactor # this is a float
if not cpuTimeLeft:
# now we know that we have to find the CPUTimeLeft by looking in the CS
# this is not granted to be correct as the CS units may not be real seconds
gridCE = gConfig.getValue('/LocalSite/GridCE')
ceQueue = gConfig.getValue('/LocalSite/CEQueue')
if not ceQueue:
# we have to look for a CEQueue in the CS
# A bit hacky. We should better profit from something generic
gLogger.warn(
"No CEQueue in local configuration, looking to find one in CS")
siteName = gConfig.getValue('/LocalSite/Site')
queueSection = '/Resources/Sites/%s/%s/CEs/%s/Queues' % (
siteName.split('.')[0], siteName, gridCE)
res = gConfig.getSections(queueSection)
if not res['OK']:
raise RuntimeError(res['Message'])
queues = res['Value']
cpuTimes = [
gConfig.getValue(queueSection + '/' + queue + '/maxCPUTime',
10000.) for queue in queues
]
# These are (real, wall clock) minutes - damn BDII!
cpuTimeLeft = min(cpuTimes) * 60
else:
queueInfo = getQueueInfo('%s/%s' % (gridCE, ceQueue))
cpuTimeLeft = 3600.
if not queueInfo['OK'] or not queueInfo['Value']:
gLogger.warn(
"Can't find a CE/queue, defaulting CPUTime to %d" %
cpuTimeLeft)
else:
queueCSSection = queueInfo['Value']['QueueCSSection']
# These are (real, wall clock) minutes - damn BDII!
cpuTimeInMinutes = gConfig.getValue(
'%s/maxCPUTime' % queueCSSection, 0.)
if cpuTimeInMinutes:
cpuTimeLeft = cpuTimeInMinutes * 60.
gLogger.info("CPUTime for %s: %f" %
(queueCSSection, cpuTimeLeft))
else:
gLogger.warn(
"Can't find maxCPUTime for %s, defaulting CPUTime to %f"
% (queueCSSection, cpuTimeLeft))
return int(cpuTimeLeft)
def getCPUTime( cpuNormalizationFactor ):
""" Trying to get CPUTime left for execution (in seconds).
It will first look to get the work left looking for batch system information useing the TimeLeft utility.
If it succeeds, it will convert it in real second, and return it.
If it fails, it tries to get it from the static info found in CS.
If it fails, it returns the default, which is a large 9999999, that we may consider as "Infinite".
This is a generic method, independent from the middleware of the resource if TimeLeft doesn't return a value
args:
cpuNormalizationFactor (float): the CPU power of the current Worker Node. If not passed in, it's get from the local configuration
returns:
cpuTimeLeft (int): the CPU time left, in seconds
"""
cpuTimeLeft = 0.
cpuWorkLeft = gConfig.getValue( '/LocalSite/CPUTimeLeft', 0 )
if not cpuWorkLeft:
# Try and get the information from the CPU left utility
result = TimeLeft().getTimeLeft()
if result['OK']:
cpuWorkLeft = result['Value']
if cpuWorkLeft > 0:
# This is in HS06sseconds
# We need to convert in real seconds
if not cpuNormalizationFactor: # if cpuNormalizationFactor passed in is 0, try get it from the local cfg
cpuNormalizationFactor = gConfig.getValue( '/LocalSite/CPUNormalizationFactor', 0.0 )
if cpuNormalizationFactor:
cpuTimeLeft = cpuWorkLeft / cpuNormalizationFactor # this is a float
if not cpuTimeLeft:
# now we know that we have to find the CPUTimeLeft by looking in the CS
# this is not granted to be correct as the CS units may not be real seconds
gridCE = gConfig.getValue( '/LocalSite/GridCE' )
ceQueue = gConfig.getValue( '/LocalSite/CEQueue' )
if not ceQueue:
# we have to look for a ceQueue in the CS
# A bit hacky. We should better profit from something generic
gLogger.warn( "No CEQueue in local configuration, looking to find one in CS" )
siteName = gConfig.getValue( '/LocalSite/Site' )
queueSection = '/Resources/Sites/%s/%s/CEs/%s/Queues' % ( siteName.split( '.' )[0], siteName, gridCE )
res = gConfig.getSections( queueSection )
if not res['OK']:
raise RuntimeError( res['Message'] )
queues = res['Value']
cpuTimes = [gConfig.getValue( queueSection + '/' + queue + '/maxCPUTime', 9999999. ) for queue in queues]
# These are (real, wall clock) minutes - damn BDII!
cpuTimeLeft = min( cpuTimes ) * 60
else:
queueInfo = getQueueInfo( '%s/%s' % ( gridCE, ceQueue ) )
cpuTimeLeft = 9999999.
if not queueInfo['OK'] or not queueInfo['Value']:
gLogger.warn( "Can't find a CE/queue, defaulting CPUTime to %d" % cpuTimeLeft )
else:
queueCSSection = queueInfo['Value']['QueueCSSection']
# These are (real, wall clock) minutes - damn BDII!
cpuTimeInMinutes = gConfig.getValue( '%s/maxCPUTime' % queueCSSection, 0. )
if cpuTimeInMinutes:
cpuTimeLeft = cpuTimeInMinutes * 60.
gLogger.info( "CPUTime for %s: %f" % ( queueCSSection, cpuTimeLeft ) )
else:
gLogger.warn( "Can't find maxCPUTime for %s, defaulting CPUTime to %f" % ( queueCSSection, cpuTimeLeft ) )
return int( cpuTimeLeft )
def getCPUTime(cpuNormalizationFactor):
""" Trying to get CPUTime left for execution (in seconds).
It will first look to get the work left looking for batch system information useing the TimeLeft utility.
If it succeeds, it will convert it in real second, and return it.
If it fails, it tries to get it from the static info found in CS.
If it fails, it returns the default, which is a large 9999999, that we may consider as "Infinite".
This is a generic method, independent from the middleware of the resource if TimeLeft doesn't return a value
args:
cpuNormalizationFactor (float): the CPU power of the current Worker Node.
If not passed in, it's get from the local configuration
returns:
cpuTimeLeft (int): the CPU time left, in seconds
"""
cpuTimeLeft = 0.
cpuWorkLeft = gConfig.getValue('/LocalSite/CPUTimeLeft', 0)
if not cpuWorkLeft:
# Try and get the information from the CPU left utility
result = TimeLeft().getTimeLeft()
if result['OK']:
cpuWorkLeft = result['Value']
if cpuWorkLeft > 0:
# This is in HS06sseconds
# We need to convert in real seconds
if not cpuNormalizationFactor: # if cpuNormalizationFactor passed in is 0, try get it from the local cfg
cpuNormalizationFactor = gConfig.getValue(
'/LocalSite/CPUNormalizationFactor', 0.0)
if cpuNormalizationFactor:
cpuTimeLeft = cpuWorkLeft / cpuNormalizationFactor # this is a float
if not cpuTimeLeft:
# now we know that we have to find the CPUTimeLeft by looking in the CS
# this is not granted to be correct as the CS units may not be real seconds
gridCE = gConfig.getValue('/LocalSite/GridCE')
ceQueue = gConfig.getValue('/LocalSite/CEQueue')
if not ceQueue:
# we have to look for a ceQueue in the CS
# A bit hacky. We should better profit from something generic
gLogger.warn(
"No CEQueue in local configuration, looking to find one in CS")
siteName = gConfig.getValue('/LocalSite/Site')
queueSection = '/Resources/Sites/%s/%s/CEs/%s/Queues' % (
siteName.split('.')[0], siteName, gridCE)
res = gConfig.getSections(queueSection)
if not res['OK']:
raise RuntimeError(res['Message'])
queues = res['Value']
cpuTimes = [
gConfig.getValue(queueSection + '/' + queue + '/maxCPUTime',
9999999.) for queue in queues
]
# These are (real, wall clock) minutes - damn BDII!
cpuTimeLeft = min(cpuTimes) * 60
else:
queueInfo = getQueueInfo('%s/%s' % (gridCE, ceQueue))
cpuTimeLeft = 9999999.
if not queueInfo['OK'] or not queueInfo['Value']:
gLogger.warn(
"Can't find a CE/queue, defaulting CPUTime to %d" %
cpuTimeLeft)
else:
queueCSSection = queueInfo['Value']['QueueCSSection']
# These are (real, wall clock) minutes - damn BDII!
cpuTimeInMinutes = gConfig.getValue(
'%s/maxCPUTime' % queueCSSection, 0.)
if cpuTimeInMinutes:
cpuTimeLeft = cpuTimeInMinutes * 60.
gLogger.info("CPUTime for %s: %f" %
(queueCSSection, cpuTimeLeft))
else:
gLogger.warn(
"Can't find maxCPUTime for %s, defaulting CPUTime to %f"
% (queueCSSection, cpuTimeLeft))
return int(cpuTimeLeft)