def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
# Get the timestamp in the two formats we wanted
epoch = str(time.time())
now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {
'locationId': 'TIMESTAMP',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'TIMESTAMP',
'version': epoch,
'path': now,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.exception(e)
sys.stdout = sys.stderr
raise
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
# Get the timestamp in the two formats we wanted
epoch = str(time.time())
now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {'locationId': 'TIMESTAMP',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'TIMESTAMP',
'version': epoch,
'path': now,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.exception(e)
sys.stdout = sys.stderr
raise
def print_clusters(cp):
cluster_name = cp_get(cp, 'cluster', 'name', None)
if not cluster_name:
cluster_name = cp_get(cp, 'ce', 'hosting_cluster', None)
if not cluster_name:
cluster_name = cp_get(cp, 'ce', 'unique_name', None)
if not cluster_name:
getClusterName(cp)
#raise Exception("Could not determine cluster name.")
#clusterUniqueID = cp_get(cp, 'ce', 'unique_name', cluster_name)
clusterUniqueID = getClusterID(cp)
siteUniqueID = cp_get(cp, "site", "unique_name", 'UNKNOWN_SITE')
extraCEs = cp_get(cp, 'cluster', 'other_ces', [])
if extraCEs:
extraCEs = [x.strip() for x in extraCEs.split(',')]
ces = getCEList(cp, extraCEs)
glueClusters = ''
for ce in ces:
glueClusters += 'GlueForeignKey: GlueCEUniqueID=%s\n' % ce
bdii = cp_get(cp, 'gip', 'bdii', 'ldap://is.grid.iu.edu:2170')
info = { \
'cluster': cluster_name,
'clusterUniqueID': clusterUniqueID,
'tmp': cp_get(cp, "osg_dirs", "tmp", cp_get(cp, "osg_dirs", "data", \
"/tmp")),
'wn_tmp': cp_get(cp, "osg_dirs", "wn_tmp", "/tmp"),
'siteUniqueID': siteUniqueID,
'glueClusters': glueClusters,
'bdii': bdii,
}
template = getTemplate("GlueCluster", "GlueClusterUniqueID")
printTemplate(template, info)
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
osg_grid = cp_get(cp, "osg_dirs", "grid_dir", None)
if not osg_grid:
raise RuntimeError('grid_dir ($OSG_GRID) not defined!')
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {
'locationId': 'OSG_GRID',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'OSG_GRID',
'version': 1.0,
'path': osg_grid,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.error(e)
sys.stdout = sys.stderr
raise
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only:
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
osg_grid = cp_get(cp, "osg_dirs", "grid_dir", None)
if not osg_grid:
raise RuntimeError('grid_dir ($OSG_GRID) not defined!')
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {'locationId': 'OSG_GRID',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'OSG_GRID',
'version': 1.0,
'path': osg_grid,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.error(e)
sys.stdout = sys.stderr
raise
def print_clusters(cp):
cluster_name = cp_get(cp, "cluster", "name", None)
if not cluster_name:
cluster_name = cp_get(cp, "ce", "hosting_cluster", None)
if not cluster_name:
cluster_name = cp_get(cp, "ce", "unique_name", None)
if not cluster_name:
getClusterName(cp)
# raise Exception("Could not determine cluster name.")
# clusterUniqueID = cp_get(cp, 'ce', 'unique_name', cluster_name)
clusterUniqueID = getClusterID(cp)
siteUniqueID = cp_get(cp, "site", "unique_name", "UNKNOWN_SITE")
extraCEs = cp_get(cp, "cluster", "other_ces", [])
if extraCEs:
extraCEs = [x.strip() for x in extraCEs.split(",")]
ces = getCEList(cp, extraCEs)
glueClusters = ""
for ce in ces:
glueClusters += "GlueForeignKey: GlueCEUniqueID=%s\n" % ce
bdii = cp_get(cp, "gip", "bdii", "ldap://is.grid.iu.edu:2170")
info = {
"cluster": cluster_name,
"clusterUniqueID": clusterUniqueID,
"tmp": cp_get(cp, "osg_dirs", "tmp", cp_get(cp, "osg_dirs", "data", "/tmp")),
"wn_tmp": cp_get(cp, "osg_dirs", "wn_tmp", "/tmp"),
"siteUniqueID": siteUniqueID,
"glueClusters": glueClusters,
"bdii": bdii,
}
template = getTemplate("GlueCluster", "GlueClusterUniqueID")
printTemplate(template, info)
def print_Locations(cp):
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
for subClusterId in getSubClusterIDs(cp):
for entry in getApplications(cp):
entry['subClusterId'] = subClusterId
entry['clusterId'] = cluster_id
printTemplate(template, entry)
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only and 'VDT_LOCATION' in os.environ:
# get the VDT version
vdt_version_cmd = os.path.expandvars(
"$VDT_LOCATION/vdt/bin/") + 'vdt-version --no-wget'
vdt_version_out = runCommand(vdt_version_cmd).readlines()
gip_re = re.compile('Generic Information Provider\s+(.*?)\s*-.*')
gip_version = 'UNKNOWN'
for line in vdt_version_out:
m = gip_re.match(line)
if m:
gip_version = m.groups()[0]
break
gip_version += '; $Revision$'
# Get the timestamp in the two formats we wanted
now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {
'locationId': 'GIP_VERSION',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'GIP_VERSION',
'version': gip_version,
'path': now,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.exception(e)
sys.stdout = sys.stderr
raise
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only and 'VDT_LOCATION' in os.environ:
# get the VDT version
vdt_version_cmd = os.path.expandvars("$VDT_LOCATION/vdt/bin/") + 'vdt-version --no-wget'
vdt_version_out = runCommand(vdt_version_cmd).readlines()
gip_re = re.compile('Generic Information Provider\s+(.*?)\s*-.*')
gip_version = 'UNKNOWN'
for line in vdt_version_out:
m = gip_re.match(line)
if m:
gip_version = m.groups()[0]
break
gip_version += '; $Revision$'
# Get the timestamp in the two formats we wanted
now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {'locationId': 'GIP_VERSION',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'GIP_VERSION',
'version': gip_version,
'path': now,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.exception(e)
sys.stdout = sys.stderr
raise
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only and 'VDT_LOCATION' in os.environ:
# get the VDT version
vdt_version_cmd = os.path.expandvars("$VDT_LOCATION/vdt/bin/") + 'vdt-version --brief'
vdt_version = runCommand(vdt_version_cmd).readlines()[0].strip()
if (vdt_version == ""): vdt_version = "OLD_VDT"
# Get the timestamp in the two formats we wanted
now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {'locationId': 'VDT_VERSION',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'VDT_VERSION',
'version': vdt_version,
'path': now,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.exception(e)
sys.stdout = sys.stderr
raise
def main():
try:
# Load up the site configuration
cp = config()
se_only = cp_getBoolean(cp, "gip", "se_only", False)
if not se_only and 'VDT_LOCATION' in os.environ:
# get the VDT version
vdt_version_cmd = os.path.expandvars("$VDT_LOCATION/vdt/bin/") + 'vdt-version --brief'
vdt_version = runCommand(vdt_version_cmd).readlines()[0].strip()
if (vdt_version == ""): vdt_version = "OLD_VDT"
# Get the timestamp in the two formats we wanted
now = time.strftime("%a %b %d %T UTC %Y", time.gmtime())
# Load up the template for GlueLocationLocalID
# To view its contents, see $VDT_LOCATION/gip/templates/GlueCluster
template = getTemplate("GlueCluster", "GlueLocationLocalID")
cluster_id = getClusterID(cp)
for subClusterId in getSubClusterIDs(cp):
# Dictionary of data to fill in for GlueLocationLocalID
info = {'locationId': 'VDT_VERSION',
'subClusterId': subClusterId,
'clusterId': cluster_id,
'locationName': 'VDT_VERSION',
'version': vdt_version,
'path': now,
}
# Spit out our template, fill it with the appropriate info.
printTemplate(template, info)
except Exception, e:
# Log error, then report it via stderr.
log.exception(e)
sys.stdout = sys.stderr
raise
def print_CE(cp):
slurmVersion = getLrmsInfo(cp)
queueInfo = getQueueInfo(cp)
ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
CE = getTemplate("GlueCE", "GlueCEUniqueID")
try:
excludeQueues = [i.strip() for i in cp_get(cp, "slurm", \
"queue_exclude", "").split(',')]
except:
excludeQueues = []
vo_queues = getVoQueues(cp)
for queue, info in queueInfo.items():
if queue in excludeQueues:
continue
info["lrmsVersion"] = slurmVersion
info["job_manager"] = "slurm"
# if no jobs are waiting in the queue, set the number of free slots
# to (job_slots - running), or the total number of free slots on the cluster,
# whichever is less.
info["queue"] = queue
info["ceName"] = ce_name
unique_id = buildCEUniqueID(cp, ce_name, 'slurm', queue)
ceImpl, ceImplVersion = getCEImpl(cp)
port = getPort(cp)
info['ceUniqueID'] = unique_id
if "job_slots" not in info:
log.error("no job_slots found for %s!" % queue)
if "priority" not in info:
info["priority"] = 0
if "max_running" not in info:
log.error("no max_running found for %s!" % queue)
if "max_wall" not in info:
info["max_wall"] = 1440
info["free_slots"] = 0
if info["wait"] == 0:
freeSlots = info["job_slots"] - info["running"]
if freeSlots > 0:
info["free_slots"] = freeSlots
ert, wrt = responseTimes(cp,
info.get("running", 0),
info.get("wait", 0),
max_job_time=info["max_wall"])
info['ert'] = ert
info['wrt'] = wrt
info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name)
info['hostName'] = cp_get(cp, ce, 'host_name', ce_name)
info['ceImpl'] = ceImpl
info['ceImplVersion'] = ceImplVersion
contact_string = buildContactString(cp, 'slurm', queue, unique_id, log)
info['contact_string'] = contact_string
info['app_dir'] = cp_get(cp, 'osg_dirs', 'app', "/UNKNOWN_APP")
info['data_dir'] = cp_get(cp, 'osg_dirs', 'data', "/UNKNOWN_DATA")
info['default_se'] = getDefaultSE(cp)
if 'max_waiting' not in info:
info['max_waiting'] = 999999
if 'max_queuable' in info:
info['max_total'] = info['max_queuable']
info['free_slots'] = min(info['free_slots'], info['max_queuable'])
else:
info['max_total'] = info['max_waiting'] + info['max_running']
info['free_slots'] = min(info['free_slots'], info['max_total'])
# Enforce invariants:
# max_total <= max_running
# free_slots <= max_running
info['max_total'] = min(info['max_total'], info['max_running'])
info['free_slots'] = min(info['free_slots'], info['max_running'])
info['assigned'] = info['job_slots']
# Enforce invariants:
# assigned <= max_running
info['assigned'] = min(info['assigned'], info['max_running'])
info['lrmsType'] = 'slurm'
info['preemption'] = cp_get(cp, 'slurm', 'preemption', '0')
acbr = ''
has_vo = False
for vo, queue2 in vo_queues:
if queue == queue2:
acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo
has_vo = True
if not has_vo:
continue
info['acbr'] = acbr[:-1]
info['bdii'] = cp.get('bdii', 'endpoint')
gramVersion = getGramVersion(cp)
info['gramVersion'] = gramVersion
info['port'] = port
info['waiting'] = info['wait']
info['referenceSI00'] = gip_cluster.getReferenceSI00(cp)
info['clusterUniqueID'] = getClusterID(cp)
extraCapabilities = ''
if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'
htpcRSL, maxSlots = getHTPCInfo(cp, 'slurm', queue, log)
info['max_slots'] = maxSlots
if maxSlots > 1:
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'
info['extraCapabilities'] = extraCapabilities
info['htpc'] = htpcRSL
printTemplate(CE, info)
return queueInfo
def print_CE(cp):
SGEVersion = getLrmsInfo(cp)
queueInfo, _ = getQueueInfo(cp)
ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
ce_template = getTemplate("GlueCE", "GlueCEUniqueID")
queueList = getQueueList(cp)
vo_queues = getVoQueues(cp)
default_max_waiting = 999999
for queue in queueInfo.values():
if 'name' not in queue or queue['name'] not in queueList:
continue
if queue['name'] == 'waiting':
continue
unique_id = buildCEUniqueID(cp, ce_name, 'sge', queue['name'])
acbr = ''
for vo, queue2 in vo_queues:
if queue['name'] == queue2:
acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo
referenceSI00 = gip_cluster.getReferenceSI00(cp)
contact_string = buildContactString(cp, 'sge', queue['name'], unique_id, log)
extraCapabilities = ''
if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'
htpcRSL, maxSlots = getHTPCInfo(cp, 'sge', queue, log)
if maxSlots > 1:
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'
gramVersion = getGramVersion(cp)
port = getPort(cp)
ceImpl, ceImplVersion = getCEImpl(cp)
max_wall = queue["max_wall"]
if cp.has_option("sge", "max_wall"):
max_wall = cp_getInt(cp, "sge", "max_wall", 1440)
info = { \
"ceUniqueID" : unique_id,
"ceName" : ce_name,
"ceImpl" : ceImpl,
"ceImplVersion" : ceImplVersion,
"clusterUniqueID" : getClusterID(cp),
"queue" : queue['name'],
"priority" : queue['priority'],
"lrmsType" : 'sge',
"lrmsVersion" : SGEVersion,
"job_manager" : "sge",
"job_slots" : queue["slots_total"],
"free_slots" : queue["slots_free"],
"running" : queue["slots_used"],
"status" : queue['status'],
"total" : queue['slots_used'] + queue['waiting'],
"ert" : 3600,
"wrt" : 3600,
"hostingCluster" : cp_get(cp, ce, 'hosting_cluster', ce_name),
"hostName" : cp_get(cp, ce, 'host_name', ce_name),
"contact_string" : contact_string,
"app_dir" : cp_get(cp, 'osg_dirs', 'app', "/OSG_APP_UNKNOWN"),
"data_dir" : cp_get(cp, 'osg_dirs', 'data', "/OSG_DATA_UNKNOWN"),
"default_se" : getDefaultSE(cp),
"max_running" : queue["slots_total"],
"max_wall" : max_wall,
"max_waiting" : default_max_waiting,
"max_slots" : maxSlots,
"max_total" : default_max_waiting + queue["slots_total"],
"assigned" : queue["slots_used"],
"preemption" : cp_get(cp, 'sge', 'preemption', '0'),
"acbr" : acbr[:-1],
"bdii": cp.get('bdii', 'endpoint'),
"gramVersion" : gramVersion,
"port" : port,
"waiting" : queue['waiting'],
"referenceSI00": referenceSI00,
'extraCapabilities' : extraCapabilities,
"htpc" : htpcRSL
}
printTemplate(ce_template, info)
return queueInfo
def print_CE(cp):
pbsVersion = getLrmsInfo(cp)
queueInfo = getQueueInfo(cp)
totalCpu, freeCpu, queueCpus = parseNodes(cp, pbsVersion)
log.debug("totalCpu, freeCpu, queueCPus: %s %s %s" % (totalCpu, freeCpu, queueCpus))
ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
CE = getTemplate("GlueCE", "GlueCEUniqueID")
try:
excludeQueues = [i.strip() for i in cp_get(cp, "pbs", \
"queue_exclude", "").split(',')]
except:
excludeQueues = []
vo_queues = getVoQueues(cp)
for queue, info in queueInfo.items():
if queue in excludeQueues:
continue
info["lrmsVersion"] = pbsVersion
info["job_manager"] = "pbs"
# if no jobs are waiting in the queue, set the number of free slots
# to (job_slots - running), or the total number of free slots on the cluster,
# whichever is less.
info["queue"] = queue
info["ceName"] = ce_name
unique_id = buildCEUniqueID(cp, ce_name, 'pbs', queue)
ceImpl, ceImplVersion = getCEImpl(cp)
port = getPort(cp)
info['ceUniqueID'] = unique_id
if "job_slots" not in info:
info["job_slots"] = totalCpu
if "priority" not in info:
info["priority"] = 0
if "max_running" not in info:
info["max_running"] = info["job_slots"]
if "max_wall" not in info:
info["max_wall"] = 1440
info["free_slots"] = 0
if info["wait"] == 0:
freeSlots = info["job_slots"] - info["running"]
if freeSlots > 0:
info["free_slots"] = min(freeSlots, freeCpu)
log.debug("queue info: %s %s" % (queue, info))
ert, wrt = responseTimes(cp, info.get("running", 0),
info.get("wait", 0), max_job_time=info["max_wall"])
info["job_slots"] = min(totalCpu, info["job_slots"])
info['ert'] = ert
info['wrt'] = wrt
info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name)
info['hostName'] = cp_get(cp, ce, 'host_name', ce_name)
info['ceImpl'] = ceImpl
info['ceImplVersion'] = ceImplVersion
contact_string = buildContactString(cp, 'pbs', queue, unique_id, log)
info['contact_string'] = contact_string
info['app_dir'] = cp_get(cp, 'osg_dirs', 'app', "/UNKNOWN_APP")
info['data_dir'] = cp_get(cp, 'osg_dirs', 'data', "/UNKNOWN_DATA")
info['default_se'] = getDefaultSE(cp)
if 'max_waiting' not in info:
info['max_waiting'] = 999999
if 'max_queuable' in info:
info['max_total'] = info['max_queuable']
info['free_slots'] = min(info['free_slots'], info['max_queuable'])
else:
info['max_total'] = info['max_waiting'] + info['max_running']
info['free_slots'] = min(info['free_slots'], info['max_total'])
# Enforce invariants:
# max_total <= max_running
# free_slots <= max_running
info['max_total'] = min(info['max_total'], info['max_running'])
info['free_slots'] = min(info['free_slots'], info['max_running'])
info['assigned'] = info['job_slots']
# Enforce invariants:
# assigned <= max_running
info['assigned'] = min(info['assigned'], info['max_running'])
info['lrmsType'] = 'pbs'
info['preemption'] = cp_get(cp, 'pbs', 'preemption', '0')
acbr = ''
has_vo = False
for vo, queue2 in vo_queues:
if queue == queue2:
acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo
has_vo = True
if not has_vo:
continue
info['acbr'] = acbr[:-1]
info['bdii'] = cp.get('bdii', 'endpoint')
gramVersion = getGramVersion(cp)
info['gramVersion'] = gramVersion
info['port'] = port
info['waiting'] = info['wait']
info['referenceSI00'] = gip_cluster.getReferenceSI00(cp)
info['clusterUniqueID'] = getClusterID(cp)
extraCapabilities = ''
if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'
htpcRSL, maxSlots = getHTPCInfo(cp, 'pbs', queue, log)
info['max_slots'] = maxSlots
if maxSlots > 1:
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'
info['extraCapabilities'] = extraCapabilities
info['htpc'] = htpcRSL
print CE % info
return queueInfo, totalCpu, freeCpu, queueCpus
def print_CE(cp):
slurmVersion = getLrmsInfo(cp)
queueInfo = getQueueInfo(cp)
ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
CE = getTemplate("GlueCE", "GlueCEUniqueID")
try:
excludeQueues = [i.strip() for i in cp_get(cp, "slurm", "queue_exclude", "").split(",")]
except:
excludeQueues = []
vo_queues = getVoQueues(cp)
for queue, info in queueInfo.items():
if queue in excludeQueues:
continue
info["lrmsVersion"] = slurmVersion
info["job_manager"] = "slurm"
# if no jobs are waiting in the queue, set the number of free slots
# to (job_slots - running), or the total number of free slots on the cluster,
# whichever is less.
info["queue"] = queue
info["ceName"] = ce_name
unique_id = buildCEUniqueID(cp, ce_name, "slurm", queue)
ceImpl, ceImplVersion = getCEImpl(cp)
port = getPort(cp)
info["ceUniqueID"] = unique_id
if "job_slots" not in info:
log.error("no job_slots found for %s!" % queue)
if "priority" not in info:
info["priority"] = 0
if "max_running" not in info:
log.error("no max_running found for %s!" % queue)
if "max_wall" not in info:
info["max_wall"] = 1440
info["free_slots"] = 0
if info["wait"] == 0:
freeSlots = info["job_slots"] - info["running"]
if freeSlots > 0:
info["free_slots"] = freeSlots
ert, wrt = responseTimes(cp, info.get("running", 0), info.get("wait", 0), max_job_time=info["max_wall"])
info["ert"] = ert
info["wrt"] = wrt
info["hostingCluster"] = cp_get(cp, ce, "hosting_cluster", ce_name)
info["hostName"] = cp_get(cp, ce, "host_name", ce_name)
info["ceImpl"] = ceImpl
info["ceImplVersion"] = ceImplVersion
contact_string = buildContactString(cp, "slurm", queue, unique_id, log)
info["contact_string"] = contact_string
info["app_dir"] = cp_get(cp, "osg_dirs", "app", "/UNKNOWN_APP")
info["data_dir"] = cp_get(cp, "osg_dirs", "data", "/UNKNOWN_DATA")
info["default_se"] = getDefaultSE(cp)
if "max_waiting" not in info:
info["max_waiting"] = 999999
if "max_queuable" in info:
info["max_total"] = info["max_queuable"]
info["free_slots"] = min(info["free_slots"], info["max_queuable"])
else:
info["max_total"] = info["max_waiting"] + info["max_running"]
info["free_slots"] = min(info["free_slots"], info["max_total"])
# Enforce invariants:
# max_total <= max_running
# free_slots <= max_running
info["max_total"] = min(info["max_total"], info["max_running"])
info["free_slots"] = min(info["free_slots"], info["max_running"])
info["assigned"] = info["job_slots"]
# Enforce invariants:
# assigned <= max_running
info["assigned"] = min(info["assigned"], info["max_running"])
info["lrmsType"] = "slurm"
info["preemption"] = cp_get(cp, "slurm", "preemption", "0")
acbr = ""
has_vo = False
for vo, queue2 in vo_queues:
if queue == queue2:
acbr += "GlueCEAccessControlBaseRule: VO:%s\n" % vo
has_vo = True
if not has_vo:
continue
info["acbr"] = acbr[:-1]
info["bdii"] = cp.get("bdii", "endpoint")
gramVersion = getGramVersion(cp)
info["gramVersion"] = gramVersion
info["port"] = port
info["waiting"] = info["wait"]
info["referenceSI00"] = gip_cluster.getReferenceSI00(cp)
info["clusterUniqueID"] = getClusterID(cp)
extraCapabilities = ""
if cp_getBoolean(cp, "site", "glexec_enabled", False):
extraCapabilities = extraCapabilities + "\n" + "GlueCECapability: glexec"
htpcRSL, maxSlots = getHTPCInfo(cp, "slurm", queue, log)
info["max_slots"] = maxSlots
if maxSlots > 1:
extraCapabilities = extraCapabilities + "\n" + "GlueCECapability: htpc"
info["extraCapabilities"] = extraCapabilities
info["htpc"] = htpcRSL
printTemplate(CE, info)
return queueInfo
def print_CE(cp):
"""
Print out the GlueCE objects for LSF; one GlueCE per grid queue.
"""
try:
lsfVersion = getLrmsInfo(cp)
except:
lsfVersion = 'Unknown'
log.debug('Using LSF version %s' % lsfVersion)
queueInfo = getQueueInfo(cp)
try:
totalCpu, freeCpu, queueCpus = parseNodes(queueInfo, cp)
except:
#raise
totalCpu, freeCpu, queueCpus = 0, 0, {}
log.debug('Total, Free CPU: (%s, %s)' % (totalCpu, freeCpu))
ce_name = cp.get(ce, "name")
CE = getTemplate("GlueCE", "GlueCEUniqueID")
try:
excludeQueues = [i.strip() for i in cp.get("lsf", \
"queue_exclude").split(',')]
except:
excludeQueues = []
vo_queues = getVoQueues(queueInfo, cp)
for queue, info in queueInfo.items():
if queue in excludeQueues:
continue
log.debug('Processing queue %s' % queue)
if 'running' not in info:
info['running'] = 0
if 'status' not in info:
# There really should be an unknown status...
info['status'] = 'Closed'
if 'total' not in info:
info['total'] = 0
info["lrmsVersion"] = lsfVersion
info["job_manager"] = "lsf"
if int(info.get("wait", 0)) > 0:
info["free_slots"] = 0
else:
if queue in queueCpus and 'max' in queueCpus[queue] and 'njobs' in queueCpus[queue]:
info["free_slots"] = queueCpus[queue]['max'] - queueCpus[queue]['njobs']
else:
info["free_slots"] = freeCpu
info["queue"] = queue
info["ceName"] = ce_name
unique_id = buildCEUniqueID(cp, ce_name, 'lsf', queue)
info['ceUniqueID'] = unique_id
if "job_slots" not in info:
if queue in queueCpus and 'max' in queueCpus[queue]:
log.debug('queue %s, info is %s' % (queue, queueCpus[queue]))
info['job_slots'] = queueCpus[queue]['max']
else:
info["job_slots"] = totalCpu
if "priority" not in info:
info["priority"] = 0
if "max_running" not in info:
info["max_running"] = info["job_slots"]
elif not info['max_running'] or info['max_running'] == '-':
info['max_running'] = 999999
if cp.has_option("lsf", "max_wall"):
info["max_wall"] = cp_getInt(cp, "lsf", "max_wall", 1440)
else:
if "max_wall" not in info:
info["max_wall"] = 1440
info["max_wall"] = int(info["max_wall"]) # glue proscribes ints
info["job_slots"] = min(totalCpu, info["job_slots"])
ert, wrt = responseTimes(cp, info["running"], info["wait"],
max_job_time=info["max_wall"])
contact_string = buildContactString(cp, 'lsf', queue, unique_id, log)
ceImpl, ceImplVersion = getCEImpl(cp)
info['ert'] = ert
info['wrt'] = wrt
info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name)
info['hostName'] = cp_get(cp, ce, 'host_name', ce_name)
info['ceImpl'] = ceImpl
info['ceImplVersion'] = ceImplVersion
info['contact_string'] = contact_string
info['app_dir'] = cp.get('osg_dirs', 'app')
info['data_dir'] = cp.get('osg_dirs', 'data')
info['default_se'] = getDefaultSE(cp)
info['max_waiting'] = 999999
#info['max_total'] = info['max_running']
info['max_total'] = info['max_waiting'] + info['max_running']
info['assigned'] = info['job_slots']
info['lrmsType'] = 'lsf'
info['preemption'] = str(cp_getInt(cp, 'lsf', 'preemption', '0'))
acbr = ''
for vo, queue2 in vo_queues:
if queue == queue2:
acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo.lower()
if not acbr:
continue
#print info
info['acbr'] = acbr[:-1]
info['bdii'] = cp.get('bdii', 'endpoint')
gramVersion = getGramVersion(cp)
port = getPort(cp)
info['gramVersion'] = gramVersion
info['port'] = port
info['waiting'] = info.get('wait', 0)
info['referenceSI00'] = gip_cluster.getReferenceSI00(cp)
info['clusterUniqueID'] = getClusterID(cp)
extraCapabilities = ''
if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'
htpcRSL, maxSlots = getHTPCInfo(cp, 'lsf', queue, log)
info['max_slots'] = maxSlots
info['htpc'] = htpcRSL
if maxSlots > 1:
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'
info['extraCapabilities'] = extraCapabilities
printTemplate(CE, info)
return queueInfo, totalCpu, freeCpu, queueCpus
"priority" : ginfo.get('prio', 0),
"assigned" : assigned,
"max_slots" : maxSlots,
"preemption" : str(int(cp_getBoolean(cp, "condor", \
"preemption", False))),
"max_running" : max_running,
"max_waiting" : 99999,
"max_total" : 99999,
"max_wall" : cp_getInt(cp, "condor", "max_wall", 1440),
"status" : status,
'app_dir' : cp_get(cp, 'osg_dirs', 'app', '/Unknown'),
"data_dir" : cp_get(cp, "osg_dirs", "data", "/Unknown"),
"default_se" : getDefaultSE(cp),
"acbr" : ginfo['acbr'],
"referenceSI00" : referenceSI00,
"clusterUniqueID": getClusterID(cp),
"bdii" : cp_get(cp, "bdii", "endpoint", "Unknown"),
'extraCapabilities' : extraCapabilities,
"htpc" : htpcRSL
}
printTemplate(ce_template, info)
return total_nodes, claimed, unclaimed
def print_VOViewLocal(cp):
"""
Print the GLUE VOView entity; shows the VO's view of the condor batch
system.
Config options used:
* ce.name. The human-readable name of the ce.
"priority" : ginfo.get('prio', 0),
"assigned" : assigned,
"max_slots" : maxSlots,
"preemption" : str(int(cp_getBoolean(cp, "condor", \
"preemption", False))),
"max_running" : max_running,
"max_waiting" : 99999,
"max_total" : 99999,
"max_wall" : cp_getInt(cp, "condor", "max_wall", 1440),
"status" : status,
'app_dir' : cp_get(cp, 'osg_dirs', 'app', '/Unknown'),
"data_dir" : cp_get(cp, "osg_dirs", "data", "/Unknown"),
"default_se" : getDefaultSE(cp),
"acbr" : ginfo['acbr'],
"referenceSI00" : referenceSI00,
"clusterUniqueID": getClusterID(cp),
"bdii" : cp_get(cp, "bdii", "endpoint", "Unknown"),
'extraCapabilities' : extraCapabilities,
"htpc" : htpcRSL
}
printTemplate(ce_template, info)
return total_nodes, claimed, unclaimed
def print_VOViewLocal(cp):
"""
Print the GLUE VOView entity; shows the VO's view of the condor batch
system.
Config options used:
* ce.name. The human-readable name of the ce.
* condor.status. The status of condor; defaults to "Production"
def print_CE(cp):
"""
Print out the GlueCE objects for LSF; one GlueCE per grid queue.
"""
try:
lsfVersion = getLrmsInfo(cp)
except:
lsfVersion = 'Unknown'
log.debug('Using LSF version %s' % lsfVersion)
queueInfo = getQueueInfo(cp)
try:
totalCpu, freeCpu, queueCpus = parseNodes(queueInfo, cp)
except:
#raise
totalCpu, freeCpu, queueCpus = 0, 0, {}
log.debug('Total, Free CPU: (%s, %s)' % (totalCpu, freeCpu))
ce_name = cp.get(ce, "name")
CE = getTemplate("GlueCE", "GlueCEUniqueID")
try:
excludeQueues = [i.strip() for i in cp.get("lsf", \
"queue_exclude").split(',')]
except:
excludeQueues = []
vo_queues = getVoQueues(queueInfo, cp)
for queue, info in queueInfo.items():
if queue in excludeQueues:
continue
log.debug('Processing queue %s' % queue)
if 'running' not in info:
info['running'] = 0
if 'status' not in info:
# There really should be an unknown status...
info['status'] = 'Closed'
if 'total' not in info:
info['total'] = 0
info["lrmsVersion"] = lsfVersion
info["job_manager"] = "lsf"
if int(info.get("wait", 0)) > 0:
info["free_slots"] = 0
else:
if queue in queueCpus and 'max' in queueCpus[
queue] and 'njobs' in queueCpus[queue]:
info["free_slots"] = queueCpus[queue]['max'] - queueCpus[
queue]['njobs']
else:
info["free_slots"] = freeCpu
info["queue"] = queue
info["ceName"] = ce_name
unique_id = buildCEUniqueID(cp, ce_name, 'lsf', queue)
info['ceUniqueID'] = unique_id
if "job_slots" not in info:
if queue in queueCpus and 'max' in queueCpus[queue]:
log.debug('queue %s, info is %s' % (queue, queueCpus[queue]))
info['job_slots'] = queueCpus[queue]['max']
else:
info["job_slots"] = totalCpu
if "priority" not in info:
info["priority"] = 0
if "max_running" not in info:
info["max_running"] = info["job_slots"]
elif not info['max_running'] or info['max_running'] == '-':
info['max_running'] = 999999
if cp.has_option("lsf", "max_wall"):
info["max_wall"] = cp_getInt(cp, "lsf", "max_wall", 1440)
else:
if "max_wall" not in info:
info["max_wall"] = 1440
info["max_wall"] = int(info["max_wall"]) # glue proscribes ints
info["job_slots"] = min(totalCpu, info["job_slots"])
ert, wrt = responseTimes(cp,
info["running"],
info["wait"],
max_job_time=info["max_wall"])
contact_string = buildContactString(cp, 'lsf', queue, unique_id, log)
ceImpl, ceImplVersion = getCEImpl(cp)
info['ert'] = ert
info['wrt'] = wrt
info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name)
info['hostName'] = cp_get(cp, ce, 'host_name', ce_name)
info['ceImpl'] = ceImpl
info['ceImplVersion'] = ceImplVersion
info['contact_string'] = contact_string
info['app_dir'] = cp.get('osg_dirs', 'app')
info['data_dir'] = cp.get('osg_dirs', 'data')
info['default_se'] = getDefaultSE(cp)
info['max_waiting'] = 999999
#info['max_total'] = info['max_running']
info['max_total'] = info['max_waiting'] + info['max_running']
info['assigned'] = info['job_slots']
info['lrmsType'] = 'lsf'
info['preemption'] = str(cp_getInt(cp, 'lsf', 'preemption', '0'))
acbr = ''
for vo, queue2 in vo_queues:
if queue == queue2:
acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo.lower()
if not acbr:
continue
#print info
info['acbr'] = acbr[:-1]
info['bdii'] = cp.get('bdii', 'endpoint')
gramVersion = getGramVersion(cp)
port = getPort(cp)
info['gramVersion'] = gramVersion
info['port'] = port
info['waiting'] = info.get('wait', 0)
info['referenceSI00'] = gip_cluster.getReferenceSI00(cp)
info['clusterUniqueID'] = getClusterID(cp)
extraCapabilities = ''
if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'
htpcRSL, maxSlots = getHTPCInfo(cp, 'lsf', queue, log)
info['max_slots'] = maxSlots
info['htpc'] = htpcRSL
if maxSlots > 1:
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'
info['extraCapabilities'] = extraCapabilities
printTemplate(CE, info)
return queueInfo, totalCpu, freeCpu, queueCpus
def print_CE(cp):
SGEVersion = getLrmsInfo(cp)
queueInfo, _ = getQueueInfo(cp)
ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE")
ce_template = getTemplate("GlueCE", "GlueCEUniqueID")
queueList = getQueueList(cp)
vo_queues = getVoQueues(cp)
default_max_waiting = 999999
for queue in queueInfo.values():
if 'name' not in queue or queue['name'] not in queueList:
continue
if queue['name'] == 'waiting':
continue
unique_id = buildCEUniqueID(cp, ce_name, 'sge', queue['name'])
acbr = ''
for vo, queue2 in vo_queues:
if queue['name'] == queue2:
acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo
referenceSI00 = gip_cluster.getReferenceSI00(cp)
contact_string = buildContactString(cp, 'sge', queue['name'],
unique_id, log)
extraCapabilities = ''
if cp_getBoolean(cp, 'site', 'glexec_enabled', False):
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec'
htpcRSL, maxSlots = getHTPCInfo(cp, 'sge', queue, log)
if maxSlots > 1:
extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc'
gramVersion = getGramVersion(cp)
port = getPort(cp)
ceImpl, ceImplVersion = getCEImpl(cp)
max_wall = queue["max_wall"]
if cp.has_option("sge", "max_wall"):
max_wall = cp_getInt(cp, "sge", "max_wall", 1440)
info = { \
"ceUniqueID" : unique_id,
"ceName" : ce_name,
"ceImpl" : ceImpl,
"ceImplVersion" : ceImplVersion,
"clusterUniqueID" : getClusterID(cp),
"queue" : queue['name'],
"priority" : queue['priority'],
"lrmsType" : 'sge',
"lrmsVersion" : SGEVersion,
"job_manager" : "sge",
"job_slots" : queue["slots_total"],
"free_slots" : queue["slots_free"],
"running" : queue["slots_used"],
"status" : queue['status'],
"total" : queue['slots_used'] + queue['waiting'],
"ert" : 3600,
"wrt" : 3600,
"hostingCluster" : cp_get(cp, ce, 'hosting_cluster', ce_name),
"hostName" : cp_get(cp, ce, 'host_name', ce_name),
"contact_string" : contact_string,
"app_dir" : cp_get(cp, 'osg_dirs', 'app', "/OSG_APP_UNKNOWN"),
"data_dir" : cp_get(cp, 'osg_dirs', 'data', "/OSG_DATA_UNKNOWN"),
"default_se" : getDefaultSE(cp),
"max_running" : queue["slots_total"],
"max_wall" : max_wall,
"max_waiting" : default_max_waiting,
"max_slots" : maxSlots,
"max_total" : default_max_waiting + queue["slots_total"],
"assigned" : queue["slots_used"],
"preemption" : cp_get(cp, 'sge', 'preemption', '0'),
"acbr" : acbr[:-1],
"bdii": cp.get('bdii', 'endpoint'),
"gramVersion" : gramVersion,
"port" : port,
"waiting" : queue['waiting'],
"referenceSI00": referenceSI00,
'extraCapabilities' : extraCapabilities,
"htpc" : htpcRSL
}
printTemplate(ce_template, info)
return queueInfo
