def main(): try: cp = config() pbs_path = cp_get(cp, "pbs", "pbs_path", ".") addToPath(pbs_path) # adding pbs_path/bin to the path as well, since pbs/torque home # points to /usr/local and the binaries exist in /usr/local/bin addToPath(pbs_path + "/bin") vo_map = VoMapper(cp) pbsVersion = getLrmsInfo(cp) queueInfo, totalCpu, freeCpu, queueCpus = print_CE(cp) print_VOViewLocal(queueInfo, cp) except Exception, e: sys.stdout = sys.stderr log.exception(e) raise
def print_CE(cp): pbsVersion = getLrmsInfo(cp) queueInfo = getQueueInfo(cp) totalCpu, freeCpu, queueCpus = parseNodes(cp, pbsVersion) log.debug("totalCpu, freeCpu, queueCPus: %s %s %s" % (totalCpu, freeCpu, queueCpus)) ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE") CE = getTemplate("GlueCE", "GlueCEUniqueID") try: excludeQueues = [i.strip() for i in cp_get(cp, "pbs", \ "queue_exclude", "").split(',')] except: excludeQueues = [] vo_queues = getVoQueues(cp) for queue, info in queueInfo.items(): if queue in excludeQueues: continue info["lrmsVersion"] = pbsVersion info["job_manager"] = "pbs" # if no jobs are waiting in the queue, set the number of free slots # to (job_slots - running), or the total number of free slots on the cluster, # whichever is less. info["queue"] = queue info["ceName"] = ce_name unique_id = buildCEUniqueID(cp, ce_name, 'pbs', queue) ceImpl, ceImplVersion = getCEImpl(cp) port = getPort(cp) info['ceUniqueID'] = unique_id if "job_slots" not in info: info["job_slots"] = totalCpu if "priority" not in info: info["priority"] = 0 if "max_running" not in info: info["max_running"] = info["job_slots"] if cp.has_option("pbs", "max_wall"): info["max_wall"] = cp_getInt(cp, "pbs", "max_wall", 1440) else: if "max_wall" not in info: info["max_wall"] = 1440 info["free_slots"] = 0 if info["wait"] == 0: freeSlots = info["job_slots"] - info["running"] if freeSlots > 0: info["free_slots"] = min(freeSlots, freeCpu) log.debug("queue info: %s %s" % (queue, info)) ert, wrt = responseTimes(cp, info.get("running", 0), info.get("wait", 0), max_job_time=info["max_wall"]) info["job_slots"] = min(totalCpu, info["job_slots"]) info['ert'] = ert info['wrt'] = wrt info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name) info['hostName'] = cp_get(cp, ce, 'host_name', ce_name) info['ceImpl'] = ceImpl info['ceImplVersion'] = ceImplVersion contact_string = buildContactString(cp, 'pbs', queue, unique_id, log) info['contact_string'] = contact_string info['app_dir'] = cp_get(cp, 'osg_dirs', 'app', "/UNKNOWN_APP") info['data_dir'] = cp_get(cp, 'osg_dirs', 'data', "/UNKNOWN_DATA") info['default_se'] = getDefaultSE(cp) if 'max_waiting' not in info: info['max_waiting'] = 999999 if 'max_queuable' in info: info['max_total'] = info['max_queuable'] info['free_slots'] = min(info['free_slots'], info['max_queuable']) else: info['max_total'] = info['max_waiting'] + info['max_running'] info['free_slots'] = min(info['free_slots'], info['max_total']) # Enforce invariants: # max_total <= max_running # free_slots <= max_running info['max_total'] = min(info['max_total'], info['max_running']) info['free_slots'] = min(info['free_slots'], info['max_running']) info['assigned'] = info['job_slots'] # Enforce invariants: # assigned <= max_running info['assigned'] = min(info['assigned'], info['max_running']) info['lrmsType'] = 'pbs' info['preemption'] = cp_get(cp, 'pbs', 'preemption', '0') acbr = '' has_vo = False for vo, queue2 in vo_queues: if queue == queue2: acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo has_vo = True if not has_vo: continue info['acbr'] = acbr[:-1] info['bdii'] = cp.get('bdii', 'endpoint') gramVersion = getGramVersion(cp) info['gramVersion'] = gramVersion info['port'] = port info['waiting'] = info['wait'] info['referenceSI00'] = gip_cluster.getReferenceSI00(cp) info['clusterUniqueID'] = getClusterID(cp) extraCapabilities = '' if cp_getBoolean(cp, 'site', 'glexec_enabled', False): extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec' htpcRSL, maxSlots = getHTPCInfo(cp, 'pbs', queue, log) info['max_slots'] = maxSlots if maxSlots > 1: extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc' info['extraCapabilities'] = extraCapabilities info['htpc'] = htpcRSL printTemplate(CE, info) return queueInfo, totalCpu, freeCpu, queueCpus
def print_CE(cp): pbsVersion = getLrmsInfo(cp) queueInfo = getQueueInfo(cp) totalCpu, freeCpu, queueCpus = parseNodes(cp, pbsVersion) log.debug("totalCpu, freeCpu, queueCPus: %s %s %s" % (totalCpu, freeCpu, queueCpus)) ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE") CE = getTemplate("GlueCE", "GlueCEUniqueID") try: excludeQueues = [i.strip() for i in cp_get(cp, "pbs", "queue_exclude", "").split(",")] except: excludeQueues = [] vo_queues = getVoQueues(cp) for queue, info in queueInfo.items(): if queue in excludeQueues: continue info["lrmsVersion"] = pbsVersion info["job_manager"] = "pbs" # if no jobs are waiting in the queue, set the number of free slots # to (job_slots - running), or the total number of free slots on the cluster, # whichever is less. info["queue"] = queue info["ceName"] = ce_name unique_id = buildCEUniqueID(cp, ce_name, "pbs", queue) ceImpl, ceImplVersion = getCEImpl(cp) port = getPort(cp) info["ceUniqueID"] = unique_id if "job_slots" not in info: info["job_slots"] = totalCpu if "priority" not in info: info["priority"] = 0 if "max_running" not in info: info["max_running"] = info["job_slots"] if cp.has_option("pbs", "max_wall"): info["max_wall"] = cp_getInt(cp, "pbs", "max_wall", 1440) else: if "max_wall" not in info: info["max_wall"] = 1440 info["free_slots"] = 0 if info["wait"] == 0: freeSlots = info["job_slots"] - info["running"] if freeSlots > 0: info["free_slots"] = min(freeSlots, freeCpu) log.debug("queue info: %s %s" % (queue, info)) ert, wrt = responseTimes(cp, info.get("running", 0), info.get("wait", 0), max_job_time=info["max_wall"]) info["job_slots"] = min(totalCpu, info["job_slots"]) info["ert"] = ert info["wrt"] = wrt info["hostingCluster"] = cp_get(cp, ce, "hosting_cluster", ce_name) info["hostName"] = cp_get(cp, ce, "host_name", ce_name) info["ceImpl"] = ceImpl info["ceImplVersion"] = ceImplVersion contact_string = buildContactString(cp, "pbs", queue, unique_id, log) info["contact_string"] = contact_string info["app_dir"] = cp_get(cp, "osg_dirs", "app", "/UNKNOWN_APP") info["data_dir"] = cp_get(cp, "osg_dirs", "data", "/UNKNOWN_DATA") info["default_se"] = getDefaultSE(cp) if "max_waiting" not in info: info["max_waiting"] = 999999 if "max_queuable" in info: info["max_total"] = info["max_queuable"] info["free_slots"] = min(info["free_slots"], info["max_queuable"]) else: info["max_total"] = info["max_waiting"] + info["max_running"] info["free_slots"] = min(info["free_slots"], info["max_total"]) # Enforce invariants: # max_total <= max_running # free_slots <= max_running info["max_total"] = min(info["max_total"], info["max_running"]) info["free_slots"] = min(info["free_slots"], info["max_running"]) info["assigned"] = info["job_slots"] # Enforce invariants: # assigned <= max_running info["assigned"] = min(info["assigned"], info["max_running"]) info["lrmsType"] = "pbs" info["preemption"] = cp_get(cp, "pbs", "preemption", "0") acbr = "" has_vo = False for vo, queue2 in vo_queues: if queue == queue2: acbr += "GlueCEAccessControlBaseRule: VO:%s\n" % vo has_vo = True if not has_vo: continue info["acbr"] = acbr[:-1] info["bdii"] = cp.get("bdii", "endpoint") gramVersion = getGramVersion(cp) info["gramVersion"] = gramVersion info["port"] = port info["waiting"] = info["wait"] info["referenceSI00"] = gip_cluster.getReferenceSI00(cp) info["clusterUniqueID"] = getClusterID(cp) extraCapabilities = "" if cp_getBoolean(cp, "site", "glexec_enabled", False): extraCapabilities = extraCapabilities + "\n" + "GlueCECapability: glexec" htpcRSL, maxSlots = getHTPCInfo(cp, "pbs", queue, log) info["max_slots"] = maxSlots if maxSlots > 1: extraCapabilities = extraCapabilities + "\n" + "GlueCECapability: htpc" info["extraCapabilities"] = extraCapabilities info["htpc"] = htpcRSL printTemplate(CE, info) return queueInfo, totalCpu, freeCpu, queueCpus
def print_CE(cp): pbsVersion = getLrmsInfo(cp) queueInfo = getQueueInfo(cp) totalCpu, freeCpu, queueCpus = parseNodes(cp, pbsVersion) log.debug("totalCpu, freeCpu, queueCPus: %s %s %s" % (totalCpu, freeCpu, queueCpus)) ce_name = cp_get(cp, ce, "name", "UNKNOWN_CE") CE = getTemplate("GlueCE", "GlueCEUniqueID") try: excludeQueues = [i.strip() for i in cp_get(cp, "pbs", \ "queue_exclude", "").split(',')] except: excludeQueues = [] vo_queues = getVoQueues(cp) for queue, info in queueInfo.items(): if queue in excludeQueues: continue info["lrmsVersion"] = pbsVersion info["job_manager"] = "pbs" # if no jobs are waiting in the queue, set the number of free slots # to (job_slots - running), or the total number of free slots on the cluster, # whichever is less. info["queue"] = queue info["ceName"] = ce_name unique_id = buildCEUniqueID(cp, ce_name, 'pbs', queue) ceImpl, ceImplVersion = getCEImpl(cp) port = getPort(cp) info['ceUniqueID'] = unique_id if "job_slots" not in info: info["job_slots"] = totalCpu if "priority" not in info: info["priority"] = 0 if "max_running" not in info: info["max_running"] = info["job_slots"] if "max_wall" not in info: info["max_wall"] = 1440 info["free_slots"] = 0 if info["wait"] == 0: freeSlots = info["job_slots"] - info["running"] if freeSlots > 0: info["free_slots"] = min(freeSlots, freeCpu) log.debug("queue info: %s %s" % (queue, info)) ert, wrt = responseTimes(cp, info.get("running", 0), info.get("wait", 0), max_job_time=info["max_wall"]) info["job_slots"] = min(totalCpu, info["job_slots"]) info['ert'] = ert info['wrt'] = wrt info['hostingCluster'] = cp_get(cp, ce, 'hosting_cluster', ce_name) info['hostName'] = cp_get(cp, ce, 'host_name', ce_name) info['ceImpl'] = ceImpl info['ceImplVersion'] = ceImplVersion contact_string = buildContactString(cp, 'pbs', queue, unique_id, log) info['contact_string'] = contact_string info['app_dir'] = cp_get(cp, 'osg_dirs', 'app', "/UNKNOWN_APP") info['data_dir'] = cp_get(cp, 'osg_dirs', 'data', "/UNKNOWN_DATA") info['default_se'] = getDefaultSE(cp) if 'max_waiting' not in info: info['max_waiting'] = 999999 if 'max_queuable' in info: info['max_total'] = info['max_queuable'] info['free_slots'] = min(info['free_slots'], info['max_queuable']) else: info['max_total'] = info['max_waiting'] + info['max_running'] info['free_slots'] = min(info['free_slots'], info['max_total']) # Enforce invariants: # max_total <= max_running # free_slots <= max_running info['max_total'] = min(info['max_total'], info['max_running']) info['free_slots'] = min(info['free_slots'], info['max_running']) info['assigned'] = info['job_slots'] # Enforce invariants: # assigned <= max_running info['assigned'] = min(info['assigned'], info['max_running']) info['lrmsType'] = 'pbs' info['preemption'] = cp_get(cp, 'pbs', 'preemption', '0') acbr = '' has_vo = False for vo, queue2 in vo_queues: if queue == queue2: acbr += 'GlueCEAccessControlBaseRule: VO:%s\n' % vo has_vo = True if not has_vo: continue info['acbr'] = acbr[:-1] info['bdii'] = cp.get('bdii', 'endpoint') gramVersion = getGramVersion(cp) info['gramVersion'] = gramVersion info['port'] = port info['waiting'] = info['wait'] info['referenceSI00'] = gip_cluster.getReferenceSI00(cp) info['clusterUniqueID'] = getClusterID(cp) extraCapabilities = '' if cp_getBoolean(cp, 'site', 'glexec_enabled', False): extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: glexec' htpcRSL, maxSlots = getHTPCInfo(cp, 'pbs', queue, log) info['max_slots'] = maxSlots if maxSlots > 1: extraCapabilities = extraCapabilities + '\n' + 'GlueCECapability: htpc' info['extraCapabilities'] = extraCapabilities info['htpc'] = htpcRSL print CE % info return queueInfo, totalCpu, freeCpu, queueCpus