def main(): cp = config_file() # Read the CE entries from the BDII. entries = read_bdii(cp, query="(objectClass=GlueCE)") cluster_info = create_count_dict(entries) sc_info = sub_cluster_info(cluster_info.keys(), cp) specint = get_cpu_normalizations() for key, val in specint.items(): if isinstance(val, types.TupleType): specint[key] = val[0] ksi2k_info = {} site_dict = create_site_dict(entries, cp) sites = cp.get("site_normalization", "sites").split(",") sites = [i.strip() for i in sites] for cluster, cpu in cluster_info.items(): correct_sc_info(cluster, cpu, sc_info, specint) ksi2k = 0 sc_cores = 0 for sc in sc_info[cluster]: ksi2k += sc.glue["KSI2K"] sc_cores += int(sc.glue["SubClusterLogicalCPUs"]) try: site = site_dict[cluster] except: print "Problem with %s" % cluster continue if site in sites: print site, (ksi2k * 1000) / sc_cores
def main(): cp = config_file() # Read the CE entries from the BDII. entries = read_bdii(cp, query="(objectClass=GlueCE)") cluster_info = create_count_dict(entries) sc_info = sub_cluster_info(cluster_info.keys(), cp) specint = get_cpu_normalizations() for key, val in specint.items(): if isinstance(val, types.TupleType): specint[key] = val[0] ksi2k_info = {} site_dict = create_site_dict(entries, cp) sites = cp.get("site_normalization", "sites").split(",") sites = [i.strip() for i in sites] for cluster, cpu in cluster_info.items(): correct_sc_info(cluster, cpu, sc_info, specint) ksi2k = 0 sc_cores = 0 for sc in sc_info[cluster]: ksi2k += sc.glue["KSI2K"] sc_cores += int(sc.glue["SubClusterLogicalCPUs"]) try: site = site_dict[cluster] except: print "Problem with %s" % cluster continue if site in sites: print site, (ksi2k*1000) / sc_cores
def main(): cp = None for filename in ["$HOME/dbinfo/gip_password.conf", "/etc/gip_password.conf"]: filename = os.path.expandvars(filename) if os.path.exists(filename): cp = config_file(filename) break if cp == None: print "Could not find gip_password.conf in /etc!" sys.exit(1) fp = query_bdii(cp, "(objectClass=GlueVOView)") vo_entries = read_ldap(fp) fp = query_bdii(cp, "(objectClass=GlueCE)") ce_entries = read_ldap(fp) #conn = getGipDBConn(cp) #curs = conn.cursor() now = datetime.datetime.now() time_now = time.time() ProbeConfig = '/etc/osg-storage-report/ProbeConfig' Gratia.Initialize(ProbeConfig) gratia_info = {} ce_map, cluster_map, site_map = do_site_info(cp) sent_ce_entries = sets.Set() for entry in vo_entries: try: ce_entry = findCE(entry, ce_entries) except Exception, e: #print e #print entry log.error(e) continue try: info = {"time" : now, "runningJobs" : entry.glue["CEStateRunningJobs"], "totalCpus" : ce_entry.glue["CEInfoTotalCPUs"], "freeJobSlots" : entry.glue["CEStateFreeJobSlots"], "maxTotalJobs" : ce_entry.glue["CEPolicyMaxTotalJobs"], "totalJobs" : entry.glue["CEStateTotalJobs"], "status" : ce_entry.glue["CEStateStatus"], "lrmsType" : ce_entry.glue["CEInfoLRMSType"], "lrmsVersion" : ce_entry.glue["CEInfoLRMSVersion"], "vo" : entry.glue["VOViewLocalID"], "assignedJobSlots" : ce_entry.glue["CEPolicyAssignedJobSlots"], "freeCpus" : ce_entry.glue["CEStateFreeCPUs"], "waitingJobs" : entry.glue["CEStateWaitingJobs"], "maxRunningJobs" : ce_entry.glue["CEPolicyMaxRunningJobs"], "hostName" : ce_entry.glue["CEInfoHostName"], "queue" : ce_entry.glue["CEName"] } except KeyError, ke: log.exception(ke) log.warn(ce_entry) continue
def main(): cp = None for filename in ["$HOME/.cms_storage_record.conf", "/etc/cms_storage_record.conf"]: filename = os.path.expandvars(filename) if os.path.exists(filename): cp = config_file(filename) break if cp == None: print "Could not find cms_storage_record.conf in /etc!" sys.exit(1) ProbeConfig = '/etc/osg-storage-report/ProbeConfig' try: Gratia.Initialize(ProbeConfig) except Exception, e: log.exception(e) print e sys.exit(1)
def main(): cp = None for filename in [ "$HOME/.cms_storage_record.conf", "/etc/cms_storage_record.conf" ]: filename = os.path.expandvars(filename) if os.path.exists(filename): cp = config_file(filename) break if cp == None: print "Could not find cms_storage_record.conf in /etc!" sys.exit(1) ProbeConfig = '/etc/osg-storage-report/ProbeConfig' try: Gratia.Initialize(ProbeConfig) except Exception, e: log.exception(e) print e sys.exit(1)
def main(): # Load up the config file. cp = config_file() # Load the DB filename = os.path.expandvars("$HOME/dbinfo/DBParam.xml") if not os.path.exists(filename): filename = os.path.expandvars("$DBPARAM_LOCATION") if not os.path.exists(filename): filename = '/etc/DBParam.xml' x = XmlConfig(file=filename) conn = x.globals['GIPConnMan'].get_connection(None).get_connection() curs = conn.cursor() # Read the CE entries from the BDII. entries = read_bdii(cp, query="(&(objectClass=GlueCE))") cluster_info = create_count_dict(entries) # Map from the cluster hostname to the unique ID id_to_hostname = {} for entry in entries: fk = entry.glue['ForeignKey'] info = fk.split("=", 1) if len(info) != 2: continue if info[0] != "GlueClusterUniqueID": print >> sys.stderr, "Entry has unknown cluster FK: %s" % entry continue id = info[1] id_to_hostname[id] = entry.glue['CEHostingCluster'] sc_info = sub_cluster_info(id_to_hostname.keys(), cp) # For each unique cluster ID, map to one of the cluster hostnames new_sc_info = {} for id, info in sc_info.items(): if id not in id_to_hostname: print >> sys.stderr, "ID %s has no matching cluster hostname." % id continue new_sc_info[id_to_hostname[id]] = info sc_info = new_sc_info specint = get_cpu_normalizations() now = datetime.datetime.now() curs.execute("DELETE FROM cpu_score"); for cpu, score in specint.items(): if isinstance(score, types.TupleType): score = score[0] #specint[cpu] = score curs.execute("INSERT INTO cpu_score VALUES (%s, %s, %s)", (cpu, \ int(score), int(0))); site_ownership = create_site_dict(entries, cp) ownership = ownership_info(entries, cp) # Initialize the Probe's configuration ProbeConfig = '/etc/osg-storage-report/ProbeConfig' try: Gratia.Initialize(ProbeConfig) except Exception, e: print e raise
def main(): # Load up the config file. cp = config_file() # Load the DB filename = os.path.expandvars("$HOME/dbinfo/DBParam.xml") if not os.path.exists(filename): filename = os.path.expandvars("$DBPARAM_LOCATION") if not os.path.exists(filename): filename = '/etc/DBParam.xml' x = XmlConfig(file=filename) conn = x.globals['GIPConnMan'].get_connection(None).get_connection() curs = conn.cursor() # Read the CE entries from the BDII. entries = read_bdii(cp, query="(&(objectClass=GlueCE))") cluster_info = create_count_dict(entries) # Map from the cluster hostname to the unique ID id_to_hostname = {} for entry in entries: fk = entry.glue['ForeignKey'] info = fk.split("=", 1) if len(info) != 2: continue if info[0] != "GlueClusterUniqueID": print >> sys.stderr, "Entry has unknown cluster FK: %s" % entry continue id = info[1] id_to_hostname[id] = entry.glue['CEHostingCluster'] sc_info = sub_cluster_info(id_to_hostname.keys(), cp) # For each unique cluster ID, map to one of the cluster hostnames new_sc_info = {} for id, info in sc_info.items(): if id not in id_to_hostname: print >> sys.stderr, "ID %s has no matching cluster hostname." % id continue new_sc_info[id_to_hostname[id]] = info sc_info = new_sc_info specint = get_cpu_normalizations() now = datetime.datetime.now() curs.execute("DELETE FROM cpu_score") for cpu, score in specint.items(): if isinstance(score, types.TupleType): score = score[0] #specint[cpu] = score curs.execute("INSERT INTO cpu_score VALUES (%s, %s, %s)", (cpu, \ int(score), int(0))) site_ownership = create_site_dict(entries, cp) ownership = ownership_info(entries, cp) # Initialize the Probe's configuration ProbeConfig = '/etc/osg-storage-report/ProbeConfig' try: Gratia.Initialize(ProbeConfig) except Exception, e: print e raise
def main(): # Determine any filters we should apply parser = optparse.OptionParser(add_help_option=False) parser.add_option("-e", "--endpoint") parser.add_option("-h", "--help") parser.add_option("-c", "--config") parser.parse_args() (options, args) = parser.parse_args() if len(args) >= 1: ce_glob = args[0] else: ce_glob = "*" # Load up the config file. cp = config_file() # Read the CE entries from the BDII. entries = read_bdii(cp, query="(&(objectClass=GlueCE)(GlueCEInfoHostName=%s))" % ce_glob) cluster_info = create_count_dict(entries) sc_info = sub_cluster_info(cluster_info.keys(), cp) specint = get_cpu_normalizations() for key, val in specint.items(): if isinstance(val, types.TupleType): specint[key] = val[0] correction = eval(cp.get("cpu_count", "correction")) duplicate = eval(cp.get("cpu_count", "duplicate")) msi2k_ctr = 0.0 ksi2k_info = {} ownership = ownership_info(entries, cp) gk_ctr = 0 add_missing = cp.getboolean("cpu_count", "add_missing") do_not_add_missing = cp.get("cpu_count", "do_not_add_missing").split(',') do_not_add_missing = [i.strip() for i in do_not_add_missing] for cluster, cpu in cluster_info.items(): print "* Cluster: ", cluster my_sc_cores = 0 ksi2k_ctr = 0 correct_sc_info(cluster, cpu, sc_info, specint) # Print out SC info. if len(sc_info[cluster]) > 0: print " - Sub-clusters:" for sc in sc_info[cluster]: ksi2k = sc.glue["KSI2K"] msi2k_ctr += ksi2k / 1000.0 my_sc_cores += int(sc.glue["SubClusterLogicalCPUs"]) print " - %(SubClusterUniqueID)s, CPU Model:" \ " %(HostProcessorModel)s, Cores: %(SubClusterLogicalCPUs)s," \ " KSI2K: %(KSI2K)s" % sc.glue ksi2k_ctr += ksi2k # Do any KSI2K/CPU adjustments necessary. if my_sc_cores == 0: avg_ksi2k = 1.3 else: avg_ksi2k = ksi2k_ctr / float(my_sc_cores) if my_sc_cores > cpu: # Not enough CPUs; use sum from SC. cpu = my_sc_cores cluster_info[cluster] = cpu elif my_sc_cores < cpu and add_missing and (cluster not in \ do_not_add_missing): # Not enough KSI2K; add average froom SCs. addl_ksi2k = avg_ksi2k * (cpu - my_sc_cores) print " - Additional kSI2K for %s: %i" % (cluster, addl_ksi2k) ksi2k_ctr += addl_ksi2k msi2k_ctr += addl_ksi2k / 1000.0 ksi2k_info[cluster] = ksi2k_ctr # Print out any correction factors or duplicate clusters if cluster in correction: print " - Correction factor: %s" % correction[cluster] if cluster in duplicate: print " - Duplicates of this cluster: %s" % duplicate[cluster] try: print " - Ownership:", pretty_ownership(ownership[cluster]) except: pass print " - Core count:", cpu print " - KSI2K: %.1f" % ksi2k_ctr gk_ctr += 1 print "----------------------------" core_count, msi2k_count, vo_info = correct_count(cluster_info, ksi2k_info, ownership, correction, duplicate) print "----------------------------" print "OSG cores sum:", core_count print "OSG MSI2K: %.2f" % msi2k_count print "OSG gatekeepers count:", gk_ctr print "----------------------------" other_cores = 0 other_msi2k = 0 other_vos = [] print_vos = [i.strip() for i in cp.get("cpu_count", "print_vos").split(',')] for vo, info in vo_info.items(): if vo not in print_vos: other_cores += info[0] other_msi2k += info[1] other_vos.append(vo) continue print "%s cores sum: %i" % (vo, info[0]) print "%s MSI2K: %.2f" % (vo, info[1]) print "Other cores sum: %i" % other_cores print "Other MSI2K: %.2f" % other_msi2k print "Other VOs:", other_vos