def do_se_info(cp): site_entries = read_bdii(cp, "(objectClass=GlueSite)") se_entries = read_bdii(cp, "(objectClass=GlueSE)") #conn = getGipDBConn(cp) #curs = conn.cursor() today = datetime.date.today() time_now = time.time() gratia_info = {} for entry in se_entries: try: site = join_FK(entry, site_entries, "SiteUniqueID") except ValueError, ve: log.warn("Unable to match SE:\n%s" % entry) continue try: site_name = site.glue['SiteName'] total = int(entry.glue['SESizeTotal']) free = int(entry.glue['SESizeFree']) se_name = entry.glue['SEName'] except: log.warn("Unable to parse attributes:\n%s" % entry) continue #curs.execute(insert_se_info, {'date': today, 'site': site_name, 'se': \ # se_name, 'total': total, 'free': free}) if total == 0 and free == 0: continue unique_id = entry.glue['SEUniqueID'] probeName = 'gip_storage:%s:%s' % (unique_id, hostname) Gratia.Config.setMeterName(probeName) Gratia.Config.setSiteName(se_name) se = StorageElement.StorageElement() space_unique_id = "%s:%s:%s" % (unique_id, "SE", se_name) se.UniqueID(space_unique_id) se.SE(se_name) se.Name(se_name) se.SpaceType("SE") se.Timestamp(time_now) se.Implementation(entry.glue['SEImplementationName']) se.Version(entry.glue['SEImplementationVersion']) se.Status(entry.glue['SEStatus']) se_list = gratia_info.setdefault((probeName, se_name), []) se_list.append(se) ser = StorageElementRecord.StorageElementRecord() ser.UniqueID(space_unique_id) ser.MeasurementType("raw") ser.StorageType("disk") ser.Timestamp(time_now) ser.TotalSpace(total*1000**3) ser.FreeSpace(free*1000**3) ser.UsedSpace((total-free)*1000**3) se_list.append(ser)
def main(): cp = config_file() # Read the CE entries from the BDII. entries = read_bdii(cp, query="(objectClass=GlueCE)") cluster_info = create_count_dict(entries) sc_info = sub_cluster_info(cluster_info.keys(), cp) specint = get_cpu_normalizations() for key, val in specint.items(): if isinstance(val, types.TupleType): specint[key] = val[0] ksi2k_info = {} site_dict = create_site_dict(entries, cp) sites = cp.get("site_normalization", "sites").split(",") sites = [i.strip() for i in sites] for cluster, cpu in cluster_info.items(): correct_sc_info(cluster, cpu, sc_info, specint) ksi2k = 0 sc_cores = 0 for sc in sc_info[cluster]: ksi2k += sc.glue["KSI2K"] sc_cores += int(sc.glue["SubClusterLogicalCPUs"]) try: site = site_dict[cluster] except: print "Problem with %s" % cluster continue if site in sites: print site, (ksi2k * 1000) / sc_cores
def main(): cp = config_file() # Read the CE entries from the BDII. entries = read_bdii(cp, query="(objectClass=GlueCE)") cluster_info = create_count_dict(entries) sc_info = sub_cluster_info(cluster_info.keys(), cp) specint = get_cpu_normalizations() for key, val in specint.items(): if isinstance(val, types.TupleType): specint[key] = val[0] ksi2k_info = {} site_dict = create_site_dict(entries, cp) sites = cp.get("site_normalization", "sites").split(",") sites = [i.strip() for i in sites] for cluster, cpu in cluster_info.items(): correct_sc_info(cluster, cpu, sc_info, specint) ksi2k = 0 sc_cores = 0 for sc in sc_info[cluster]: ksi2k += sc.glue["KSI2K"] sc_cores += int(sc.glue["SubClusterLogicalCPUs"]) try: site = site_dict[cluster] except: print "Problem with %s" % cluster continue if site in sites: print site, (ksi2k*1000) / sc_cores
def create_site_dict(ce_entries, cp): """ Determine site ownership of CEs. """ # Query BDII for the cluster and site entries cluster_entries = read_bdii(cp, query="(objectClass=GlueCluster)", multi=True) site_entries = read_bdii(cp, query="(objectClass=GlueSite)") ownership = {} # Determine the site's advertised ownership. for ce in ce_entries: try: # First, we join the CE to the cluster: cluster = join_FK(ce, cluster_entries, "ClusterUniqueID") if ce.glue['CEHostingCluster'] == 'red.unl.edu': print cluster # Then, join the cluster to the site: site = join_FK(cluster, site_entries, "SiteUniqueID") ownership[ce.glue["CEHostingCluster"]] = site.glue["SiteName"] except Exception, e: print e pass
def do_site_info(cp): ce_entries = read_bdii(cp, "(objectClass=GlueCE)") cluster_entries = read_bdii(cp, "(objectClass=GlueCluster)", multi=True) site_entries = read_bdii(cp, "(objectClass=GlueSite)") ce_map = {} ce_map2 = {} for ce in ce_entries: try: cluster = ce.glue['ForeignKey'].split('=')[1] except: continue ce_map[ce.glue['CEHostingCluster']] = cluster ce_map2[ce.glue['CEUniqueID']] = cluster cluster_map = {} for cluster in cluster_entries: try: site = None for key in cluster.glue['ForeignKey']: kind, name = key.split('=', 1) if kind != 'GlueSiteUniqueID': continue site = name if not site: continue except: continue cluster_map[cluster.glue['ClusterName'][0]] = site site_map = {} for site in site_entries: site_map[site.glue['SiteUniqueID']] = site.glue['SiteName'] #conn = getGipDBConn(cp) #curs = conn.cursor() #for ce, cluster in ce_map.items(): # my_cluster = cluster_map.get(cluster, None) # my_site = site_map.get(my_cluster, None) # if my_site: # curs.execute(insert_site_info, {'site': my_site, 'cename': ce}) #conn.commit() return ce_map2, cluster_map, site_map
def sub_cluster_info(ce_list, cp): """ Given a list of CE names (not LDAP entries), return a dictionary where the key is the CE name and the value is a list of SubClusters associated with that CE. """ sc_entries = read_bdii(cp, query="(objectClass=GlueSubCluster)") sc_info = {} sc_total = {} for ce in ce_list: my_sc = sc_info.get(ce, []) sc_info[ce] = my_sc for sc in sc_entries: if "999999" in sc.glue['SubClusterLogicalCPUs']: continue desired_ck = "GlueClusterUniqueID=%s" % ce if "ChunkKey" in sc.glue and sc.glue["ChunkKey"] == desired_ck: my_sc.append(sc) return sc_info
def main(): # Load up the config file. cp = config_file() # Load the DB filename = os.path.expandvars("$HOME/dbinfo/DBParam.xml") if not os.path.exists(filename): filename = os.path.expandvars("$DBPARAM_LOCATION") if not os.path.exists(filename): filename = '/etc/DBParam.xml' x = XmlConfig(file=filename) conn = x.globals['GIPConnMan'].get_connection(None).get_connection() curs = conn.cursor() # Read the CE entries from the BDII. entries = read_bdii(cp, query="(&(objectClass=GlueCE))") cluster_info = create_count_dict(entries) # Map from the cluster hostname to the unique ID id_to_hostname = {} for entry in entries: fk = entry.glue['ForeignKey'] info = fk.split("=", 1) if len(info) != 2: continue if info[0] != "GlueClusterUniqueID": print >> sys.stderr, "Entry has unknown cluster FK: %s" % entry continue id = info[1] id_to_hostname[id] = entry.glue['CEHostingCluster'] sc_info = sub_cluster_info(id_to_hostname.keys(), cp) # For each unique cluster ID, map to one of the cluster hostnames new_sc_info = {} for id, info in sc_info.items(): if id not in id_to_hostname: print >> sys.stderr, "ID %s has no matching cluster hostname." % id continue new_sc_info[id_to_hostname[id]] = info sc_info = new_sc_info specint = get_cpu_normalizations() now = datetime.datetime.now() curs.execute("DELETE FROM cpu_score"); for cpu, score in specint.items(): if isinstance(score, types.TupleType): score = score[0] #specint[cpu] = score curs.execute("INSERT INTO cpu_score VALUES (%s, %s, %s)", (cpu, \ int(score), int(0))); site_ownership = create_site_dict(entries, cp) ownership = ownership_info(entries, cp) # Initialize the Probe's configuration ProbeConfig = '/etc/osg-storage-report/ProbeConfig' try: Gratia.Initialize(ProbeConfig) except Exception, e: print e raise
def do_cms_se_info(cp): site_entries = read_bdii(cp, "(objectClass=GlueSite)") se_entries = read_bdii(cp, "(objectClass=GlueSE)") sa_entries = read_bdii(cp, "(objectClass=GlueSA)", multi=True) today = datetime.date.today() time_now = time.time() gratia_info = {} for sa in sa_entries: if 'SAAccessControlBaseRule' not in sa.glue: continue supports_cms = False for acbr in sa.glue['SAAccessControlBaseRule']: if 'cms' in acbr.lower(): supports_cms = True break if not supports_cms: continue try: total = int(sa.glue['SATotalOnlineSize'][0]) free = int(sa.glue['SAFreeOnlineSize'][0]) used = int(sa.glue['SAUsedOnlineSize'][0]) except: log.warn("Unable to parse attributes:\n%s" % sa) continue try: se = join_FK(sa, se_entries, "SEUniqueID", join_fk_name="ChunkKey") except Exception, ve: log.warn("Unable to match SA to SE:\n%s" % sa) continue site = match_se_to_site(se, site_entries) if not site: log.warn("Unable to match SE %s to site." % \ se.glue['SEUniqueID']) continue print sa.glue['ChunkKey'][0] se_unique_id = se.glue['SEUniqueID'] sa_name = sa.glue['SAName'][0] probeName = 'gip_storage:%s' % se_unique_id se_name = se.glue['SEName'] gse = StorageElement.StorageElement() space_unique_id = "%s:%s:%s" % (se_unique_id, "GlueStorageArea", sa_name) gse.ParentID("%s:%s:%s" % (se_unique_id, "SE", se_name)) gse.UniqueID(space_unique_id) gse.SE(se_name) gse.Name(sa_name) gse.SpaceType("GlueStorageArea") gse.Timestamp(time_now) gse.Implementation(se.glue['SEImplementationName']) gse.Version(se.glue['SEImplementationVersion']) gse.VO("cms") gse.Status(se.glue['SEStatus']) se_list = gratia_info.setdefault((probeName, se_name), []) se_list.append(gse) ser = StorageElementRecord.StorageElementRecord() ser.UniqueID(space_unique_id) ser.MeasurementType("logical") ser.StorageType("disk") ser.Timestamp(time_now) ser.TotalSpace(total*1000**3) ser.FreeSpace(free*1000**3) ser.UsedSpace(used*1000**3) se_list.append(ser)
def do_cms_se_info(cp): site_entries = read_bdii(cp, "(objectClass=GlueSite)") se_entries = read_bdii(cp, "(objectClass=GlueSE)") sa_entries = read_bdii(cp, "(objectClass=GlueSA)", multi=True) today = datetime.date.today() time_now = time.time() gratia_info = {} for sa in sa_entries: if 'SAAccessControlBaseRule' not in sa.glue: continue supports_cms = False for acbr in sa.glue['SAAccessControlBaseRule']: if 'cms' in acbr.lower(): supports_cms = True break if not supports_cms: continue try: total = int(sa.glue['SATotalOnlineSize'][0]) free = int(sa.glue['SAFreeOnlineSize'][0]) used = int(sa.glue['SAUsedOnlineSize'][0]) except: log.warn("Unable to parse attributes:\n%s" % sa) continue try: se = join_FK(sa, se_entries, "SEUniqueID", join_fk_name="ChunkKey") except Exception, ve: log.warn("Unable to match SA to SE:\n%s" % sa) continue site = match_se_to_site(se, site_entries) if not site: log.warn("Unable to match SE %s to site." % \ se.glue['SEUniqueID']) continue print sa.glue['ChunkKey'][0] se_unique_id = se.glue['SEUniqueID'] sa_name = sa.glue['SAName'][0] probeName = 'gip_storage:%s' % se_unique_id se_name = se.glue['SEName'] gse = StorageElement.StorageElement() space_unique_id = "%s:%s:%s" % (se_unique_id, "GlueStorageArea", sa_name) gse.ParentID("%s:%s:%s" % (se_unique_id, "SE", se_name)) gse.UniqueID(space_unique_id) gse.SE(se_name) gse.Name(sa_name) gse.SpaceType("GlueStorageArea") gse.Timestamp(time_now) gse.Implementation(se.glue['SEImplementationName']) gse.Version(se.glue['SEImplementationVersion']) gse.VO("cms") gse.Status(se.glue['SEStatus']) se_list = gratia_info.setdefault((probeName, se_name), []) se_list.append(gse) ser = StorageElementRecord.StorageElementRecord() ser.UniqueID(space_unique_id) ser.MeasurementType("logical") ser.StorageType("disk") ser.Timestamp(time_now) ser.TotalSpace(total * 1000**3) ser.FreeSpace(free * 1000**3) ser.UsedSpace(used * 1000**3) se_list.append(ser)
def main(): # Load up the config file. cp = config_file() # Load the DB filename = os.path.expandvars("$HOME/dbinfo/DBParam.xml") if not os.path.exists(filename): filename = os.path.expandvars("$DBPARAM_LOCATION") if not os.path.exists(filename): filename = '/etc/DBParam.xml' x = XmlConfig(file=filename) conn = x.globals['GIPConnMan'].get_connection(None).get_connection() curs = conn.cursor() # Read the CE entries from the BDII. entries = read_bdii(cp, query="(&(objectClass=GlueCE))") cluster_info = create_count_dict(entries) # Map from the cluster hostname to the unique ID id_to_hostname = {} for entry in entries: fk = entry.glue['ForeignKey'] info = fk.split("=", 1) if len(info) != 2: continue if info[0] != "GlueClusterUniqueID": print >> sys.stderr, "Entry has unknown cluster FK: %s" % entry continue id = info[1] id_to_hostname[id] = entry.glue['CEHostingCluster'] sc_info = sub_cluster_info(id_to_hostname.keys(), cp) # For each unique cluster ID, map to one of the cluster hostnames new_sc_info = {} for id, info in sc_info.items(): if id not in id_to_hostname: print >> sys.stderr, "ID %s has no matching cluster hostname." % id continue new_sc_info[id_to_hostname[id]] = info sc_info = new_sc_info specint = get_cpu_normalizations() now = datetime.datetime.now() curs.execute("DELETE FROM cpu_score") for cpu, score in specint.items(): if isinstance(score, types.TupleType): score = score[0] #specint[cpu] = score curs.execute("INSERT INTO cpu_score VALUES (%s, %s, %s)", (cpu, \ int(score), int(0))) site_ownership = create_site_dict(entries, cp) ownership = ownership_info(entries, cp) # Initialize the Probe's configuration ProbeConfig = '/etc/osg-storage-report/ProbeConfig' try: Gratia.Initialize(ProbeConfig) except Exception, e: print e raise
def ownership_info(ce_entries, cp): """ Determine ownership of clusters from the sites's SiteSponsor attribute. """ # Query BDII for the cluster and site entries cluster_entries = read_bdii(cp, query="(objectClass=GlueCluster)", multi=True) site_entries = read_bdii(cp, query="(objectClass=GlueSite)") ownership = {} # Determine the site's advertised ownership. for ce in ce_entries: # First, we join the CE to the cluster: try: cluster = join_FK(ce, cluster_entries, "ClusterUniqueID") except: print "Unable to find cluster for CE; skipping\n%s" % ce continue #print cluster # Then, join the cluster to the site: try: site = join_FK(cluster, site_entries, "SiteUniqueID") except: continue #print site try: ownership[ce] = site.glue["SiteSponsor"] #print site.glue["SiteName"], site.glue["SiteSponsor"] except: ownership[ce] = "unknown" # Refine ownership; turn string into list of tuples and make sure # that everything adds up to 100%. # This is a bit awkward. I should consider turning list of tuples # into a dictionary. refined_ownership = {} for ce, val in ownership.items(): val = val.lower() val = val.replace('"', '') val = val.replace("'", '') refined = [] ctr = 0 #print val, ownership_re.findall(val) for entry in ownership_re.findall(val): info = entry.split(':') vo = correct_vo(info[0], cp) if len(info) == 1: refined.append((vo, 100)) ctr += 100 else: try: amt = info[1] amt = amt.replace("'", "") refined.append((vo, int(amt))) ctr += int(amt) except: print entry raise if ctr < 100: new_refined = [] has_unknown = False for entry in refined: if entry[0] == "unknown": new_refined.append(("unknown", entry[1]+(100-ctr))) has_unknown = True else: new_refined.append(entry) if not has_unknown: new_refined.append(("unknown", 100-ctr)) refined = new_refined refined_ownership[ce.glue["CEInfoHostName"]] = refined return refined_ownership
def main(): # Determine any filters we should apply parser = optparse.OptionParser(add_help_option=False) parser.add_option("-e", "--endpoint") parser.add_option("-h", "--help") parser.add_option("-c", "--config") parser.parse_args() (options, args) = parser.parse_args() if len(args) >= 1: ce_glob = args[0] else: ce_glob = "*" # Load up the config file. cp = config_file() # Read the CE entries from the BDII. entries = read_bdii(cp, query="(&(objectClass=GlueCE)(GlueCEInfoHostName=%s))" % ce_glob) cluster_info = create_count_dict(entries) sc_info = sub_cluster_info(cluster_info.keys(), cp) specint = get_cpu_normalizations() for key, val in specint.items(): if isinstance(val, types.TupleType): specint[key] = val[0] correction = eval(cp.get("cpu_count", "correction")) duplicate = eval(cp.get("cpu_count", "duplicate")) msi2k_ctr = 0.0 ksi2k_info = {} ownership = ownership_info(entries, cp) gk_ctr = 0 add_missing = cp.getboolean("cpu_count", "add_missing") do_not_add_missing = cp.get("cpu_count", "do_not_add_missing").split(',') do_not_add_missing = [i.strip() for i in do_not_add_missing] for cluster, cpu in cluster_info.items(): print "* Cluster: ", cluster my_sc_cores = 0 ksi2k_ctr = 0 correct_sc_info(cluster, cpu, sc_info, specint) # Print out SC info. if len(sc_info[cluster]) > 0: print " - Sub-clusters:" for sc in sc_info[cluster]: ksi2k = sc.glue["KSI2K"] msi2k_ctr += ksi2k / 1000.0 my_sc_cores += int(sc.glue["SubClusterLogicalCPUs"]) print " - %(SubClusterUniqueID)s, CPU Model:" \ " %(HostProcessorModel)s, Cores: %(SubClusterLogicalCPUs)s," \ " KSI2K: %(KSI2K)s" % sc.glue ksi2k_ctr += ksi2k # Do any KSI2K/CPU adjustments necessary. if my_sc_cores == 0: avg_ksi2k = 1.3 else: avg_ksi2k = ksi2k_ctr / float(my_sc_cores) if my_sc_cores > cpu: # Not enough CPUs; use sum from SC. cpu = my_sc_cores cluster_info[cluster] = cpu elif my_sc_cores < cpu and add_missing and (cluster not in \ do_not_add_missing): # Not enough KSI2K; add average froom SCs. addl_ksi2k = avg_ksi2k * (cpu - my_sc_cores) print " - Additional kSI2K for %s: %i" % (cluster, addl_ksi2k) ksi2k_ctr += addl_ksi2k msi2k_ctr += addl_ksi2k / 1000.0 ksi2k_info[cluster] = ksi2k_ctr # Print out any correction factors or duplicate clusters if cluster in correction: print " - Correction factor: %s" % correction[cluster] if cluster in duplicate: print " - Duplicates of this cluster: %s" % duplicate[cluster] try: print " - Ownership:", pretty_ownership(ownership[cluster]) except: pass print " - Core count:", cpu print " - KSI2K: %.1f" % ksi2k_ctr gk_ctr += 1 print "----------------------------" core_count, msi2k_count, vo_info = correct_count(cluster_info, ksi2k_info, ownership, correction, duplicate) print "----------------------------" print "OSG cores sum:", core_count print "OSG MSI2K: %.2f" % msi2k_count print "OSG gatekeepers count:", gk_ctr print "----------------------------" other_cores = 0 other_msi2k = 0 other_vos = [] print_vos = [i.strip() for i in cp.get("cpu_count", "print_vos").split(',')] for vo, info in vo_info.items(): if vo not in print_vos: other_cores += info[0] other_msi2k += info[1] other_vos.append(vo) continue print "%s cores sum: %i" % (vo, info[0]) print "%s MSI2K: %.2f" % (vo, info[1]) print "Other cores sum: %i" % other_cores print "Other MSI2K: %.2f" % other_msi2k print "Other VOs:", other_vos