def rpt_services(layout, output): output.write('\n<h2>Service Counts</h2>\n') lcl_services = [] fields = ['Service', 'Count'] services = aggregate_components(layout) for service in services: lcl_service = {} lcl_service['Service'] = service lcl_service['Count'] = services[service] lcl_services.append(lcl_service) writehtmltable(lcl_services, fields, output)
def rpt_mem_allocations(hostMatrix, control, output): output.write('\n<h2>Host Memory Allocations</h2>\n') fields = ['Hostname', 'Gb', 'Allocated', 'Components'] mem_recs = [] cluster_total_mem = 0 for hostKey in hostMatrix: mem_rec = {} host = hostMatrix[hostKey] mem_rec['Hostname'] = host['Hostname'] mem_rec['Gb'] = host['Gb'] mem_rec_component_heaps = {} mem_rec['Components'] = {} for controlKey in control: for component in control[controlKey]: for hostGroupKey in host['components']: if hostGroupKey == controlKey: for hostComponentKey in host['components'][ hostGroupKey]: mem = {} try: mem['heap'] = host['components'][hostGroupKey][ hostComponentKey]['heap'] try: mem['off.heap'] = host['components'][ hostGroupKey][hostComponentKey][ 'off.heap'] cluster_total_mem += mem['off.heap'] except: # No off.heap information pass except: no_heap = 'No HEAP Information->' + host[ 'Hostname'] + ':' + component + ':' + hostGroupKey + ':' + hostComponentKey # print hostComponentKey if len(mem) > 0: mem_rec['Components'][hostComponentKey] = mem # print 'host' total_mem = 0 for mem_alloc_key in mem_rec['Components']: mem_type = mem_rec['Components'][mem_alloc_key] for type in mem_type: mem_raw = mem_type[type] try: mem = int(mem_raw) except: mem = int(mem_raw[:-1]) total_mem += mem mem_rec['Allocated'] = total_mem / 1024 mem_recs.append(mem_rec) writehtmltable(mem_recs, fields, output) output.write("<br/>") output.write("<br/>") output.write("<h3>Total Memory Footprint: " + str(cluster_total_mem) + " GB </h3>")
def rpt_totals(hosttable, output): output.write('\n<h2>Totals</h2>\n') totalFields = [[0, "Type"], [1, "Count"], [2, "OS"], [3, "CPU-Min"], [4, "CPU-Max"], [5, "Mem-Min"], [6, "Mem-Max"]] totalType = [] datanodes = ["Data Nodes", 0, [], 10000, 0, 100000, 0] for record in hosttable: if record[9] == 'X': datanodes[1] += 1 if (record[1].encode('utf-8') not in datanodes[2]): datanodes[2].append(record[1].encode('utf-8')) # CPU Min if record[2] < datanodes[3]: datanodes[3] = record[2] # CPU Max if record[2] > datanodes[4]: datanodes[4] = record[2] # Mem Min if record[3] < datanodes[5]: datanodes[5] = record[3] # Mem Max if record[3] > datanodes[6]: datanodes[6] = record[3] totalType.append(datanodes) computeNodes = ["Compute Nodes", 0, [], 10000, 0, 100000, 0] for record in hosttable: if record[11] == 'X': computeNodes[1] += 1 if (record[1].encode('utf-8') not in computeNodes[2]): computeNodes[2].append(record[1].encode('utf-8')) # CPU Min if record[2] < computeNodes[3]: computeNodes[3] = record[2] # CPU Max if record[2] > computeNodes[4]: computeNodes[4] = record[2] # Mem Min if record[3] < computeNodes[5]: computeNodes[5] = record[3] # Mem Max if record[3] > computeNodes[6]: computeNodes[6] = record[3] totalType.append(computeNodes) writehtmltable(totalType, totalFields, output)
def rpt_hosttable(hostMatrix, control, output): output.write('\n<h2>Host Table</h2>\n') # master = datanode & compute fields_base = ['Hostname', 'OS', 'vC', 'Gb', 'Rack'] paths, bfields = build_field_path_from_abbr(control, ['KX', 'NN', 'JN', 'ZKFC', 'DN', 'RM', 'NM', 'ZK', 'HMS', 'HS2', 'HS2i', 'OZ', 'HM', 'RS', 'KB', 'NF', 'LV2', 'S2H', 'DR', 'DO', 'DB', 'DM', 'DH', 'DH']) fields = fields_base + bfields hosttable = [] for hostKey in hostMatrix: host = hostMatrix[hostKey] hostRec = get_hostbase(host, fields_base) populate_components(paths, host['components'], hostRec) hosttable.append(hostRec) writehtmltable(hosttable, fields, output)
def rpt_hoststorage(hostMatrix, control, output): output.write('\n<h2>Host Storage</h2>\n') fields_base = ['Hostname', 'vC', 'Gb', 'Rack'] paths, bfields = build_field_path_from_abbr(control, ['NN', 'JN', 'DN', 'ZK', 'NM', 'KB', 'NF']) fields = fields_base + bfields fields.append('DataDirs') fields.append('LogsDirs') fields.append('Disks') hosttable = [] for hostKey in hostMatrix: host = hostMatrix[hostKey] hostRec = get_hostbase(host, fields_base) populate_components(paths, host['components'], hostRec) hostRec['DataDirs'] = getDataDirs(host['components']) hostRec['LogsDirs'] = getLogsDirs(host['components']) hostRec['Disks'] = host['Disks'] hosttable.append(hostRec) writehtmltable(hosttable, fields, output)
def rpt_count_type(blueprint, layout, types, output, componentDict): cluster_creation_template = {} output.write('\n<h2>Count Types</h2>\n') # layout = json.loads(open(layoutFile).read()) items = layout['items'] # master = datanode & compute # type = { category: ['DN','NN']} table = [] fields = ['Category', 'Types', 'Count', 'Min Cores', 'Max Cores', 'Min Gb', 'Max Gb'] for category in types: type_rec = {} type_rec['Category'] = category type_rec['Count'] = 0 type_rec['Types'] = types[category] type_rec['Min Cores'] = 10000 type_rec['Max Cores'] = 0 type_rec['Min Gb'] = 10000 type_rec['Max Gb'] = 0 table.append(type_rec) for item in items: found = 0 for comp in types[category]: componentFound, hgbitmask = is_component(item, comp, componentDict) if componentFound: found += 1 host = item['Hosts'] mem = host['total_mem'] / (1024 * 1024) # CPU Min if host['cpu_count'] < type_rec['Min Cores']: type_rec['Min Cores'] = host['cpu_count'] # CPU Max if host['cpu_count'] > type_rec['Max Cores']: type_rec['Max Cores'] = host['cpu_count'] # Mem Min if mem < type_rec['Min Gb']: type_rec['Min Gb'] = mem # Mem Max if mem > type_rec['Max Gb']: type_rec['Max Gb'] = mem if found == 1: found += 1; type_rec['Count'] += 1 writehtmltable(table, fields, output) daemon_count = 0 for cat in table: if cat['Category'] == 'LLAP' and cat['Count'] > 0: for config in blueprint['configurations']: if 'hive-interactive-env' in config.keys(): daemon_count = int(config['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons']) if config['hive-interactive-env']['properties']['num_llap_nodes'] > daemon_count: daemon_count = int(config['hive-interactive-env']['properties']['num_llap_nodes']) break if daemon_count > 0: output.write('\n<h2>LLAP Daemon Count: ' + str(daemon_count) + '</h2>\n') output.write('\n<h2>Unique Host Count: ' + str(len(layout['items'])) + '</h2>\n') # Generate Counts for Blueprint Host Groups. # Go through the Merged Blueprint and count the hosts in each host_group. hg_table = [] hg_fields = ['Host Group', 'Count', 'Components', 'Hosts'] cluster_creation_template['blueprint'] = 'need-to-set-me' host_groups = blueprint['host_groups'] cct_host_groups = [] for host_group in host_groups: cct_host_group = {} hgrec = {} hgrec['Host Group'] = host_group['name'] cct_host_group['name'] = host_group['name'] cct_hosts = [] hgrec['Count'] = len(host_group['hosts']) hgrec_components = [] for comps in host_group['components']: hgrec_components.append(comps['name']) hgrec['Components'] = hgrec_components hgrec_hosts = [] for hst in host_group['hosts']: hgrec_hosts.append(hst['hostname']) cct_host = {} cct_host['fqdn'] = hst['hostname'] cct_hosts.append(cct_host) cct_host_group['hosts'] = cct_hosts cct_host_groups.append(cct_host_group) hgrec['Hosts'] = hgrec_hosts hg_table.append(hgrec) cluster_creation_template['host_groups'] = cct_host_groups output.write('\n<h2>Ambari Host Group Info</h2>\n') writehtmltable(hg_table, hg_fields, output) return cluster_creation_template