def rpt_services(layout, output):
output.write('\n<h2>Service Counts</h2>\n')
lcl_services = []
fields = ['Service', 'Count']
services = aggregate_components(layout)
for service in services:
lcl_service = {}
lcl_service['Service'] = service
lcl_service['Count'] = services[service]
lcl_services.append(lcl_service)
writehtmltable(lcl_services, fields, output)
def rpt_mem_allocations(hostMatrix, control, output):
output.write('\n<h2>Host Memory Allocations</h2>\n')
fields = ['Hostname', 'Gb', 'Allocated', 'Components']
mem_recs = []
cluster_total_mem = 0
for hostKey in hostMatrix:
mem_rec = {}
host = hostMatrix[hostKey]
mem_rec['Hostname'] = host['Hostname']
mem_rec['Gb'] = host['Gb']
mem_rec_component_heaps = {}
mem_rec['Components'] = {}
for controlKey in control:
for component in control[controlKey]:
for hostGroupKey in host['components']:
if hostGroupKey == controlKey:
for hostComponentKey in host['components'][
hostGroupKey]:
mem = {}
try:
mem['heap'] = host['components'][hostGroupKey][
hostComponentKey]['heap']
try:
mem['off.heap'] = host['components'][
hostGroupKey][hostComponentKey][
'off.heap']
cluster_total_mem += mem['off.heap']
except:
# No off.heap information
pass
except:
no_heap = 'No HEAP Information->' + host[
'Hostname'] + ':' + component + ':' + hostGroupKey + ':' + hostComponentKey
# print hostComponentKey
if len(mem) > 0:
mem_rec['Components'][hostComponentKey] = mem
# print 'host'
total_mem = 0
for mem_alloc_key in mem_rec['Components']:
mem_type = mem_rec['Components'][mem_alloc_key]
for type in mem_type:
mem_raw = mem_type[type]
try:
mem = int(mem_raw)
except:
mem = int(mem_raw[:-1])
total_mem += mem
mem_rec['Allocated'] = total_mem / 1024
mem_recs.append(mem_rec)
writehtmltable(mem_recs, fields, output)
output.write("<br/>")
output.write("<br/>")
output.write("<h3>Total Memory Footprint: " + str(cluster_total_mem) +
" GB </h3>")
def rpt_totals(hosttable, output):
output.write('\n<h2>Totals</h2>\n')
totalFields = [[0, "Type"], [1, "Count"], [2, "OS"], [3, "CPU-Min"],
[4, "CPU-Max"], [5, "Mem-Min"], [6, "Mem-Max"]]
totalType = []
datanodes = ["Data Nodes", 0, [], 10000, 0, 100000, 0]
for record in hosttable:
if record[9] == 'X':
datanodes[1] += 1
if (record[1].encode('utf-8') not in datanodes[2]):
datanodes[2].append(record[1].encode('utf-8'))
# CPU Min
if record[2] < datanodes[3]:
datanodes[3] = record[2]
# CPU Max
if record[2] > datanodes[4]:
datanodes[4] = record[2]
# Mem Min
if record[3] < datanodes[5]:
datanodes[5] = record[3]
# Mem Max
if record[3] > datanodes[6]:
datanodes[6] = record[3]
totalType.append(datanodes)
computeNodes = ["Compute Nodes", 0, [], 10000, 0, 100000, 0]
for record in hosttable:
if record[11] == 'X':
computeNodes[1] += 1
if (record[1].encode('utf-8') not in computeNodes[2]):
computeNodes[2].append(record[1].encode('utf-8'))
# CPU Min
if record[2] < computeNodes[3]:
computeNodes[3] = record[2]
# CPU Max
if record[2] > computeNodes[4]:
computeNodes[4] = record[2]
# Mem Min
if record[3] < computeNodes[5]:
computeNodes[5] = record[3]
# Mem Max
if record[3] > computeNodes[6]:
computeNodes[6] = record[3]
totalType.append(computeNodes)
writehtmltable(totalType, totalFields, output)
def rpt_hosttable(hostMatrix, control, output):
output.write('\n<h2>Host Table</h2>\n')
# master = datanode & compute
fields_base = ['Hostname', 'OS', 'vC', 'Gb', 'Rack']
paths, bfields = build_field_path_from_abbr(control, ['KX', 'NN', 'JN', 'ZKFC', 'DN', 'RM', 'NM',
'ZK', 'HMS', 'HS2', 'HS2i', 'OZ', 'HM', 'RS',
'KB', 'NF', 'LV2', 'S2H', 'DR', 'DO', 'DB',
'DM', 'DH', 'DH'])
fields = fields_base + bfields
hosttable = []
for hostKey in hostMatrix:
host = hostMatrix[hostKey]
hostRec = get_hostbase(host, fields_base)
populate_components(paths, host['components'], hostRec)
hosttable.append(hostRec)
writehtmltable(hosttable, fields, output)
def rpt_hoststorage(hostMatrix, control, output):
output.write('\n<h2>Host Storage</h2>\n')
fields_base = ['Hostname', 'vC', 'Gb', 'Rack']
paths, bfields = build_field_path_from_abbr(control, ['NN', 'JN', 'DN', 'ZK', 'NM', 'KB', 'NF'])
fields = fields_base + bfields
fields.append('DataDirs')
fields.append('LogsDirs')
fields.append('Disks')
hosttable = []
for hostKey in hostMatrix:
host = hostMatrix[hostKey]
hostRec = get_hostbase(host, fields_base)
populate_components(paths, host['components'], hostRec)
hostRec['DataDirs'] = getDataDirs(host['components'])
hostRec['LogsDirs'] = getLogsDirs(host['components'])
hostRec['Disks'] = host['Disks']
hosttable.append(hostRec)
writehtmltable(hosttable, fields, output)
def rpt_count_type(blueprint, layout, types, output, componentDict):
cluster_creation_template = {}
output.write('\n<h2>Count Types</h2>\n')
# layout = json.loads(open(layoutFile).read())
items = layout['items']
# master = datanode & compute
# type = { category: ['DN','NN']}
table = []
fields = ['Category', 'Types', 'Count', 'Min Cores', 'Max Cores', 'Min Gb', 'Max Gb']
for category in types:
type_rec = {}
type_rec['Category'] = category
type_rec['Count'] = 0
type_rec['Types'] = types[category]
type_rec['Min Cores'] = 10000
type_rec['Max Cores'] = 0
type_rec['Min Gb'] = 10000
type_rec['Max Gb'] = 0
table.append(type_rec)
for item in items:
found = 0
for comp in types[category]:
componentFound, hgbitmask = is_component(item, comp, componentDict)
if componentFound:
found += 1
host = item['Hosts']
mem = host['total_mem'] / (1024 * 1024)
# CPU Min
if host['cpu_count'] < type_rec['Min Cores']:
type_rec['Min Cores'] = host['cpu_count']
# CPU Max
if host['cpu_count'] > type_rec['Max Cores']:
type_rec['Max Cores'] = host['cpu_count']
# Mem Min
if mem < type_rec['Min Gb']:
type_rec['Min Gb'] = mem
# Mem Max
if mem > type_rec['Max Gb']:
type_rec['Max Gb'] = mem
if found == 1:
found += 1;
type_rec['Count'] += 1
writehtmltable(table, fields, output)
daemon_count = 0
for cat in table:
if cat['Category'] == 'LLAP' and cat['Count'] > 0:
for config in blueprint['configurations']:
if 'hive-interactive-env' in config.keys():
daemon_count = int(config['hive-interactive-env']['properties']['num_llap_nodes_for_llap_daemons'])
if config['hive-interactive-env']['properties']['num_llap_nodes'] > daemon_count:
daemon_count = int(config['hive-interactive-env']['properties']['num_llap_nodes'])
break
if daemon_count > 0:
output.write('\n<h2>LLAP Daemon Count: ' + str(daemon_count) + '</h2>\n')
output.write('\n<h2>Unique Host Count: ' + str(len(layout['items'])) + '</h2>\n')
# Generate Counts for Blueprint Host Groups.
# Go through the Merged Blueprint and count the hosts in each host_group.
hg_table = []
hg_fields = ['Host Group', 'Count', 'Components', 'Hosts']
cluster_creation_template['blueprint'] = 'need-to-set-me'
host_groups = blueprint['host_groups']
cct_host_groups = []
for host_group in host_groups:
cct_host_group = {}
hgrec = {}
hgrec['Host Group'] = host_group['name']
cct_host_group['name'] = host_group['name']
cct_hosts = []
hgrec['Count'] = len(host_group['hosts'])
hgrec_components = []
for comps in host_group['components']:
hgrec_components.append(comps['name'])
hgrec['Components'] = hgrec_components
hgrec_hosts = []
for hst in host_group['hosts']:
hgrec_hosts.append(hst['hostname'])
cct_host = {}
cct_host['fqdn'] = hst['hostname']
cct_hosts.append(cct_host)
cct_host_group['hosts'] = cct_hosts
cct_host_groups.append(cct_host_group)
hgrec['Hosts'] = hgrec_hosts
hg_table.append(hgrec)
cluster_creation_template['host_groups'] = cct_host_groups
output.write('\n<h2>Ambari Host Group Info</h2>\n')
writehtmltable(hg_table, hg_fields, output)
return cluster_creation_template
