def get_actual_usage(dom_obj, host_ip):
dom_name = dom_obj.name()
dom_stats = get_current_dom_resource_usage(dom_obj, host_ip)
"""Fetch previous domain stat value from cache"""
prev_dom_stats = current.cache.disk(str(dom_name), lambda:dom_stats, 86400) # @UndefinedVariable
rrd_logger.debug(prev_dom_stats)
#calulate usage
usage = {'ram' : dom_stats['memory']} #ram in Bytes usage
usage.update({'cpu' : (dom_stats['cputime'] - prev_dom_stats['cputime'])/(float(prev_dom_stats['cpus'])*10000000*STEP)}) #percent cpu usage
usage.update({'tx' : (dom_stats['tx'] - prev_dom_stats['tx'])}) #in KBytes
usage.update({'rx' : (dom_stats['rx'] - prev_dom_stats['rx'])}) #in KBytes
usage.update({'dr' : (dom_stats['diskr'] - prev_dom_stats['diskr'])}) #in KBytes
usage.update({'dw' : (dom_stats['diskw'] - prev_dom_stats['diskw'])}) #in KBytes
current.cache.disk.clear(str(dom_name)) # @UndefinedVariable
"""Cache the current CPU utilization, so that difference can be calculated in next instance"""
latest_dom_stats = current.cache.disk(str(dom_name), lambda:dom_stats, 86400) # @UndefinedVariable
rrd_logger.debug(latest_dom_stats)
return usage
def get_performance_graph(graph_type, vm, graph_period):
error = None
img = IMG(_src = URL("static" , "images/no_graph.jpg") , _style = "height:100px")
try:
rrd_file = get_rrd_file(vm)
if os.path.exists(rrd_file):
if create_graph(vm, graph_type, rrd_file, graph_period):
img_pos = "images/vm_graphs/" + vm + "_" + graph_type + ".png"
img = IMG(_src = URL("static", img_pos), _style = "height:100%")
rrd_logger.info("Graph created successfully")
else:
rrd_logger.warn("Unable to create graph from rrd file!!!")
error = "Unable to create graph from rrd file"
else:
rrd_logger.warn("VMs RRD File Unavailable!!!")
error = "VMs RRD File Unavailable!!!"
except:
rrd_logger.warn("Error occured while creating graph.")
import sys, traceback
etype, value, tb = sys.exc_info()
error = ''.join(traceback.format_exception(etype, value, tb, 10))
rrd_logger.debug(error)
finally:
if error != None:
return error
else:
rrd_logger.info("Returning image.")
return img
def update_host_rrd(host_ip,m_type=None):
"""
Updates the RRD file for host with resource utilization data
"""
try:
rrd_file = _get_rrd_file(host_ip.replace(".","_"))
rrd_logger.info(rrd_file)
timestamp_now = time.time()
rrd_logger.info(timestamp_now)
if not (os.path.exists(rrd_file)):
rrd_logger.warn("RRD file (%s) does not exists" % (rrd_file))
rrd_logger.warn("Creating new RRD file")
create_rrd(rrd_file)
else:
rrd_logger.info("updating RRD file")
if m_type is None:
host_stats = get_host_resources_usage(host_ip)
else:
host_stats = get_host_resources_usage(host_ip,m_type)
rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, host_stats['cpu'], host_stats['ram'], host_stats['dr'], host_stats['dw'], host_stats['tx'], host_stats['rx']))
except Exception, e:
rrd_logger.debug("Error occured while creating/updating rrd for host: %s" % (host_ip))
rrd_logger.debug(e)
def get_dom_mem_usage(dom_name, host):
rrd_logger.debug("Fetching memory usage of domain %s defined on host %s" % (dom_name, host))
cmd = "output=`ps -ef --sort=start_time | grep '%s.qcow2' | grep -v grep | awk '{print $2}'`;smem -c 'pid pss'| grep $output | awk '{print $2}'" % dom_name
#"ps aux | grep '\-name " + dom_name + " ' | grep kvm"
output = execute_remote_cmd(host, "root", cmd, None, True)
return (int(output[0]))*1024 #return memory in Bytes by default
def set_domain_memoryStatsperiod(host_ip):
try:
hypervisor_conn = libvirt.open("qemu+ssh://root@" + host_ip + "/system")
active_dom_ids = hypervisor_conn.listDomainsID()
for dom_id in active_dom_ids:
dom_info = hypervisor_conn.lookupByID(dom_id)
dom_info.setMemoryStatsPeriod(2)
except Exception, e:
rrd_logger.debug(e)
def get_host_cpu_usage(host_ip):
command = "iostat -c | sed '1,2d'"
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
rrd_logger.debug(type(command_output))
cpu_stats = re.split('\s+', command_output[1])
rrd_logger.debug(cpu_stats)
rrd_logger.info("CPU stats of host %s is %s" % ( host_ip, (cpu_stats[1] + cpu_stats[2] + cpu_stats[3])))
return (float(cpu_stats[1]) + float(cpu_stats[2]) + float(cpu_stats[3]))
def set_domain_memoryStatsperiod(host_ip):
try:
hypervisor_conn = libvirt.open("qemu+ssh://root@" + host_ip + "/system")
active_dom_ids = hypervisor_conn.listDomainsID()
for dom_id in active_dom_ids:
dom_info = hypervisor_conn.lookupByID(dom_id)
dom_info.setMemoryStatsPeriod(2)
except Exception, e:
rrd_logger.debug(e)
def get_host_power_usage(host_ip):
rrd_logger.info("Checking power on host"+str(host_ip))
command='ipmitool sdr elist full'
try:
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
except Exception, e:
pwr_usage=0
ret=""
rrd_logger.debug("ipmitool not installed")
def fetch_rrd_data(rrd_file_name, period=VM_UTIL_24_HOURS, period_no=24):
rrd_file = get_rrd_file(rrd_file_name)
start_time = 'now-' + str(24*60*60)
end_time = 'now'
if period == HOURS:
start_time = 'now-' + str(period_no*60*60)
elif period == MINS:
start_time = 'now-' + str(period_no*60)
elif period == VM_UTIL_10_MINS:
start_time = 'now-' + str(10*60)
elif period == VM_UTIL_ONE_WEEK:
start_time = '-1w'
elif period == VM_UTIL_ONE_MNTH:
start_time = '-1m'
elif period == VM_UTIL_THREE_MNTH:
start_time = '-3m'
elif period == VM_UTIL_ONE_YEAR:
start_time = '-1y'
cpu_data = []
mem_data = []
dskr_data = []
dskw_data = []
nwr_data = []
nww_data = []
if os.path.exists(rrd_file):
rrd_ret =rrdtool.fetch(rrd_file, 'MIN', '--start', start_time, '--end', end_time)
fld_info = rrd_ret[1]
data_info = rrd_ret[2]
cpu_idx = fld_info.index('cpu')
mem_idx = fld_info.index('ram')
dskr_idx = fld_info.index('dr')
dskw_idx = fld_info.index('dw')
nwr_idx = fld_info.index('tx')
nww_idx = fld_info.index('rx')
#Ignore the None values before computing average
for row in data_info:
if row[cpu_idx] != None: cpu_data.append(float(row[cpu_idx]))
if row[mem_idx] != None: mem_data.append(float(row[mem_idx]))
if row[dskr_idx] != None: dskr_data.append(float(row[dskr_idx]))
if row[dskw_idx] != None: dskw_data.append(float(row[dskw_idx]))
if row[nwr_idx] != None: nwr_data.append(float(row[nwr_idx]))
if row[nww_idx] != None: nww_data.append(float(row[nww_idx]))
if str(rrd_file)=="/mnt/datastore/vm_rrds/IITD_cs5100299_Openvswitch_Host.rrd":
rrd_logger.debug("cpu_utilization over 24hrs is :" + str(cpu_data))
rrd_logger.debug("cpu_utilization over 24hrs is :" + str(sum(cpu_data)) + " " + str(len(cpu_data)))
return (sum(mem_data)/float(len(mem_data)) if len(mem_data) > 0 else 0,
sum(cpu_data)/float(len(cpu_data)) if len(cpu_data) > 0 else 0,
sum(dskr_data)/float(len(dskr_data)) if len(dskr_data) > 0 else 0,
sum(dskw_data)/float(len(dskw_data)) if len(dskw_data) > 0 else 0,
sum(nwr_data)/float(len(nwr_data)) if len(nwr_data) > 0 else 0,
sum(nww_data)/float(len(nww_data)) if len(nww_data) > 0 else 0)
def fetch_rrd_data(rrd_file_name, period=VM_UTIL_24_HOURS, period_no=24):
rrd_file = get_rrd_file(rrd_file_name)
start_time = 'now-' + str(24*60*60)
end_time = 'now'
if period == HOURS:
start_time = 'now-' + str(period_no*60*60)
elif period == MINS:
start_time = 'now-' + str(period_no*60)
elif period == VM_UTIL_10_MINS:
start_time = 'now-' + str(10*60)
elif period == VM_UTIL_ONE_WEEK:
start_time = '-1w'
elif period == VM_UTIL_ONE_MNTH:
start_time = '-1m'
elif period == VM_UTIL_THREE_MNTH:
start_time = '-3m'
elif period == VM_UTIL_ONE_YEAR:
start_time = '-1y'
cpu_data = []
mem_data = []
dskr_data = []
dskw_data = []
nwr_data = []
nww_data = []
if os.path.exists(rrd_file):
rrd_ret =rrdtool.fetch(rrd_file, 'MIN', '--start', start_time, '--end', end_time)
fld_info = rrd_ret[1]
data_info = rrd_ret[2]
cpu_idx = fld_info.index('cpu')
mem_idx = fld_info.index('ram')
dskr_idx = fld_info.index('dr')
dskw_idx = fld_info.index('dw')
nwr_idx = fld_info.index('tx')
nww_idx = fld_info.index('rx')
#Ignore the None values before computing average
for row in data_info:
if row[cpu_idx] != None: cpu_data.append(float(row[cpu_idx]))
if row[mem_idx] != None: mem_data.append(float(row[mem_idx]))
if row[dskr_idx] != None: dskr_data.append(float(row[dskr_idx]))
if row[dskw_idx] != None: dskw_data.append(float(row[dskw_idx]))
if row[nwr_idx] != None: nwr_data.append(float(row[nwr_idx]))
if row[nww_idx] != None: nww_data.append(float(row[nww_idx]))
if str(rrd_file)=="/mnt/datastore/vm_rrds/IITD_cs5100299_Openvswitch_Host.rrd":
rrd_logger.debug("cpu_utilization over 24hrs is :" + str(cpu_data))
rrd_logger.debug("cpu_utilization over 24hrs is :" + str(sum(cpu_data)) + " " + str(len(cpu_data)))
return (sum(mem_data)/float(len(mem_data)) if len(mem_data) > 0 else 0,
sum(cpu_data)/float(len(cpu_data)) if len(cpu_data) > 0 else 0,
sum(dskr_data)/float(len(dskr_data)) if len(dskr_data) > 0 else 0,
sum(dskw_data)/float(len(dskw_data)) if len(dskw_data) > 0 else 0,
sum(nwr_data)/float(len(nwr_data)) if len(nwr_data) > 0 else 0,
sum(nww_data)/float(len(nww_data)) if len(nww_data) > 0 else 0)
def _set_domain_memoryStatsperiod(host_ip):
"""
Dynamically changes the domain memory balloon driver statistics collection period
"""
try:
hypervisor_conn = libvirt.open("qemu+ssh://root@" + host_ip + "/system")
active_dom_ids = hypervisor_conn.listDomainsID()
for dom_id in active_dom_ids:
dom_info = hypervisor_conn.lookupByID(dom_id)
dom_info.setMemoryStatsPeriod(2)
except Exception, e:
rrd_logger.debug(e)
def get_host_temp_usage(host_ip):
rrd_logger.info("Checking temp on host"+str(type(host_ip)))
command='ipmitool sdr elist full'
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
if str(ret).find("Inlet Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Inlet Temp"',host_ip)
if str(ret).find("Ambient Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Ambient Temp"',host_ip)
rrd_logger.info("temp stats" +str(type(temp)))
rrd_logger.info("temp stats" +str(temp))
return temp # returning temperature in degree Celsius
def get_host_temp_usage(host_ip):
rrd_logger.info("Checking temp on host"+str(type(host_ip)))
command='ipmitool sdr elist full'
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
if str(ret).find("Inlet Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Inlet Temp"',host_ip)
if str(ret).find("Ambient Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Ambient Temp"',host_ip)
rrd_logger.info("temp stats" +str(type(temp)))
rrd_logger.info("temp stats" +str(temp))
return temp # returning temperature in degree Celsius
def update_host_rrd(host_ip,m_type=None):
try:
rrd_file = get_rrd_file(host_ip.replace(".","_"))
rrd_logger.info(rrd_file)
timestamp_now = time.time()
rrd_logger.info(timestamp_now)
if not (os.path.exists(rrd_file)):
rrd_logger.warn("RRD file (%s) does not exists" % (rrd_file))
rrd_logger.warn("Creating new RRD file")
if m_type is None:
create_rrd(rrd_file,"host")
else:
create_rrd(rrd_file)
else:
rrd_logger.info("updating RRD file")
if m_type is None:
rrd_logger.debug("host_ip is"+ str(host_ip))
host_stats = get_host_resources_usage(host_ip)
output=rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, host_stats['cpu'], host_stats['ram'], host_stats['dr'], host_stats['dw'], host_stats['tx'], host_stats['rx'],host_stats['tmp'], host_stats['pwr']))
rrd_logger.debug("update status"+str(output))
else:
host_stats = get_host_resources_usage(host_ip,m_type)
rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, host_stats['cpu'], host_stats['ram'], host_stats['dr'], host_stats['dw'], host_stats['tx'], host_stats['rx']))
except Exception, e:
rrd_logger.debug("Error occured while creating/updating rrd for host: %s" % (host_ip))
rrd_logger.debug(e)
def update_host_rrd(host_ip,m_type=None):
try:
rrd_file = get_rrd_file(host_ip.replace(".","_"))
rrd_logger.info(rrd_file)
timestamp_now = time.time()
rrd_logger.info(timestamp_now)
if not (os.path.exists(rrd_file)):
rrd_logger.warn("RRD file (%s) does not exists" % (rrd_file))
rrd_logger.warn("Creating new RRD file")
if m_type is None:
create_rrd(rrd_file,"host")
else:
create_rrd(rrd_file)
else:
rrd_logger.info("updating RRD file")
if m_type is None:
rrd_logger.debug("host_ip is"+ str(host_ip))
host_stats = get_host_resources_usage(host_ip)
output=rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, host_stats['cpu'], host_stats['ram'], host_stats['dr'], host_stats['dw'], host_stats['tx'], host_stats['rx'],host_stats['tmp'], host_stats['pwr']))
rrd_logger.debug("update status"+str(output))
else:
host_stats = get_host_resources_usage(host_ip,m_type)
rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, host_stats['cpu'], host_stats['ram'], host_stats['dr'], host_stats['dw'], host_stats['tx'], host_stats['rx']))
except Exception, e:
rrd_logger.debug("Error occured while creating/updating rrd for host: %s" % (host_ip))
rrd_logger.debug(e)
def vm_utilization_rrd(host_ip, m_type=None):
"""
Handles periodic collection of VM and Host utilization data and updates of
respective RRD file.
"""
logger.info("ENTERING RRD UPDATION/CREATION........on host: %s" % host_ip)
try:
rrd_logger.debug("Starting RRD Processing for Host: %s" % host_ip)
rrd_logger.debug(host_ip)
if is_pingable(host_ip):
update_rrd(host_ip, m_type)
else:
rrd_logger.error("UNABLE TO UPDATE RRDs for host : %s" % host_ip)
except Exception as e:
rrd_logger.debug("ERROR OCCURED: %s" % e)
finally:
rrd_logger.debug("Completing RRD Processing for Host: %s" % host_ip)
logger.debug("EXITING RRD UPDATION/CREATION........on host: %s" %
host_ip)
def vm_utilization_rrd(host_ip,m_type=None):
"""
Handles periodic collection of VM and Host utilization data and updation of respective RRD file."""
logger.info("ENTERING RRD UPDATION/CREATION........on host: %s" % host_ip)
try:
rrd_logger.debug("Starting RRD Processing for Host: %s" % host_ip)
rrd_logger.debug(host_ip)
if is_pingable(host_ip):
if m_type is None:
update_rrd(host_ip)
else:
update_rrd(host_ip,m_type)
else:
rrd_logger.error("UNABLE TO UPDATE RRDs for host : %s" % host_ip)
except Exception as e:
rrd_logger.debug("ERROR OCCURED: %s" % e)
finally:
rrd_logger.debug("Completing RRD Processing for Host: %s" % host_ip)
logger.debug("EXITING RRD UPDATION/CREATION........on host: %s" % host_ip)
def get_impi_output(cmd,host_ip):
rrd_logger.debug("entering ipmi")
output=execute_remote_cmd(host_ip,'root', cmd, None, True)
rrd_logger.debug(output)
rrd_logger.debug(output[0].split("|")[4].split(" ")[1])
ret=output[0].split("|")[4].split(" ")[1]
rrd_logger.debug("cheecking ret"+str(ret))
return int(ret)
def get_impi_output(cmd,host_ip):
rrd_logger.debug("entering ipmi")
output=execute_remote_cmd(host_ip,'root', cmd, None, True)
rrd_logger.debug(output)
rrd_logger.debug(output[0].split("|")[4].split(" ")[1])
ret=output[0].split("|")[4].split(" ")[1]
rrd_logger.debug("cheecking ret"+str(ret))
return int(ret)
def shutdown_nonactive_hosts(host_list):
logger.debug(host_list)
for host in host_list:
host_ip = host.host_ip.private_ip
logger.debug("host_ip %s" %host_ip)
hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" + host_ip + "/system")
rrd_logger.debug(hypervisor_conn.getHostname())
active_dom_ids = hypervisor_conn.listDomainsID()
rrd_logger.info(active_dom_ids)
all_dom_objs = hypervisor_conn.listAllDomains()
rrd_logger.info(all_dom_objs)
rrd_logger.info(type(all_dom_objs))
if not all_dom_objs:
logger.debug("Host1 %s is free...Shutting it down " %str(host.host_ip.private_ip))
def get_actual_usage(dom_obj, host_ip):
dom_name = dom_obj.name()
cache_id = str(dom_name)
dom_stats = get_current_dom_resource_usage(dom_obj, host_ip)
"""Fetch previous domain stat value from cache"""
current.cache.disk(cache_id, lambda: dom_stats,
86400) # @UndefinedVariable
prev_dom_stats = current.cache.disk(cache_id, lambda: dom_stats,
86400) # @UndefinedVariable
rrd_logger.debug("prev_dom_stats for vm:" + str(dom_name) + " are: " +
str(prev_dom_stats))
#calulate usage
timestamp = float(dom_stats['timestamp'] -
prev_dom_stats['timestamp']) #keerti
if timestamp == 0:
cputime = dom_stats['cputime'] - prev_dom_stats['cputime']
else:
cputime = float(
(dom_stats['cputime'] - prev_dom_stats['cputime']) * 300) / float(
dom_stats['timestamp'] - prev_dom_stats['timestamp']) #keerti
usage = {'ram': dom_stats['memory']} #ram in Bytes usage
usage.update({'cpu': cputime})
usage.update({'tx': (dom_stats['tx'] - prev_dom_stats['tx'])}) #in KBytes
usage.update({'rx': (dom_stats['rx'] - prev_dom_stats['rx'])}) #in KBytes
usage.update({'dr':
(dom_stats['diskr'] - prev_dom_stats['diskr'])}) #in KBytes
usage.update({'dw':
(dom_stats['diskw'] - prev_dom_stats['diskw'])}) #in KBytes
current.cache.disk.clear(cache_id + "$") # @UndefinedVariable
"""Cache the current CPU utilization, so that difference can be calculated in next instance"""
current.cache.disk(cache_id, lambda: dom_stats,
86400) # @UndefinedVariable
latest_dom_stats = current.cache.disk(cache_id, lambda: dom_stats,
86400) # @UndefinedVariable
rrd_logger.debug("latest_dom_stats for vm:" + str(dom_name) + " are: " +
str(latest_dom_stats))
return usage
def update_vm_rrd(dom, active_dom_ids, host_ip):
dom_name = dom.name()
rrd_file = get_rrd_file(dom.name())
if not (os.path.exists(rrd_file)):
rrd_logger.warn("RRD file (%s) does not exists" % (rrd_file))
rrd_logger.warn("Creating new RRD file")
create_rrd(rrd_file)
else:
timestamp_now = time.time()
if dom.ID() in active_dom_ids:
vm_usage = get_actual_usage(dom, host_ip)
rrd_logger.debug("Usage Info for VM %s: %s" % (dom_name, vm_usage))
rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, vm_usage['cpu'], vm_usage['ram'], vm_usage['dr'], vm_usage['dw'], vm_usage['tx'], vm_usage['rx']))
else:
rrdtool.update(rrd_file, "%s:0:0:0:0:0:0" % (timestamp_now))
def update_vm_rrd(dom, active_dom_ids, host_ip):
dom_name = dom.name()
rrd_file = get_rrd_file(dom.name())
if not (os.path.exists(rrd_file)):
rrd_logger.warn("RRD file (%s) does not exists" % (rrd_file))
rrd_logger.warn("Creating new RRD file")
create_rrd(rrd_file)
else:
timestamp_now = time.time()
if dom.ID() in active_dom_ids:
vm_usage = get_actual_usage(dom, host_ip)
rrd_logger.debug("Usage Info for VM %s: %s" % (dom_name, vm_usage))
rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, vm_usage['cpu'], vm_usage['ram'], vm_usage['dr'], vm_usage['dw'], vm_usage['tx'], vm_usage['rx']))
else:
rrdtool.update(rrd_file, "%s:0:0:0:0:0:0" % (timestamp_now))
def shutdown_nonactive_hosts(host_list):
"""
Shutdown the host with no vm.
"""
logger.debug(host_list)
for host in host_list:
host_ip = host.host_ip.private_ip
logger.debug("host_ip %s" % host_ip)
hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" + host_ip +
"/system") # @UndefinedVariable
rrd_logger.debug(hypervisor_conn.getHostname())
active_dom_ids = hypervisor_conn.listDomainsID()
rrd_logger.info(active_dom_ids)
all_dom_objs = hypervisor_conn.listAllDomains()
rrd_logger.info(all_dom_objs)
rrd_logger.info(type(all_dom_objs))
if not all_dom_objs:
logger.debug("Host1 %s is free...Shutting it down " %
str(host.host_ip.private_ip))
def get_actual_usage(dom_obj, host_ip):
"""
Get actual usage by computing the difference between current values and values
stored during previous cycle.
Previous domain stat value is fetched from cache
"""
dom_name = dom_obj.name()
cache_id = str(dom_name)
dom_stats = get_current_dom_resource_usage(dom_obj, host_ip)
"""Fetch previous domain stat value from cache"""
current.cache.disk(cache_id, lambda:dom_stats, 86400) # @UndefinedVariable
prev_dom_stats = current.cache.disk(cache_id, lambda:dom_stats, 86400) # @UndefinedVariable
rrd_logger.debug("prev_dom_stats for vm:"+str(dom_name)+" are: "+str(prev_dom_stats))
#calulate usage
timestamp = float(dom_stats['timestamp'] - prev_dom_stats['timestamp']) #keerti
if timestamp == 0:
cputime = dom_stats['cputime'] - prev_dom_stats['cputime']
else:
cputime = float((dom_stats['cputime'] - prev_dom_stats['cputime'])*300)/float(dom_stats['timestamp'] - prev_dom_stats['timestamp']) #keerti
usage = {'ram' : dom_stats['memory']} #ram in Bytes usage
usage.update({'cpu' : cputime})
usage.update({'tx' : (dom_stats['tx'] - prev_dom_stats['tx'])}) #in KBytes
usage.update({'rx' : (dom_stats['rx'] - prev_dom_stats['rx'])}) #in KBytes
usage.update({'dr' : (dom_stats['diskr'] - prev_dom_stats['diskr'])}) #in KBytes
usage.update({'dw' : (dom_stats['diskw'] - prev_dom_stats['diskw'])}) #in KBytes
current.cache.disk.clear(cache_id+"$") # @UndefinedVariable
"""Cache the current CPU utilization, so that difference can be calculated in next instance"""
current.cache.disk(cache_id, lambda:dom_stats, 86400) # @UndefinedVariable
latest_dom_stats = current.cache.disk(cache_id, lambda:dom_stats, 86400) # @UndefinedVariable
rrd_logger.debug("latest_dom_stats for vm:"+str(dom_name)+" are: "+str(latest_dom_stats))
return usage
def update_host_rrd(host_ip):
try:
rrd_file = get_rrd_file(host_ip.replace(".","_"))
timestamp_now = time.time()
if not (os.path.exists(rrd_file)):
rrd_logger.warn("RRD file (%s) does not exists" % (rrd_file))
rrd_logger.warn("Creating new RRD file")
create_rrd(rrd_file)
else:
host_stats = get_host_resources_usage(host_ip)
rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, host_stats['cpu'], host_stats['ram'], host_stats['dr'], host_stats['dw'], host_stats['tx'], host_stats['rx']))
except Exception, e:
rrd_logger.debug("Error occured while creating/updating rrd for host: %s" % (host_ip))
rrd_logger.debug(e)
def update_rrd(host_ip, m_type=None):
#UPDATE CONTROLLER AND NAT RRD
if m_type is not None:
rrd_logger.info("Startiing rrd updation for nat/controller %s" %
(host_ip))
update_host_rrd(host_ip, m_type)
rrd_logger.info("Ending rrd updation for nat/controller %s" %
(host_ip))
#UPDATE HOST RRD
else:
rrd_logger.info("Startiing rrd updation for host %s" % (host_ip))
update_host_rrd(host_ip)
rrd_logger.info("Ending rrd updation for host %s" % (host_ip))
rrd_logger.info("Startiing rrd updation for VMs on host %s" %
(host_ip))
#Adding the call for setting stat periods for all VM's on the host
set_domain_memoryStatsperiod(host_ip)
#UPDATE RRD for ALL VMs on GIVEN HOST
hypervisor_conn = None
try:
hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" +
host_ip + "/system")
rrd_logger.debug(hypervisor_conn.getHostname())
active_dom_ids = hypervisor_conn.listDomainsID()
rrd_logger.info(active_dom_ids)
all_dom_objs = hypervisor_conn.listAllDomains()
rrd_logger.info(all_dom_objs)
for dom_obj in all_dom_objs:
try:
rrd_logger.info(
"Starting rrd updation for vm %s on host %s" %
(dom_obj.name(), host_ip))
update_vm_rrd(dom_obj, active_dom_ids, host_ip)
except Exception, e:
rrd_logger.debug(e)
rrd_logger.debug(
"Error occured while creating/updating rrd for VM : %s"
% dom_obj.name())
finally:
rrd_logger.info(
"Ending rrd updation for vm %s on host %s" %
(dom_obj.name(), host_ip))
def get_host_cpu_usage(host_ip, m_type=None):
rrd_logger.info("getting cpu info")
command = "cat /proc/stat | awk 'FNR == 1{print $2+$3+$4}'"
if m_type == "controller":
command_output = execute_remote_cmd("localhost", 'root', command, None,
True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None,
True)
rrd_logger.debug(command_output[0])
timestamp_now = time.time()
cpu_stats = {
'cputicks': float(command_output[0])
} # (cpu time in clock ticks
cpu_stats.update({'timestamp': timestamp_now})
prev_cpu_stats = current.cache.disk(str(host_ip), lambda: cpu_stats,
86400) # @UndefinedVariable
rrd_logger.debug(prev_cpu_stats)
timestamp = float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp'])
rrd_logger.debug("timestamp %s" % str(timestamp))
#cpu_usage = cpu_stats - prev_cpu_stats # cpu usage in last 5 min (cputime-in ticks)
if timestamp == 0:
cpu_usage = float(cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])
else:
cpu_usage = float(
(cpu_stats['cputicks'] - prev_cpu_stats['cputicks']) *
300) / float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp']
) # keerti uncomment above line and comment it
current.cache.disk.clear(str(host_ip)) # @UndefinedVariable
"""Cache the current CPU utilization, so that difference can be calculated in next instance"""
latest_cpu_stats = current.cache.disk(str(host_ip), lambda: cpu_stats,
86400) # @UndefinedVariable
rrd_logger.debug(latest_cpu_stats)
clock_ticks = os.sysconf(
os.sysconf_names['SC_CLK_TCK']) # (ticks per second)
cpu_usage = float(cpu_usage * 1000000000) / float(
clock_ticks) # (cpu time in ns)
rrd_logger.info("CPU stats of host %s is %s" % (host_ip, cpu_usage))
return (cpu_usage)
def get_host_cpu_usage(host_ip,m_type=None):
"""
Uses iostat tool to capture CPU statistics of host
"""
rrd_logger.info("getting cpu info")
command = "cat /proc/stat | awk 'FNR == 1{print $2+$3+$4}'"
if m_type=="controller":
command_output =execute_remote_cmd("localhost", 'root', command, None, True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
rrd_logger.debug(command_output[0])
timestamp_now = time.time()
cpu_stats = {'cputicks' : float(command_output[0])} # (cpu time in clock ticks
cpu_stats.update({'timestamp' : timestamp_now})
prev_cpu_stats = current.cache.disk(str(host_ip), lambda:cpu_stats, 86400) # @UndefinedVariable
rrd_logger.debug(prev_cpu_stats)
timestamp = float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp'])
rrd_logger.debug("timestamp %s" % str(timestamp))
#cpu_usage = cpu_stats - prev_cpu_stats # cpu usage in last 5 min (cputime-in ticks)
if timestamp == 0:
cpu_usage = float(cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])
else:
cpu_usage = float((cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])*300) / float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp']) # keerti uncomment above line and comment it
current.cache.disk.clear(str(host_ip)) # @UndefinedVariable
"""Cache the current CPU utilization, so that difference can be calculated in next instance"""
latest_cpu_stats = current.cache.disk(str(host_ip), lambda:cpu_stats, 86400) # @UndefinedVariable
rrd_logger.debug(latest_cpu_stats)
clock_ticks = os.sysconf(os.sysconf_names['SC_CLK_TCK']) # (ticks per second)
cpu_usage = float(cpu_usage*1000000000) / float(clock_ticks) # (cpu time in ns)
rrd_logger.info("CPU stats of host %s is %s" % ( host_ip, cpu_usage))
return (cpu_usage)
def update_rrd(host_ip,m_type=None):
#UPDATE CONTROLLER AND NAT RRD
if m_type is not None:
rrd_logger.info("Startiing rrd updation for nat/controller %s" % (host_ip))
update_host_rrd(host_ip,m_type)
rrd_logger.info("Ending rrd updation for nat/controller %s" % (host_ip))
#UPDATE HOST RRD
else:
rrd_logger.info("Startiing rrd updation for host %s" % (host_ip))
update_host_rrd(host_ip)
rrd_logger.info("Ending rrd updation for host %s" % (host_ip))
rrd_logger.info("Startiing rrd updation for VMs on host %s" % (host_ip))
#Adding the call for setting stat periods for all VM's on the host
set_domain_memoryStatsperiod(host_ip)
#UPDATE RRD for ALL VMs on GIVEN HOST
hypervisor_conn = None
try:
hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" + host_ip + "/system")
rrd_logger.debug(hypervisor_conn.getHostname())
active_dom_ids = hypervisor_conn.listDomainsID()
rrd_logger.info(active_dom_ids)
all_dom_objs = hypervisor_conn.listAllDomains()
rrd_logger.info(all_dom_objs)
for dom_obj in all_dom_objs:
try:
rrd_logger.info("Starting rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
update_vm_rrd(dom_obj, active_dom_ids, host_ip)
except Exception, e:
rrd_logger.debug(e)
rrd_logger.debug("Error occured while creating/updating rrd for VM : %s" % dom_obj.name())
finally:
rrd_logger.info("Ending rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
def update_rrd(host_ip):
#UPDATE HOST RRD
rrd_logger.info("Startiing rrd updation for host %s" % (host_ip))
update_host_rrd(host_ip)
rrd_logger.info("Ending rrd updation for host %s" % (host_ip))
#UPDATE RRD for ALL VMs on GIVEN HOST
rrd_logger.info("Startiing rrd updation for VMs on host %s" % (host_ip))
hypervisor_conn = None
try:
hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" + host_ip + "/system")
rrd_logger.debug(hypervisor_conn.getHostname())
active_dom_ids = hypervisor_conn.listDomainsID()
all_dom_objs = hypervisor_conn.listAllDomains()
for dom_obj in all_dom_objs:
try:
rrd_logger.info("Starting rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
update_vm_rrd(dom_obj, active_dom_ids, host_ip)
except Exception, e:
rrd_logger.debug(e)
rrd_logger.debug("Error occured while creating/updating rrd for VM : %s" % dom_obj.name())
finally:
rrd_logger.info("Ending rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
def create_graph(rrd_file_name, graph_type, rrd_file_path, graph_period):
rrd_logger.debug(rrd_file_name+" : "+graph_type+" : "+rrd_file_path+" : "+graph_period)
#rrd_file = rrd_file_name + '.rrd'
#shutil.copyfile(rrd_file_path, rrd_file)
graph_file = rrd_file_name + "_" + graph_type + ".png"
graph_file_dir = os.path.join(get_context_path(), 'static' + get_constant('graph_file_dir'))
graph_file_path = graph_file_dir + os.sep + graph_file
start_time = None
consolidation = 'MIN'
ds = ds1 = ds2 = None
line = line1 = line2 = None
if graph_period == 'hour':
start_time = 'now - ' + str(24*60*60)
elif graph_period == 'day':
start_time = '-1w'
elif graph_period == 'month':
start_time = '-1y'
elif graph_period == 'week':
start_time = '-1m'
elif graph_period == 'year':
start_time = '-5y'
if ((graph_type == 'ram') or (graph_type == 'cpu')):
if graph_type == 'ram':
ds = 'DEF:ram=' + rrd_file_path + ':ram:' + consolidation
line = 'LINE1:ram#0000FF:Memory'
graph_type += " (Bytes/Sec)"
upper_limit = ""
elif graph_type == 'cpu':
ds = 'DEF:cpu=' + rrd_file_path + ':cpu:' + consolidation
line = 'LINE1:cpu#0000FF:CPU'
graph_type += " (%)"
upper_limit = "-u 100"
rrdtool.graph(graph_file_path, '--start', start_time, '--end', 'now', '--vertical-label', graph_type, '--watermark', time.asctime(), '-t', ' ' + rrd_file_name, ds, line, "-l 0 --alt-y-grid -L 6" + upper_limit )
else:
if graph_type == 'nw':
ds1 = 'DEF:nwr=' + rrd_file_path + ':tx:' + consolidation
ds2 = 'DEF:nww=' + rrd_file_path + ':rx:' + consolidation
line1 = 'LINE1:nwr#0000FF:Transmit'
line2 = 'LINE1:nww#FF7410:Receive'
elif graph_type == 'disk':
ds1 = 'DEF:diskr=' + rrd_file_path + ':dr:' + consolidation
ds2 = 'DEF:diskw=' + rrd_file_path + ':dw:' + consolidation
line1 = 'LINE1:diskr#0000FF:DiskRead'
line2 = 'LINE1:diskw#FF7410:DiskWrite'
graph_type += " (Bytes/Sec)"
rrdtool.graph(graph_file_path, '--start', start_time, '--end', 'now', '--vertical-label', graph_type, '--watermark', time.asctime(), '-t', ' ' + rrd_file_name, ds1, ds2, line1, line2, "-l 0 --alt-y-grid -L 6" )
rrd_logger.debug(graph_file_path)
if os.path.exists(graph_file_path):
return True
else:
return False
def fetch_info_graph(vm_identity,graph_period,g_type,vm_ram,m_type,host_cpu):
rrd_logger.debug("fetch_info graph")
start_time = None
# consolidation = 'MIN'
end_time = 'now'
rrd_file = get_rrd_file(vm_identity)
if graph_period == 'hour':
start_time = 'now - ' + str(12*300)
elif graph_period == 'day':
start_time = 'now - ' + str(12*300*24)
elif graph_period == 'month':
start_time = 'now - ' + str(12*300*24*30)
elif graph_period == 'week':
start_time = 'now - ' + str(12*300*24*7)
elif graph_period == 'year':
start_time = 'now - ' + str(12*300*24*365)
result=[]
result1=[]
result2=[]
result3=[]
rrd_logger.debug(rrd_file)
if os.path.exists(rrd_file):
rrd_ret =rrdtool.fetch(rrd_file, 'MIN', '--start', start_time, '--end', end_time)
rrd_logger.debug("fetched rrd data")
fld_info = rrd_ret[1]
data_info = rrd_ret[2]
tim_info=rrd_ret[0][0]
cpu_idx = fld_info.index('cpu')
mem_idx = fld_info.index('ram')
dskr_idx = fld_info.index('dr')
dskw_idx = fld_info.index('dw')
nwr_idx = fld_info.index('tx')
nww_idx = fld_info.index('rx')
if m_type=="host":
tmp_idx = fld_info.index('tmp')
power_idx = fld_info.index('pwr')
rrd_logger.debug("tmp poser")
timeinterval=1
for data in data_info:
info1={}
info={}
time_info=(int(tim_info) + 300*timeinterval)*1000 #changing timestamp from rrd into javascript timezone to represent time on graph
timeinterval+=1
if g_type=="cpu":
if data[cpu_idx] != None:
cpu_no=host_cpu.split(" ")
info['y']=round((float(data[cpu_idx])*100)/(float(int(cpu_no[0])*5*60*1000000000)),3) #dividing the rrd value by no of cores*5min(converted into nanoseconds)
else:
info['y']=float(0)
info['x']=time_info
result3.append(info)
if g_type=="ram":
if data[mem_idx] != None and data[mem_idx]>0:
if (int(vm_ram)>1024) or (m_type=='host'):
mem=round(float(data[mem_idx])/(1024*1024*1024),2)
else:
mem=round(float(data[mem_idx])/(1024*1024),2)
info['y']=mem
else:
info['y']=float(0)
info['x']=time_info
result3.append(info)
if m_type=="host":
if g_type=="tmp":
if data[tmp_idx] != None and data[tmp_idx]>0:
info['y']=round(float(data[tmp_idx]),2)
else:
info['y']=float(0)
info['x']=time_info
result3.append(info)
if g_type=="power":
if data[power_idx] != None and data[power_idx]>0:
info['y']=data[power_idx]
else:
info['y']=float(0)
info['x']=time_info
result3.append(info)
if g_type=="disk":
if data[dskr_idx] != None:
info1['y']=round(float(data[dskr_idx])/(1024*1024),2)
else:
info1['y']=float(0)
if data[dskw_idx] != None:
info['y']=round(float(data[dskw_idx])/(1024*1024),2)
else:
info['y']=float(0)
info['x']=time_info
info1['x']=time_info
result1.append(info1)
result2.append(info)
if g_type=="nw":
if data[nwr_idx] != None:
info1['y']=round(float(data[nwr_idx])/(1024*1024),2)
else:
info1['y']=float(0)
if data[nww_idx] != None:
info['y']=round(float(data[nww_idx])/(1024*1024),2)
else:
info['y']=float(0)
info['x']=time_info
info1['x']=time_info
result1.append(info1)
result2.append(info)
if g_type=='ram' or g_type=='cpu' or g_type=='tmp' or g_type=='power':
rrd_logger.debug("result3")
return result3
if g_type=='nw' or g_type=='disk':
result.append(result1)
result.append(result2)
rrd_logger.debug("result")
return result
active_dom_ids = hypervisor_conn.listDomainsID()
all_dom_objs = hypervisor_conn.listAllDomains()
for dom_obj in all_dom_objs:
try:
rrd_logger.info("Starting rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
update_vm_rrd(dom_obj, active_dom_ids, host_ip)
except Exception, e:
rrd_logger.debug(e)
rrd_logger.debug("Error occured while creating/updating rrd for VM : %s" % dom_obj.name())
finally:
rrd_logger.info("Ending rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
except Exception, e:
rrd_logger.debug(e)
finally:
if hypervisor_conn:
hypervisor_conn.close()
rrd_logger.info("Starting rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
update_vm_rrd(dom_obj, active_dom_ids, host_ip)
except Exception, e:
rrd_logger.debug(e)
rrd_logger.debug("Error occured while creating/updating rrd for VM : %s" % dom_obj.name())
finally:
rrd_logger.info("Ending rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
except Exception, e:
rrd_logger.debug(e)
finally:
rrd_logger.info("Ending rrd updation for vms on host %s" % host_ip)
if hypervisor_conn:
hypervisor_conn.close()
#fetch graph data for vm/host from their respective rrd file
def fetch_info_graph(vm_identity,graph_period,g_type,vm_ram,m_type,host_cpu):
rrd_logger.debug("fetch_info graph")
start_time = None
# consolidation = 'MIN'
end_time = 'now'
rrd_file = get_rrd_file(vm_identity)
if graph_period == 'hour':
start_time = 'now - ' + str(12*300)
elif graph_period == 'day':
rrd_logger.debug("cheecking ret"+str(ret))
return int(ret)
"""Uses IMPITOOL to get host temp """
def get_host_temp_usage(host_ip):
rrd_logger.info("Checking temp on host"+str(type(host_ip)))
command='ipmitool sdr elist full'
try:
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
except Exception, e:
temp=0
ret=""
rrd_logger.debug("ipmitool not installed")
if str(ret).find("Inlet Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Inlet Temp"',host_ip)
if str(ret).find("Ambient Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Ambient Temp"',host_ip)
rrd_logger.info("temp stats" +str(type(temp)))
rrd_logger.info("temp stats" +str(temp))
return temp # returning temperature in degree Celsius
"""Uses IMPITOOL to get host power consumption"""
def get_host_power_usage(host_ip):
rrd_logger.info("Checking power on host"+str(host_ip))
command='ipmitool sdr elist full'
try:
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
except Exception, e:
def fetch_info_graph(vm_identity,graph_period,g_type,vm_ram,m_type,host_cpu):
rrd_logger.debug("fetch_info graph")
start_time = None
# consolidation = 'MIN'
end_time = 'now'
rrd_file = get_rrd_file(vm_identity)
if graph_period == 'hour':
start_time = 'now - ' + str(12*300)
elif graph_period == 'day':
start_time = 'now - ' + str(12*300*24)
elif graph_period == 'month':
start_time = 'now - ' + str(12*300*24*30)
elif graph_period == 'week':
start_time = 'now - ' + str(12*300*24*7)
elif graph_period == 'year':
start_time = 'now - ' + str(12*300*24*365)
result=[]
result1=[]
result2=[]
result3=[]
rrd_logger.debug(rrd_file)
if os.path.exists(rrd_file):
rrd_ret =rrdtool.fetch(rrd_file, 'MIN', '--start', start_time, '--end', end_time)
rrd_logger.debug("fetched rrd data")
fld_info = rrd_ret[1]
data_info = rrd_ret[2]
tim_info=rrd_ret[0][0]
cpu_idx = fld_info.index('cpu')
mem_idx = fld_info.index('ram')
dskr_idx = fld_info.index('dr')
dskw_idx = fld_info.index('dw')
nwr_idx = fld_info.index('tx')
nww_idx = fld_info.index('rx')
if m_type=="host":
tmp_idx = fld_info.index('tmp')
power_idx = fld_info.index('pwr')
rrd_logger.debug("tmp poser")
timeinterval=1
for data in data_info:
info1={}
info={}
time_info=(int(tim_info) + 300*timeinterval)*1000 #changing timestamp from rrd into javascript timezone to represent time on graph
timeinterval+=1
if g_type=="cpu":
if data[cpu_idx] != None:
cpu_no=host_cpu.split(" ")
info['y']=round((float(data[cpu_idx])*100)/(float(int(cpu_no[0])*5*60*1000000000)),3) #dividing the rrd value by no of cores*5min(converted into nanoseconds)
else:
info['y']=float(0)
info['x']=time_info
result3.append(info)
if g_type=="ram":
if data[mem_idx] != None and data[mem_idx]>0:
if (int(vm_ram)>1024) or (m_type=='host'):
mem=round(float(data[mem_idx])/(1024*1024*1024),2)
else:
mem=round(float(data[mem_idx])/(1024*1024),2)
info['y']=mem
else:
info['y']=float(0)
info['x']=time_info
result3.append(info)
if m_type=="host":
if g_type=="tmp":
if data[tmp_idx] != None and data[tmp_idx]>0:
info['y']=round(float(data[tmp_idx]),2)
else:
info['y']=float(0)
info['x']=time_info
result3.append(info)
if g_type=="power":
if data[power_idx] != None and data[power_idx]>0:
info['y']=data[power_idx]
else:
info['y']=float(0)
info['x']=time_info
result3.append(info)
if g_type=="disk":
if data[dskr_idx] != None:
info1['y']=round(float(data[dskr_idx])/(1024*1024),2)
else:
info1['y']=float(0)
if data[dskw_idx] != None:
info['y']=round(float(data[dskw_idx])/(1024*1024),2)
else:
info['y']=float(0)
info['x']=time_info
info1['x']=time_info
result1.append(info1)
result2.append(info)
if g_type=="nw":
if data[nwr_idx] != None:
info1['y']=round(float(data[nwr_idx])/(1024*1024),2)
else:
info1['y']=float(0)
if data[nww_idx] != None:
info['y']=round(float(data[nww_idx])/(1024*1024),2)
else:
info['y']=float(0)
info['x']=time_info
info1['x']=time_info
result1.append(info1)
result2.append(info)
if g_type=='ram' or g_type=='cpu' or g_type=='tmp' or g_type=='power':
rrd_logger.debug("result3")
return result3
if g_type=='nw' or g_type=='disk':
result.append(result1)
result.append(result2)
rrd_logger.debug("result")
return result