def get_host_resources_usage(host_ip,m_type=None):
"""
Get following resource usage information from host
- CPU
- RAM
- Disk
- Network
"""
rrd_logger.info("getting data for RRD file")
if m_type is None:
host_cpu_usage = get_host_cpu_usage(host_ip)
host_disk_usage = get_host_disk_usage(host_ip)
host_mem_usage = get_host_mem_usage(host_ip)
host_nw_usage = get_host_nw_usage(host_ip)
else:
host_cpu_usage = get_host_cpu_usage(host_ip,m_type)
host_disk_usage = get_host_disk_usage(host_ip,m_type)
host_mem_usage = get_host_mem_usage(host_ip,m_type)
host_nw_usage = get_host_nw_usage(host_ip,m_type)
host_usage = {'cpu' : host_cpu_usage} #percent cpu usage
host_usage.update({'dr' : host_disk_usage[0]*1024}) #Bytes/s
host_usage.update({'dw' : host_disk_usage[1]*1024})
host_usage.update({'ram' : host_mem_usage*1024}) #Bytes
host_usage.update({'rx' : host_nw_usage[0]}) #in Bytes
host_usage.update({'tx' : host_nw_usage[1]}) #in Bytes
rrd_logger.info("Host %s stats: %s" % (host_ip, host_usage))
return host_usage
def compare_rrd_data_with_threshold(rrd_file_name,thresholdcontext):
start_time = 'now - ' + str(VM_UTIL_20_DAYS*24*60*60)
''' start_time = 'now - ' + str(60*60) '''
end_time = 'now'
rrd_file = get_rrd_file(rrd_file_name)
cpu_data = []
mem_data = []
dskr_data = []
dskw_data = []
nwr_data = []
nww_data = []
rrd_logger.info("inside compare_rrd_data_with_threshold function rrd file is :"+str(rrd_file_name))
if os.path.exists(rrd_file):
rrd_ret =rrdtool.fetch(rrd_file, 'MIN', '--start', start_time, '--end', end_time)
fld_info = rrd_ret[1]
data_info = rrd_ret[2]
cpu_idx = fld_info.index('cpu')
mem_idx = fld_info.index('ram')
dskr_idx = fld_info.index('dr')
dskw_idx = fld_info.index('dw')
nwr_idx = fld_info.index('tx')
nww_idx = fld_info.index('rx')
cpu_threshold='N'
read_threshold='N'
write_threshold='N'
CPUNoneIdentifier='Y'
NWRNoneIdentifier='Y'
NWWNoneIdentifier='Y'
for row in data_info:
if (row[cpu_idx] != None) :
CPUNoneIdentifier='N'
if int(row[cpu_idx]) > int(thresholdcontext['CPUThreshold']) : cpu_threshold='Y'
if (row[nwr_idx] != None) :
NWRNoneIdentifier='N'
if int(row[nwr_idx]) > int(thresholdcontext['ReadThreshold']) : read_threshold='Y'
if (row[nww_idx] != None) :
NWWNoneIdentifier='N'
if int(row[nww_idx]) > int(thresholdcontext['WriteThreshold']) : write_threshold='Y'
if (cpu_threshold=='Y' or read_threshold=='Y' or write_threshold=='Y') :
rrd_logger.info("Info about the VM:%s,row[cpu_idx]:%s,row[nwr_idx]:%s,row[nww_idx]:%s" %(rrd_file_name,row[cpu_idx],row[nwr_idx],row[nww_idx]))
rrd_logger.info("Threshold is reached once.. VM:"+str(rrd_file_name)+" is in use")
return False
## If only none values are read from the rrd .. do not send a warning email
if(CPUNoneIdentifier=='N' and NWRNoneIdentifier=='N' and NWWNoneIdentifier =='N'):
rrd_logger.info("Returning true to send warning email as threshold is never reached for VM:%s" %(rrd_file_name) )
return True
else:
rrd_logger.info("RRD capturing is not correct... returning all null values only for VM:%s" %(rrd_file_name) )
return False
def get_dom_mem_usage(dom, host):
"""
Captures memory utilization of a VM from the host.
VMs run on host as individual processes. Memory utilization of the process is derived.
"""
rrd_logger.info("memory stats for VM:%s is %s" %(dom.name(),dom.memoryStats()))
domain_memUsed=(dom.memoryStats()['available']-dom.memoryStats()['unused'])
rrd_logger.info("domain_memUsed is : "+str(domain_memUsed))
return (domain_memUsed*1024)
def get_host_power_usage(host_ip):
rrd_logger.info("Checking power on host"+str(host_ip))
command='ipmitool sdr elist full'
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
if str(ret).find("System Level")!=int(-1):
pwr_usage=get_impi_output('ipmitool sdr elist full | grep "System Level"',host_ip)
if str(ret).find("Pwr Consumption")!=int(-1):
pwr_usage=get_impi_output('ipmitool sdr elist full | grep "Pwr Consumption"',host_ip)
rrd_logger.info("power stats of host %s is %s" % (host_ip, str(pwr_usage)))
return pwr_usage # returining power in Watt
def task_rrd():
list_host=[]
ip_cmd="/sbin/ifconfig | grep 'inet addr' | sed -n '1p' | awk '{print $2}' | cut -d ':' -f 2"
controller_ip = execute_remote_cmd("localhost", 'root', ip_cmd, None, True).strip()
rrd_logger.info(controller_ip[0].strip("\n"))
nat_ip=execute_remote_cmd("localhost", 'root', ip_cmd, None, True).strip()
list_host.append(controller_ip)
list_host.append(nat_ip)
for ip in list_host:
m_type="controller" if controller_ip==ip else "nat"
vm_utilization_rrd(ip,m_type)
def get_host_disk_usage(host_ip,m_type=None):
command = "iostat -d | sed '1,2d'"
if m_type=="controller":
command_output =execute_remote_cmd("localhost", 'root', command, None, True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
disk_stats = re.split('\s+', command_output[1])
rrd_logger.info("Disk stats of host %s is dr: %s dw: %s" % (host_ip, disk_stats[2], disk_stats[3]))
return [float(disk_stats[2]), float(disk_stats[3])] #return memory in KB/sec
def get_host_mem_usage(host_ip,m_type=None):
command = "free -k | grep 'buffers/cache' | awk '{print $3}'"
if m_type=="controller":
command_output =execute_remote_cmd("localhost", 'root', command, None, True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
rrd_logger.info("command_output is %s"%(command_output[0]))
used_mem_in_kb = int(command_output[0])
rrd_logger.info("Mem stats of host %s is %s" % (host_ip, used_mem_in_kb))
return used_mem_in_kb #return memory in KB
def get_host_nw_usage(host_ip,m_type=None):
command = "ifconfig baadal-br-int | grep 'RX bytes:'"
if m_type=="controller":
command_output =execute_remote_cmd("localhost", 'root', command, None, True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
nw_stats = re.split('\s+', command_output[0])
rx = int(re.split(':', nw_stats[2])[1])
tx = int(re.split(':', nw_stats[6])[1])
rrd_logger.info("Network stats of host %s is rx: %s tx: %s" % (host_ip, rx, tx))
return [rx, tx] # return network in Bytes
def get_host_resources_usage(host_ip,m_type=None):
rrd_logger.info("getting data for RRD file")
if m_type is None:
host_cpu_usage = get_host_cpu_usage(host_ip)
host_disk_usage = get_host_disk_usage(host_ip)
host_mem_usage = get_host_mem_usage(host_ip)
host_nw_usage = get_host_nw_usage(host_ip)
host_temp_usage =get_host_temp_usage(host_ip)
host_power_usage =get_host_power_usage(host_ip)
rrd_logger.info("host_cpu_usage"+str(type(host_cpu_usage)))
rrd_logger.info("host_temp_usage"+str(type(host_temp_usage)))
else:
host_cpu_usage = get_host_cpu_usage(host_ip,m_type)
host_disk_usage = get_host_disk_usage(host_ip,m_type)
host_mem_usage = get_host_mem_usage(host_ip,m_type)
host_nw_usage = get_host_nw_usage(host_ip,m_type)
host_usage = {'cpu' : host_cpu_usage} #percent cpu usage
host_usage.update({'dr' : host_disk_usage[0]*1024}) #Bytes/s
host_usage.update({'dw' : host_disk_usage[1]*1024})
host_usage.update({'ram' : host_mem_usage*1024}) #Bytes
host_usage.update({'rx' : host_nw_usage[0]}) #in Bytes
host_usage.update({'tx' : host_nw_usage[1]}) #in Bytes
if m_type is None:
host_usage.update({'tmp' : host_temp_usage}) #in Degree Celsius
host_usage.update({'pwr' : host_power_usage}) #in Watt
rrd_logger.info("Host %s stats: %s" % (host_ip, host_usage))
return host_usage
def get_dom_nw_usage(dom_obj):
tree = ElementTree.fromstring(dom_obj.XMLDesc(0))
rx = 0
tx = 0
for target in tree.findall("devices/interface/target"):
device = target.get("dev")
stats = dom_obj.interfaceStats(device)
rx += stats[0]
tx += stats[4]
rrd_logger.info("%s%s" % (rx, tx))
return [rx, tx] #returned value in Bytes by default
def task_rrd():
list_host=[]
controller_ip=controller_ip=os.popen("ifconfig | grep 'inet addr' | sed -n '1p' | awk '{print $2}' | cut -d ':' -f 2").readlines()
rrd_logger.info(controller_ip[0].strip("\n"))
controller_ip=controller_ip[0].strip("\n")
nat_ip=os.popen("route -n | sed -n '3p' | awk '{print $2}'").readlines()
nat_ip=nat_ip[0].strip("\n")
list_host.append(controller_ip)
list_host.append(nat_ip)
for ip in list_host:
if controller_ip==ip:
m_type="controller"
else:
m_type="nat"
vm_utilization_rrd(ip,m_type)
def get_dom_nw_usage(dom_obj):
"""
Uses libvirt function to extract interface device statistics for a domain
to find network usage of virtual machine.
"""
tree = ElementTree.fromstring(dom_obj.XMLDesc(0))
rx = 0
tx = 0
for target in tree.findall("devices/interface/target"):
device = target.get("dev")
stats = dom_obj.interfaceStats(device)
rx += stats[0]
tx += stats[4]
rrd_logger.info("%s%s" % (rx, tx))
return [rx, tx] #returned value in Bytes by default
def get_dom_disk_usage(dom_obj):
tree = ElementTree.fromstring(dom_obj.XMLDesc(0))
bytesr = 0
bytesw = 0
rreq = 0
wreq = 0
for target in tree.findall("devices/disk/target"):
device = target.get("dev")
stats = dom_obj.blockStats(device)
rreq += stats[0]
bytesr += stats[1]
wreq += stats[2]
bytesw += stats[3]
rrd_logger.info("rreq: %s bytesr: %s wreq: %s bytesw: %s" % (rreq, bytesr, wreq, bytesw))
return [bytesr, bytesw] #returned value in Bytes by default
def get_current_dom_resource_usage(dom, host_ip):
dom_memusage = get_dom_mem_usage(dom, host_ip)
dom_nw_usage = get_dom_nw_usage(dom)
dom_disk_usage = get_dom_disk_usage(dom)
timestamp_now = time.time() #keerti
dom_stats = {'rx' : dom_nw_usage[0]}
dom_stats.update({'tx' : dom_nw_usage[1]})
dom_stats.update({'diskr' : dom_disk_usage[0]})
dom_stats.update({'diskw' : dom_disk_usage[1]})
dom_stats.update({'memory' : dom_memusage})
dom_stats.update({'cputime' : dom.info()[4]})
dom_stats.update({'cpus' : dom.info()[3]})
dom_stats.update({'timestamp': timestamp_now})
rrd_logger.info(dom_stats)
rrd_logger.warn("As we get VM mem usage info from rrs size of the process running on host therefore it is observed that the memused is sometimes greater than max mem specified in case when the VM uses memory near to its mam memory")
return dom_stats
def update_host_rrd(host_ip,m_type=None):
"""
Updates the RRD file for host with resource utilization data
"""
try:
rrd_file = _get_rrd_file(host_ip.replace(".","_"))
rrd_logger.info(rrd_file)
timestamp_now = time.time()
rrd_logger.info(timestamp_now)
if not (os.path.exists(rrd_file)):
rrd_logger.warn("RRD file (%s) does not exists" % (rrd_file))
rrd_logger.warn("Creating new RRD file")
create_rrd(rrd_file)
else:
rrd_logger.info("updating RRD file")
if m_type is None:
host_stats = get_host_resources_usage(host_ip)
else:
host_stats = get_host_resources_usage(host_ip,m_type)
rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, host_stats['cpu'], host_stats['ram'], host_stats['dr'], host_stats['dw'], host_stats['tx'], host_stats['rx']))
except Exception, e:
rrd_logger.debug("Error occured while creating/updating rrd for host: %s" % (host_ip))
rrd_logger.debug(e)
def update_host_rrd(host_ip,m_type=None):
try:
rrd_file = get_rrd_file(host_ip.replace(".","_"))
rrd_logger.info(rrd_file)
timestamp_now = time.time()
rrd_logger.info(timestamp_now)
if not (os.path.exists(rrd_file)):
rrd_logger.warn("RRD file (%s) does not exists" % (rrd_file))
rrd_logger.warn("Creating new RRD file")
if m_type is None:
create_rrd(rrd_file,"host")
else:
create_rrd(rrd_file)
else:
rrd_logger.info("updating RRD file")
if m_type is None:
rrd_logger.debug("host_ip is"+ str(host_ip))
host_stats = get_host_resources_usage(host_ip)
output=rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, host_stats['cpu'], host_stats['ram'], host_stats['dr'], host_stats['dw'], host_stats['tx'], host_stats['rx'],host_stats['tmp'], host_stats['pwr']))
rrd_logger.debug("update status"+str(output))
else:
host_stats = get_host_resources_usage(host_ip,m_type)
rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, host_stats['cpu'], host_stats['ram'], host_stats['dr'], host_stats['dw'], host_stats['tx'], host_stats['rx']))
except Exception, e:
rrd_logger.debug("Error occured while creating/updating rrd for host: %s" % (host_ip))
rrd_logger.debug(e)
def get_host_cpu_usage(host_ip,m_type=None):
"""
Uses iostat tool to capture CPU statistics of host
"""
rrd_logger.info("getting cpu info")
command = "cat /proc/stat | awk 'FNR == 1{print $2+$3+$4}'"
if m_type=="controller":
command_output =execute_remote_cmd("localhost", 'root', command, None, True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
rrd_logger.debug(command_output[0])
timestamp_now = time.time()
cpu_stats = {'cputicks' : float(command_output[0])} # (cpu time in clock ticks
cpu_stats.update({'timestamp' : timestamp_now})
prev_cpu_stats = current.cache.disk(str(host_ip), lambda:cpu_stats, 86400) # @UndefinedVariable
rrd_logger.debug(prev_cpu_stats)
timestamp = float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp'])
rrd_logger.debug("timestamp %s" % str(timestamp))
#cpu_usage = cpu_stats - prev_cpu_stats # cpu usage in last 5 min (cputime-in ticks)
if timestamp == 0:
cpu_usage = float(cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])
else:
cpu_usage = float((cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])*300) / float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp']) # keerti uncomment above line and comment it
current.cache.disk.clear(str(host_ip)) # @UndefinedVariable
"""Cache the current CPU utilization, so that difference can be calculated in next instance"""
latest_cpu_stats = current.cache.disk(str(host_ip), lambda:cpu_stats, 86400) # @UndefinedVariable
rrd_logger.debug(latest_cpu_stats)
clock_ticks = os.sysconf(os.sysconf_names['SC_CLK_TCK']) # (ticks per second)
cpu_usage = float(cpu_usage*1000000000) / float(clock_ticks) # (cpu time in ns)
rrd_logger.info("CPU stats of host %s is %s" % ( host_ip, cpu_usage))
return (cpu_usage)
def get_dom_disk_usage(dom_obj):
"""
Uses libvirt function to extract block device statistics for a domain
to find disk usage of virtual machine.
"""
tree = ElementTree.fromstring(dom_obj.XMLDesc(0))
bytesr = 0
bytesw = 0
rreq = 0
wreq = 0
for target in tree.findall("devices/disk/target"):
device = target.get("dev")
stats = dom_obj.blockStats(device)
rreq += stats[0]
bytesr += stats[1]
wreq += stats[2]
bytesw += stats[3]
rrd_logger.info("rreq: %s bytesr: %s wreq: %s bytesw: %s" % (rreq, bytesr, wreq, bytesw))
return [bytesr, bytesw] #returned value in Bytes by default
def update_rrd(host_ip):
#UPDATE HOST RRD
rrd_logger.info("Startiing rrd updation for host %s" % (host_ip))
update_host_rrd(host_ip)
rrd_logger.info("Ending rrd updation for host %s" % (host_ip))
#UPDATE RRD for ALL VMs on GIVEN HOST
rrd_logger.info("Startiing rrd updation for VMs on host %s" % (host_ip))
hypervisor_conn = None
try:
hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" + host_ip + "/system")
rrd_logger.debug(hypervisor_conn.getHostname())
active_dom_ids = hypervisor_conn.listDomainsID()
all_dom_objs = hypervisor_conn.listAllDomains()
for dom_obj in all_dom_objs:
try:
rrd_logger.info("Starting rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
update_vm_rrd(dom_obj, active_dom_ids, host_ip)
except Exception, e:
rrd_logger.debug(e)
rrd_logger.debug("Error occured while creating/updating rrd for VM : %s" % dom_obj.name())
finally:
rrd_logger.info("Ending rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
def get_host_cpu_usage(host_ip,m_type=None):
rrd_logger.info("getting cpu info")
command = "cat /proc/stat | awk 'FNR == 1{print $2+$3+$4}'"
if m_type=="controller":
command_output =execute_remote_cmd("localhost", 'root', command, None, True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
rrd_logger.debug(command_output[0])
timestamp_now = time.time()
cpu_stats = {'cputicks' : float(command_output[0])} # (cpu time in clock ticks
cpu_stats.update({'timestamp' : timestamp_now})
prev_cpu_stats = current.cache.disk(str(host_ip), lambda:cpu_stats, 86400) # @UndefinedVariable
rrd_logger.debug(prev_cpu_stats)
timestamp = float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp'])
rrd_logger.debug("timestamp %s" % str(timestamp))
#cpu_usage = cpu_stats - prev_cpu_stats # cpu usage in last 5 min (cputime-in ticks)
if timestamp == 0:
cpu_usage = float(cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])
else:
cpu_usage = float((cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])*300) / float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp']) # keerti uncomment above line and comment it
current.cache.disk.clear(str(host_ip)) # @UndefinedVariable
"""Cache the current CPU utilization, so that difference can be calculated in next instance"""
latest_cpu_stats = current.cache.disk(str(host_ip), lambda:cpu_stats, 86400) # @UndefinedVariable
rrd_logger.debug(latest_cpu_stats)
clock_ticks = os.sysconf(os.sysconf_names['SC_CLK_TCK']) # (ticks per second)
cpu_usage = float(cpu_usage*1000000000) / float(clock_ticks) # (cpu time in ns)
rrd_logger.info("CPU stats of host %s is %s" % ( host_ip, cpu_usage))
return (cpu_usage)
def get_dom_mem_usage(dom, host):
rrd_logger.info("memory stats for VM:%s is %s" %(dom.name(),dom.memoryStats()))
rrd_logger.warn(dom.memoryStats())
try:
domain_memUsed=(dom.memoryStats()['available']-dom.memoryStats()['unused'])
rrd_logger.info("domain_memUsed is : "+str(domain_memUsed))
except:
domain_memUsed=(dom.memoryStats()['rss'])
rrd_logger.info("in except and rss value is : "+str(domain_memUsed))
return (domain_memUsed*1024)
def get_dom_mem_usage(dom, host):
rrd_logger.info("memory stats for VM:%s is %s" %(dom.name(),dom.memoryStats()))
rrd_logger.warn(dom.memoryStats())
try:
domain_memUsed=(dom.memoryStats()['available']-dom.memoryStats()['unused'])
rrd_logger.info("domain_memUsed is : "+str(domain_memUsed))
except:
domain_memUsed=(dom.memoryStats()['rss'])
rrd_logger.info("in except and rss value is : "+str(domain_memUsed))
return (domain_memUsed*1024)
def get_host_temp_usage(host_ip):
rrd_logger.info("Checking temp on host"+str(type(host_ip)))
command='ipmitool sdr elist full'
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
if str(ret).find("Inlet Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Inlet Temp"',host_ip)
if str(ret).find("Ambient Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Ambient Temp"',host_ip)
rrd_logger.info("temp stats" +str(type(temp)))
rrd_logger.info("temp stats" +str(temp))
return temp # returning temperature in degree Celsius
def get_host_temp_usage(host_ip):
rrd_logger.info("Checking temp on host"+str(type(host_ip)))
command='ipmitool sdr elist full'
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
if str(ret).find("Inlet Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Inlet Temp"',host_ip)
if str(ret).find("Ambient Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Ambient Temp"',host_ip)
rrd_logger.info("temp stats" +str(type(temp)))
rrd_logger.info("temp stats" +str(temp))
return temp # returning temperature in degree Celsius
def shutdown_nonactive_hosts(host_list):
logger.debug(host_list)
for host in host_list:
host_ip = host.host_ip.private_ip
logger.debug("host_ip %s" %host_ip)
hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" + host_ip + "/system")
rrd_logger.debug(hypervisor_conn.getHostname())
active_dom_ids = hypervisor_conn.listDomainsID()
rrd_logger.info(active_dom_ids)
all_dom_objs = hypervisor_conn.listAllDomains()
rrd_logger.info(all_dom_objs)
rrd_logger.info(type(all_dom_objs))
if not all_dom_objs:
logger.debug("Host1 %s is free...Shutting it down " %str(host.host_ip.private_ip))
def update_host_rrd(host_ip, m_type=None):
try:
rrd_file = get_rrd_file(host_ip.replace(".", "_"))
rrd_logger.info(rrd_file)
timestamp_now = time.time()
rrd_logger.info(timestamp_now)
if not (os.path.exists(rrd_file)):
rrd_logger.warn("RRD file (%s) does not exists" % (rrd_file))
rrd_logger.warn("Creating new RRD file")
if m_type is None:
create_rrd(rrd_file, "host")
else:
create_rrd(rrd_file)
else:
rrd_logger.info("updating RRD file")
if m_type is None:
rrd_logger.debug("host_ip is" + str(host_ip))
host_stats = get_host_resources_usage(host_ip)
output = rrdtool.update(
rrd_file, "%s:%s:%s:%s:%s:%s:%s:%s:%s" %
(timestamp_now, host_stats['cpu'], host_stats['ram'],
host_stats['dr'], host_stats['dw'], host_stats['tx'],
host_stats['rx'], host_stats['tmp'], host_stats['pwr']))
rrd_logger.debug("update status" + str(output))
else:
host_stats = get_host_resources_usage(host_ip, m_type)
rrdtool.update(
rrd_file, "%s:%s:%s:%s:%s:%s:%s" %
(timestamp_now, host_stats['cpu'], host_stats['ram'],
host_stats['dr'], host_stats['dw'], host_stats['tx'],
host_stats['rx']))
except Exception, e:
rrd_logger.debug(
"Error occured while creating/updating rrd for host: %s" %
(host_ip))
rrd_logger.debug(e)
def shutdown_nonactive_hosts(host_list):
"""
Shutdown the host with no vm.
"""
logger.debug(host_list)
for host in host_list:
host_ip = host.host_ip.private_ip
logger.debug("host_ip %s" % host_ip)
hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" + host_ip +
"/system") # @UndefinedVariable
rrd_logger.debug(hypervisor_conn.getHostname())
active_dom_ids = hypervisor_conn.listDomainsID()
rrd_logger.info(active_dom_ids)
all_dom_objs = hypervisor_conn.listAllDomains()
rrd_logger.info(all_dom_objs)
rrd_logger.info(type(all_dom_objs))
if not all_dom_objs:
logger.debug("Host1 %s is free...Shutting it down " %
str(host.host_ip.private_ip))
def update_rrd(host_ip,m_type=None):
#UPDATE CONTROLLER AND NAT RRD
if m_type is not None:
rrd_logger.info("Startiing rrd updation for nat/controller %s" % (host_ip))
update_host_rrd(host_ip,m_type)
rrd_logger.info("Ending rrd updation for nat/controller %s" % (host_ip))
#UPDATE HOST RRD
else:
rrd_logger.info("Startiing rrd updation for host %s" % (host_ip))
update_host_rrd(host_ip)
rrd_logger.info("Ending rrd updation for host %s" % (host_ip))
rrd_logger.info("Startiing rrd updation for VMs on host %s" % (host_ip))
#Adding the call for setting stat periods for all VM's on the host
set_domain_memoryStatsperiod(host_ip)
#UPDATE RRD for ALL VMs on GIVEN HOST
hypervisor_conn = None
try:
hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" + host_ip + "/system")
rrd_logger.debug(hypervisor_conn.getHostname())
active_dom_ids = hypervisor_conn.listDomainsID()
rrd_logger.info(active_dom_ids)
all_dom_objs = hypervisor_conn.listAllDomains()
rrd_logger.info(all_dom_objs)
for dom_obj in all_dom_objs:
try:
rrd_logger.info("Starting rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
update_vm_rrd(dom_obj, active_dom_ids, host_ip)
except Exception, e:
rrd_logger.debug(e)
rrd_logger.debug("Error occured while creating/updating rrd for VM : %s" % dom_obj.name())
finally:
rrd_logger.info("Ending rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
rrd_logger.info("Checking temp on host"+str(type(host_ip)))
command='ipmitool sdr elist full'
try:
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
except Exception, e:
temp=0
ret=""
rrd_logger.debug("ipmitool not installed")
if str(ret).find("Inlet Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Inlet Temp"',host_ip)
if str(ret).find("Ambient Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Ambient Temp"',host_ip)
rrd_logger.info("temp stats" +str(type(temp)))
rrd_logger.info("temp stats" +str(temp))
return temp # returning temperature in degree Celsius
"""Uses IMPITOOL to get host power consumption"""
def get_host_power_usage(host_ip):
rrd_logger.info("Checking power on host"+str(host_ip))
command='ipmitool sdr elist full'
try:
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
except Exception, e:
pwr_usage=0
ret=""
rrd_logger.debug("ipmitool not installed")
if str(ret).find("System Level")!=int(-1):
def update_rrd(host_ip,m_type=None):
#UPDATE CONTROLLER AND NAT RRD
if m_type is not None:
rrd_logger.info("Startiing rrd updation for nat/controller %s" % (host_ip))
update_host_rrd(host_ip,m_type)
rrd_logger.info("Ending rrd updation for nat/controller %s" % (host_ip))
#UPDATE HOST RRD
else:
rrd_logger.info("Startiing rrd updation for host %s" % (host_ip))
update_host_rrd(host_ip)
rrd_logger.info("Ending rrd updation for host %s" % (host_ip))
rrd_logger.info("Startiing rrd updation for VMs on host %s" % (host_ip))
#Adding the call for setting stat periods for all VM's on the host
set_domain_memoryStatsperiod(host_ip)
#UPDATE RRD for ALL VMs on GIVEN HOST
hypervisor_conn = None
try:
hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" + host_ip + "/system")
rrd_logger.debug(hypervisor_conn.getHostname())
active_dom_ids = hypervisor_conn.listDomainsID()
rrd_logger.info(active_dom_ids)
all_dom_objs = hypervisor_conn.listAllDomains()
rrd_logger.info(all_dom_objs)
for dom_obj in all_dom_objs:
try:
rrd_logger.info("Starting rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
update_vm_rrd(dom_obj, active_dom_ids, host_ip)
except Exception, e:
rrd_logger.debug(e)
rrd_logger.debug("Error occured while creating/updating rrd for VM : %s" % dom_obj.name())
finally:
rrd_logger.info("Ending rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, vm_usage['cpu'], vm_usage['ram'], vm_usage['dr'], vm_usage['dw'], vm_usage['tx'], vm_usage['rx']))
else:
rrdtool.update(rrd_file, "%s:0:0:0:0:0:0" % (timestamp_now))
def set_domain_memoryStatsperiod(host_ip):
try:
hypervisor_conn = libvirt.open("qemu+ssh://root@" + host_ip + "/system")
active_dom_ids = hypervisor_conn.listDomainsID()
for dom_id in active_dom_ids:
dom_info = hypervisor_conn.lookupByID(dom_id)
dom_info.setMemoryStatsPeriod(2)
except Exception, e:
rrd_logger.debug(e)
finally:
rrd_logger.info("Ending setting memory stats periods for VM's %s" % host_ip)
if hypervisor_conn:
hypervisor_conn.close()
def update_rrd(host_ip,m_type=None):
#UPDATE CONTROLLER AND NAT RRD
if m_type is not None:
rrd_logger.info("Startiing rrd updation for nat/controller %s" % (host_ip))
update_host_rrd(host_ip,m_type)
rrd_logger.info("Ending rrd updation for nat/controller %s" % (host_ip))
#UPDATE HOST RRD
else:
def compare_rrd_data_with_threshold(rrd_file_name, thresholdcontext):
start_time = 'now - ' + str(VM_UTIL_20_DAYS * 24 * 60 * 60)
''' start_time = 'now - ' + str(60*60) '''
end_time = 'now'
rrd_file = get_rrd_file(rrd_file_name)
cpu_data = []
mem_data = []
dskr_data = []
dskw_data = []
nwr_data = []
nww_data = []
rrd_logger.info(
"inside compare_rrd_data_with_threshold function rrd file is :" +
str(rrd_file_name))
if os.path.exists(rrd_file):
rrd_ret = rrdtool.fetch(rrd_file, 'MIN', '--start', start_time,
'--end', end_time)
fld_info = rrd_ret[1]
data_info = rrd_ret[2]
cpu_idx = fld_info.index('cpu')
mem_idx = fld_info.index('ram')
dskr_idx = fld_info.index('dr')
dskw_idx = fld_info.index('dw')
nwr_idx = fld_info.index('tx')
nww_idx = fld_info.index('rx')
cpu_threshold = 'N'
read_threshold = 'N'
write_threshold = 'N'
CPUNoneIdentifier = 'Y'
NWRNoneIdentifier = 'Y'
NWWNoneIdentifier = 'Y'
for row in data_info:
if (row[cpu_idx] != None):
CPUNoneIdentifier = 'N'
if int(row[cpu_idx]) > int(thresholdcontext['CPUThreshold']):
cpu_threshold = 'Y'
if (row[nwr_idx] != None):
NWRNoneIdentifier = 'N'
if int(row[nwr_idx]) > int(thresholdcontext['ReadThreshold']):
read_threshold = 'Y'
if (row[nww_idx] != None):
NWWNoneIdentifier = 'N'
if int(row[nww_idx]) > int(thresholdcontext['WriteThreshold']):
write_threshold = 'Y'
if (cpu_threshold == 'Y' or read_threshold == 'Y'
or write_threshold == 'Y'):
rrd_logger.info(
"Info about the VM:%s,row[cpu_idx]:%s,row[nwr_idx]:%s,row[nww_idx]:%s"
%
(rrd_file_name, row[cpu_idx], row[nwr_idx], row[nww_idx]))
rrd_logger.info("Threshold is reached once.. VM:" +
str(rrd_file_name) + " is in use")
return False
## If only none values are read from the rrd .. do not send a warning email
if (CPUNoneIdentifier == 'N' and NWRNoneIdentifier == 'N'
and NWWNoneIdentifier == 'N'):
rrd_logger.info(
"Returning true to send warning email as threshold is never reached for VM:%s"
% (rrd_file_name))
return True
else:
rrd_logger.info(
"RRD capturing is not correct... returning all null values only for VM:%s"
% (rrd_file_name))
return False
rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, vm_usage['cpu'], vm_usage['ram'], vm_usage['dr'], vm_usage['dw'], vm_usage['tx'], vm_usage['rx']))
else:
rrdtool.update(rrd_file, "%s:0:0:0:0:0:0" % (timestamp_now))
def set_domain_memoryStatsperiod(host_ip):
try:
hypervisor_conn = libvirt.open("qemu+ssh://root@" + host_ip + "/system")
active_dom_ids = hypervisor_conn.listDomainsID()
for dom_id in active_dom_ids:
dom_info = hypervisor_conn.lookupByID(dom_id)
dom_info.setMemoryStatsPeriod(2)
except Exception, e:
rrd_logger.debug(e)
finally:
rrd_logger.info("Ending setting memory stats periods for VM's %s" % host_ip)
if hypervisor_conn:
hypervisor_conn.close()
def update_rrd(host_ip,m_type=None):
#UPDATE CONTROLLER AND NAT RRD
if m_type is not None:
rrd_logger.info("Startiing rrd updation for nat/controller %s" % (host_ip))
update_host_rrd(host_ip,m_type)
rrd_logger.info("Ending rrd updation for nat/controller %s" % (host_ip))
#UPDATE HOST RRD
else: