def get_host_resources_usage(host_ip,m_type=None):
    """
    Get following resource usage information from host
        - CPU
        - RAM
        - Disk
        - Network
    """
    rrd_logger.info("getting data for RRD file")
    if m_type is None:
        host_cpu_usage = get_host_cpu_usage(host_ip)
        host_disk_usage = get_host_disk_usage(host_ip)
        host_mem_usage = get_host_mem_usage(host_ip)
        host_nw_usage = get_host_nw_usage(host_ip)
    else:

        host_cpu_usage = get_host_cpu_usage(host_ip,m_type)
        host_disk_usage = get_host_disk_usage(host_ip,m_type)
        host_mem_usage = get_host_mem_usage(host_ip,m_type)
        host_nw_usage = get_host_nw_usage(host_ip,m_type)

    host_usage = {'cpu' : host_cpu_usage} #percent cpu usage
    host_usage.update({'dr' : host_disk_usage[0]*1024}) #Bytes/s
    host_usage.update({'dw' : host_disk_usage[1]*1024})
    host_usage.update({'ram' : host_mem_usage*1024}) #Bytes
    host_usage.update({'rx' : host_nw_usage[0]}) #in Bytes
    host_usage.update({'tx' : host_nw_usage[1]}) #in Bytes

    rrd_logger.info("Host %s stats:  %s" % (host_ip, host_usage))
    return host_usage
Beispiel #2
0
def compare_rrd_data_with_threshold(rrd_file_name,thresholdcontext):
     
    start_time = 'now - ' + str(VM_UTIL_20_DAYS*24*60*60)
    ''' start_time = 'now - ' + str(60*60) '''
    end_time = 'now'
    rrd_file = get_rrd_file(rrd_file_name)

    cpu_data = []
    mem_data = []
    dskr_data = []
    dskw_data = []
    nwr_data = []
    nww_data = []

    rrd_logger.info("inside compare_rrd_data_with_threshold function rrd file is :"+str(rrd_file_name))
    if os.path.exists(rrd_file):
        rrd_ret =rrdtool.fetch(rrd_file, 'MIN', '--start', start_time, '--end', end_time)

        fld_info = rrd_ret[1]
        data_info = rrd_ret[2]
        cpu_idx = fld_info.index('cpu')
        mem_idx = fld_info.index('ram')
        dskr_idx = fld_info.index('dr')
        dskw_idx = fld_info.index('dw')
        nwr_idx = fld_info.index('tx')
        nww_idx = fld_info.index('rx')

        cpu_threshold='N'
        read_threshold='N'
        write_threshold='N'
        CPUNoneIdentifier='Y'
        NWRNoneIdentifier='Y'
        NWWNoneIdentifier='Y'

        for row in data_info:
            if (row[cpu_idx] != None) : 
                CPUNoneIdentifier='N'
                if int(row[cpu_idx]) > int(thresholdcontext['CPUThreshold']) : cpu_threshold='Y'
            
            if (row[nwr_idx] != None) : 
                NWRNoneIdentifier='N'
                if int(row[nwr_idx]) > int(thresholdcontext['ReadThreshold']) : read_threshold='Y' 

            if (row[nww_idx] != None) : 
                NWWNoneIdentifier='N'
                if int(row[nww_idx]) > int(thresholdcontext['WriteThreshold']) : write_threshold='Y'
		
            if (cpu_threshold=='Y' or read_threshold=='Y' or write_threshold=='Y') :
                rrd_logger.info("Info about the VM:%s,row[cpu_idx]:%s,row[nwr_idx]:%s,row[nww_idx]:%s" %(rrd_file_name,row[cpu_idx],row[nwr_idx],row[nww_idx])) 
                rrd_logger.info("Threshold is reached once.. VM:"+str(rrd_file_name)+" is in use") 
                return False
           
        ## If only none values are read from the rrd .. do not send a warning email                  
        if(CPUNoneIdentifier=='N' and  NWRNoneIdentifier=='N' and NWWNoneIdentifier =='N'):
            rrd_logger.info("Returning true to send warning email as threshold is never reached for VM:%s" %(rrd_file_name) )
            return True
        else:
            rrd_logger.info("RRD capturing is not correct... returning all null values only for VM:%s" %(rrd_file_name) )
            return False 
def get_dom_mem_usage(dom, host):
    """
    Captures memory utilization of a VM from the host.
    VMs run on host as individual processes. Memory utilization of the process is derived.
    """
    rrd_logger.info("memory stats for VM:%s is %s" %(dom.name(),dom.memoryStats()))
    domain_memUsed=(dom.memoryStats()['available']-dom.memoryStats()['unused'])
    rrd_logger.info("domain_memUsed is : "+str(domain_memUsed))
    return (domain_memUsed*1024)
Beispiel #4
0
def get_host_power_usage(host_ip):
    rrd_logger.info("Checking power on host"+str(host_ip))
    command='ipmitool sdr elist full'
    ret=execute_remote_cmd(host_ip, 'root', command, None,  True)
    if str(ret).find("System Level")!=int(-1):
        pwr_usage=get_impi_output('ipmitool sdr elist full | grep "System Level"',host_ip)
    if str(ret).find("Pwr Consumption")!=int(-1):
        pwr_usage=get_impi_output('ipmitool sdr elist full | grep "Pwr Consumption"',host_ip)
    rrd_logger.info("power stats of host %s is %s" % (host_ip, str(pwr_usage)))
    return pwr_usage # returining power in Watt
def task_rrd():
    list_host=[]
    ip_cmd="/sbin/ifconfig | grep 'inet addr' | sed -n '1p' | awk '{print $2}' | cut -d ':' -f 2"
    controller_ip = execute_remote_cmd("localhost", 'root', ip_cmd, None,  True).strip()
    rrd_logger.info(controller_ip[0].strip("\n"))
    nat_ip=execute_remote_cmd("localhost", 'root', ip_cmd, None,  True).strip()
    list_host.append(controller_ip)
    list_host.append(nat_ip)
    for ip in list_host:
        m_type="controller" if controller_ip==ip else "nat"
    vm_utilization_rrd(ip,m_type)
Beispiel #6
0
def get_host_disk_usage(host_ip,m_type=None):

    command = "iostat -d | sed '1,2d'"
    
    if m_type=="controller":
        command_output =execute_remote_cmd("localhost", 'root', command, None,  True)
    
    else:
        command_output = execute_remote_cmd(host_ip, 'root', command, None,  True)
    disk_stats = re.split('\s+', command_output[1])
    rrd_logger.info("Disk stats of host %s is dr: %s dw: %s" % (host_ip, disk_stats[2], disk_stats[3]))  
    return [float(disk_stats[2]), float(disk_stats[3])] #return memory in KB/sec
Beispiel #7
0
def get_host_mem_usage(host_ip,m_type=None):

    command = "free -k | grep 'buffers/cache' | awk '{print $3}'"
    if m_type=="controller":
        command_output =execute_remote_cmd("localhost", 'root', command, None,  True)
   
    else:
        command_output = execute_remote_cmd(host_ip, 'root', command, None,  True)
    rrd_logger.info("command_output is %s"%(command_output[0]))
    used_mem_in_kb = int(command_output[0])
    rrd_logger.info("Mem stats of host %s is %s" % (host_ip, used_mem_in_kb))
    return used_mem_in_kb #return memory in KB
Beispiel #8
0
def get_host_nw_usage(host_ip,m_type=None):

    command = "ifconfig baadal-br-int | grep 'RX bytes:'"
    if m_type=="controller":
        command_output =execute_remote_cmd("localhost", 'root', command, None,  True)
    
    else:
        command_output = execute_remote_cmd(host_ip, 'root', command, None,  True)
    nw_stats = re.split('\s+', command_output[0])
    rx = int(re.split(':', nw_stats[2])[1])
    tx = int(re.split(':', nw_stats[6])[1])
    rrd_logger.info("Network stats of host %s is rx: %s tx: %s" % (host_ip, rx, tx))
    return [rx, tx] # return network in Bytes
Beispiel #9
0
def get_host_resources_usage(host_ip,m_type=None):
    
    rrd_logger.info("getting data for RRD file")
    if m_type is None:
        host_cpu_usage = get_host_cpu_usage(host_ip)
        host_disk_usage = get_host_disk_usage(host_ip)
        host_mem_usage = get_host_mem_usage(host_ip)
        host_nw_usage = get_host_nw_usage(host_ip)
        host_temp_usage =get_host_temp_usage(host_ip)
        host_power_usage =get_host_power_usage(host_ip)
	rrd_logger.info("host_cpu_usage"+str(type(host_cpu_usage)))
        rrd_logger.info("host_temp_usage"+str(type(host_temp_usage)))
    else:

        host_cpu_usage = get_host_cpu_usage(host_ip,m_type)
        host_disk_usage = get_host_disk_usage(host_ip,m_type)
        host_mem_usage = get_host_mem_usage(host_ip,m_type)
        host_nw_usage = get_host_nw_usage(host_ip,m_type)
	
    host_usage = {'cpu' : host_cpu_usage} #percent cpu usage
    host_usage.update({'dr' : host_disk_usage[0]*1024}) #Bytes/s
    host_usage.update({'dw' : host_disk_usage[1]*1024})
    host_usage.update({'ram' : host_mem_usage*1024}) #Bytes
    host_usage.update({'rx' : host_nw_usage[0]}) #in Bytes
    host_usage.update({'tx' : host_nw_usage[1]}) #in Bytes
    if m_type is None:
	host_usage.update({'tmp' : host_temp_usage}) #in Degree Celsius
        host_usage.update({'pwr' : host_power_usage}) #in Watt
        
    rrd_logger.info("Host %s stats:  %s" % (host_ip, host_usage))
    
    return host_usage
Beispiel #10
0
def get_dom_nw_usage(dom_obj):

    tree = ElementTree.fromstring(dom_obj.XMLDesc(0))
    rx = 0
    tx = 0

    for target in tree.findall("devices/interface/target"):
        device = target.get("dev")
        stats  = dom_obj.interfaceStats(device)
        rx   += stats[0]
        tx   += stats[4]

    rrd_logger.info("%s%s" % (rx, tx))

    return [rx, tx] #returned value in Bytes by default
def task_rrd():
    list_host=[]
    controller_ip=controller_ip=os.popen("ifconfig | grep 'inet addr' | sed -n '1p' | awk '{print $2}' | cut -d ':' -f 2").readlines()
    rrd_logger.info(controller_ip[0].strip("\n"))
    controller_ip=controller_ip[0].strip("\n")
    nat_ip=os.popen("route -n | sed -n '3p' | awk '{print $2}'").readlines()
    nat_ip=nat_ip[0].strip("\n")
    list_host.append(controller_ip)
    list_host.append(nat_ip)
    for ip in list_host:
        if controller_ip==ip:
	    m_type="controller"    
        else:
	    m_type="nat"  
	vm_utilization_rrd(ip,m_type)
def get_dom_nw_usage(dom_obj):
    """
    Uses libvirt function to extract interface device statistics for a domain
    to find network usage of virtual machine.
    """
    tree = ElementTree.fromstring(dom_obj.XMLDesc(0))
    rx = 0
    tx = 0

    for target in tree.findall("devices/interface/target"):
        device = target.get("dev")
        stats  = dom_obj.interfaceStats(device)
        rx   += stats[0]
        tx   += stats[4]

    rrd_logger.info("%s%s" % (rx, tx))

    return [rx, tx] #returned value in Bytes by default
Beispiel #13
0
def get_dom_disk_usage(dom_obj):

    tree = ElementTree.fromstring(dom_obj.XMLDesc(0))
    bytesr = 0
    bytesw = 0
    rreq  = 0
    wreq  = 0

    for target in tree.findall("devices/disk/target"):

        device = target.get("dev")
        stats  = dom_obj.blockStats(device)
        rreq   += stats[0]
        bytesr += stats[1]
        wreq   += stats[2]
        bytesw += stats[3]

    rrd_logger.info("rreq: %s bytesr: %s wreq: %s bytesw: %s" % (rreq, bytesr, wreq, bytesw))

    return [bytesr, bytesw] #returned value in Bytes by default
Beispiel #14
0
def get_current_dom_resource_usage(dom, host_ip):

    dom_memusage   = get_dom_mem_usage(dom, host_ip)
    dom_nw_usage   = get_dom_nw_usage(dom)
    dom_disk_usage = get_dom_disk_usage(dom)
    timestamp_now = time.time()   #keerti

    dom_stats =      {'rx'     : dom_nw_usage[0]}
    dom_stats.update({'tx'     : dom_nw_usage[1]})
    dom_stats.update({'diskr'   : dom_disk_usage[0]})
    dom_stats.update({'diskw'   : dom_disk_usage[1]})
    dom_stats.update({'memory'  : dom_memusage})
    dom_stats.update({'cputime' : dom.info()[4]})
    dom_stats.update({'cpus'    : dom.info()[3]})
    dom_stats.update({'timestamp': timestamp_now})  

    rrd_logger.info(dom_stats)
    rrd_logger.warn("As we get VM mem usage info from rrs size of the process running on host therefore it is observed that the memused is sometimes greater than max mem specified in case when the VM uses memory near to its mam memory")

    return dom_stats
def update_host_rrd(host_ip,m_type=None):
    """
    Updates the RRD file for host with resource utilization data
    """
    try:
   
        rrd_file = _get_rrd_file(host_ip.replace(".","_"))
        rrd_logger.info(rrd_file)
        timestamp_now = time.time()
        rrd_logger.info(timestamp_now)

        if not (os.path.exists(rrd_file)):

            rrd_logger.warn("RRD file (%s) does not exists" % (rrd_file))
            rrd_logger.warn("Creating new RRD file")
            create_rrd(rrd_file)
       
        else:
            rrd_logger.info("updating  RRD file")
            if m_type is None:
                host_stats = get_host_resources_usage(host_ip)
            else:
                host_stats = get_host_resources_usage(host_ip,m_type)
                
            rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, host_stats['cpu'], host_stats['ram'], host_stats['dr'], host_stats['dw'], host_stats['tx'], host_stats['rx']))
 
    except Exception, e:
 
        rrd_logger.debug("Error occured while creating/updating rrd for host: %s" % (host_ip))
        rrd_logger.debug(e)
Beispiel #16
0
def update_host_rrd(host_ip,m_type=None):
    
    try:
   
        rrd_file = get_rrd_file(host_ip.replace(".","_"))
        rrd_logger.info(rrd_file)
        timestamp_now = time.time()
        rrd_logger.info(timestamp_now)

        if not (os.path.exists(rrd_file)):

            rrd_logger.warn("RRD file (%s) does not exists" % (rrd_file))
            rrd_logger.warn("Creating new RRD file")
	    if m_type is None:
                create_rrd(rrd_file,"host")
	    else:
		create_rrd(rrd_file)
       
        else:
            rrd_logger.info("updating  RRD file")
            if m_type is None:
		rrd_logger.debug("host_ip is"+ str(host_ip))
                host_stats = get_host_resources_usage(host_ip)
                output=rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, host_stats['cpu'], host_stats['ram'], host_stats['dr'], host_stats['dw'], host_stats['tx'], host_stats['rx'],host_stats['tmp'], host_stats['pwr']))
		rrd_logger.debug("update status"+str(output))
            else:
                host_stats = get_host_resources_usage(host_ip,m_type)
                
                rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, host_stats['cpu'], host_stats['ram'], host_stats['dr'], host_stats['dw'], host_stats['tx'], host_stats['rx']))
	    
 
    except Exception, e:
 
        rrd_logger.debug("Error occured while creating/updating rrd for host: %s" % (host_ip))
        rrd_logger.debug(e)
def get_host_cpu_usage(host_ip,m_type=None):
    """
    Uses iostat tool to capture CPU statistics of host
    """
    rrd_logger.info("getting cpu info")
    command = "cat /proc/stat | awk 'FNR == 1{print $2+$3+$4}'"
    
    if m_type=="controller":
        command_output =execute_remote_cmd("localhost", 'root', command, None,  True)
    else:
        command_output = execute_remote_cmd(host_ip, 'root', command, None,  True)
    rrd_logger.debug(command_output[0])
    timestamp_now = time.time()
   
    cpu_stats = {'cputicks'    :  float(command_output[0])}     # (cpu time in clock ticks
    cpu_stats.update({'timestamp'    :  timestamp_now})
 
    prev_cpu_stats = current.cache.disk(str(host_ip), lambda:cpu_stats, 86400)  # @UndefinedVariable
    rrd_logger.debug(prev_cpu_stats)
    timestamp = float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp'])
    rrd_logger.debug("timestamp %s" % str(timestamp))
 
    #cpu_usage = cpu_stats - prev_cpu_stats # cpu usage in last 5 min (cputime-in ticks)
    if timestamp == 0:
        cpu_usage = float(cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])
    else:
        cpu_usage = float((cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])*300) / float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp']) # keerti uncomment above line and comment it
    
    current.cache.disk.clear(str(host_ip))  # @UndefinedVariable

    """Cache the current CPU utilization, so that difference can be calculated in next instance"""
    latest_cpu_stats = current.cache.disk(str(host_ip), lambda:cpu_stats, 86400)        # @UndefinedVariable
    rrd_logger.debug(latest_cpu_stats)


    clock_ticks = os.sysconf(os.sysconf_names['SC_CLK_TCK'])     # (ticks per second) 
    cpu_usage = float(cpu_usage*1000000000) / float(clock_ticks)     # (cpu time in ns)
    rrd_logger.info("CPU stats of host %s is %s" % ( host_ip, cpu_usage))
    return (cpu_usage)
def get_dom_disk_usage(dom_obj):
    """
    Uses libvirt function to extract block device statistics for a domain
    to find disk usage of virtual machine.
    """
    tree = ElementTree.fromstring(dom_obj.XMLDesc(0))
    bytesr = 0
    bytesw = 0
    rreq  = 0
    wreq  = 0

    for target in tree.findall("devices/disk/target"):

        device = target.get("dev")
        stats  = dom_obj.blockStats(device)
        rreq   += stats[0]
        bytesr += stats[1]
        wreq   += stats[2]
        bytesw += stats[3]

    rrd_logger.info("rreq: %s bytesr: %s wreq: %s bytesw: %s" % (rreq, bytesr, wreq, bytesw))

    return [bytesr, bytesw] #returned value in Bytes by default
Beispiel #19
0
def update_rrd(host_ip):

        #UPDATE HOST RRD
        rrd_logger.info("Startiing rrd updation for host %s" % (host_ip))
        update_host_rrd(host_ip)
        rrd_logger.info("Ending rrd updation for host %s" % (host_ip))


        #UPDATE RRD for ALL VMs on GIVEN HOST

        rrd_logger.info("Startiing rrd updation for VMs on host %s" % (host_ip))

        hypervisor_conn = None

        try:

            hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" + host_ip + "/system")
            rrd_logger.debug(hypervisor_conn.getHostname())

            active_dom_ids  = hypervisor_conn.listDomainsID()
            all_dom_objs    = hypervisor_conn.listAllDomains()

            for dom_obj in all_dom_objs:

                try:

                    rrd_logger.info("Starting rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
                    update_vm_rrd(dom_obj, active_dom_ids, host_ip)

                except Exception, e:
                    
                    rrd_logger.debug(e)
                    rrd_logger.debug("Error occured while creating/updating rrd for VM : %s" % dom_obj.name())
  
                finally:

                    rrd_logger.info("Ending rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
Beispiel #20
0
def get_host_cpu_usage(host_ip,m_type=None):
    rrd_logger.info("getting cpu info")
    command = "cat /proc/stat | awk 'FNR == 1{print $2+$3+$4}'"
    
    if m_type=="controller":
        command_output =execute_remote_cmd("localhost", 'root', command, None,  True)
    else:
        command_output = execute_remote_cmd(host_ip, 'root', command, None,  True)
    rrd_logger.debug(command_output[0])
    timestamp_now = time.time()
   
    cpu_stats = {'cputicks'    :  float(command_output[0])}     # (cpu time in clock ticks
    cpu_stats.update({'timestamp'    :  timestamp_now})
 
    prev_cpu_stats = current.cache.disk(str(host_ip), lambda:cpu_stats, 86400)  # @UndefinedVariable
    rrd_logger.debug(prev_cpu_stats)
    timestamp = float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp'])
    rrd_logger.debug("timestamp %s" % str(timestamp))
 
    #cpu_usage = cpu_stats - prev_cpu_stats # cpu usage in last 5 min (cputime-in ticks)
    if timestamp == 0:
        cpu_usage = float(cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])
    else:
        cpu_usage = float((cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])*300) / float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp']) # keerti uncomment above line and comment it
    
    current.cache.disk.clear(str(host_ip))  # @UndefinedVariable

    """Cache the current CPU utilization, so that difference can be calculated in next instance"""
    latest_cpu_stats = current.cache.disk(str(host_ip), lambda:cpu_stats, 86400)        # @UndefinedVariable
    rrd_logger.debug(latest_cpu_stats)


    clock_ticks = os.sysconf(os.sysconf_names['SC_CLK_TCK'])     # (ticks per second) 
    cpu_usage = float(cpu_usage*1000000000) / float(clock_ticks)     # (cpu time in ns)
    rrd_logger.info("CPU stats of host %s is %s" % ( host_ip, cpu_usage))
    return (cpu_usage)
Beispiel #21
0
def get_dom_mem_usage(dom, host):
    rrd_logger.info("memory stats for VM:%s is %s" %(dom.name(),dom.memoryStats()))
    rrd_logger.warn(dom.memoryStats())
    try:
        domain_memUsed=(dom.memoryStats()['available']-dom.memoryStats()['unused'])
        rrd_logger.info("domain_memUsed is : "+str(domain_memUsed))
    except:
        domain_memUsed=(dom.memoryStats()['rss'])
        rrd_logger.info("in except and rss value is : "+str(domain_memUsed))
    return (domain_memUsed*1024)
Beispiel #22
0
def get_dom_mem_usage(dom, host):
    rrd_logger.info("memory stats for VM:%s is %s" %(dom.name(),dom.memoryStats()))
    rrd_logger.warn(dom.memoryStats())
    try:
      domain_memUsed=(dom.memoryStats()['available']-dom.memoryStats()['unused'])
      rrd_logger.info("domain_memUsed is : "+str(domain_memUsed))
    except:
      domain_memUsed=(dom.memoryStats()['rss'])
      rrd_logger.info("in except and rss value is : "+str(domain_memUsed))
    return (domain_memUsed*1024)
Beispiel #23
0
def get_host_temp_usage(host_ip):
    rrd_logger.info("Checking temp on host"+str(type(host_ip)))
    command='ipmitool sdr elist full'
    ret=execute_remote_cmd(host_ip, 'root', command, None,  True)
    if str(ret).find("Inlet Temp")!=int(-1):
        rrd_logger.debug("Entering")
        temp=get_impi_output('ipmitool sdr elist full | grep "Inlet Temp"',host_ip)
    if str(ret).find("Ambient Temp")!=int(-1):
	rrd_logger.debug("Entering")
	temp=get_impi_output('ipmitool sdr elist full | grep "Ambient Temp"',host_ip)
    rrd_logger.info("temp stats" +str(type(temp)))
    rrd_logger.info("temp stats" +str(temp))
    return temp # returning temperature in degree Celsius
Beispiel #24
0
def get_host_temp_usage(host_ip):
    rrd_logger.info("Checking temp on host"+str(type(host_ip)))
    command='ipmitool sdr elist full'
    ret=execute_remote_cmd(host_ip, 'root', command, None,  True)
    if str(ret).find("Inlet Temp")!=int(-1):
        rrd_logger.debug("Entering")
        temp=get_impi_output('ipmitool sdr elist full | grep "Inlet Temp"',host_ip)
    if str(ret).find("Ambient Temp")!=int(-1):
        rrd_logger.debug("Entering")
        temp=get_impi_output('ipmitool sdr elist full | grep "Ambient Temp"',host_ip)
    rrd_logger.info("temp stats" +str(type(temp)))
    rrd_logger.info("temp stats" +str(temp))
    return temp # returning temperature in degree Celsius
def shutdown_nonactive_hosts(host_list):
    logger.debug(host_list)

    for host in host_list:
        host_ip = host.host_ip.private_ip
        logger.debug("host_ip %s" %host_ip)
        hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" + host_ip + "/system")
        rrd_logger.debug(hypervisor_conn.getHostname())

        active_dom_ids  = hypervisor_conn.listDomainsID()
        rrd_logger.info(active_dom_ids)
        all_dom_objs    = hypervisor_conn.listAllDomains()
        rrd_logger.info(all_dom_objs)
        rrd_logger.info(type(all_dom_objs))
        if not all_dom_objs:
            logger.debug("Host1 %s is free...Shutting it down " %str(host.host_ip.private_ip))        
Beispiel #26
0
def update_host_rrd(host_ip, m_type=None):

    try:

        rrd_file = get_rrd_file(host_ip.replace(".", "_"))
        rrd_logger.info(rrd_file)
        timestamp_now = time.time()
        rrd_logger.info(timestamp_now)

        if not (os.path.exists(rrd_file)):

            rrd_logger.warn("RRD file (%s) does not exists" % (rrd_file))
            rrd_logger.warn("Creating new RRD file")
            if m_type is None:
                create_rrd(rrd_file, "host")
            else:
                create_rrd(rrd_file)

        else:
            rrd_logger.info("updating  RRD file")
            if m_type is None:
                rrd_logger.debug("host_ip is" + str(host_ip))
                host_stats = get_host_resources_usage(host_ip)
                output = rrdtool.update(
                    rrd_file, "%s:%s:%s:%s:%s:%s:%s:%s:%s" %
                    (timestamp_now, host_stats['cpu'], host_stats['ram'],
                     host_stats['dr'], host_stats['dw'], host_stats['tx'],
                     host_stats['rx'], host_stats['tmp'], host_stats['pwr']))
                rrd_logger.debug("update status" + str(output))
            else:
                host_stats = get_host_resources_usage(host_ip, m_type)

                rrdtool.update(
                    rrd_file, "%s:%s:%s:%s:%s:%s:%s" %
                    (timestamp_now, host_stats['cpu'], host_stats['ram'],
                     host_stats['dr'], host_stats['dw'], host_stats['tx'],
                     host_stats['rx']))

    except Exception, e:

        rrd_logger.debug(
            "Error occured while creating/updating rrd for host: %s" %
            (host_ip))
        rrd_logger.debug(e)
Beispiel #27
0
def shutdown_nonactive_hosts(host_list):
    """
    Shutdown the host with no vm.
    """
    logger.debug(host_list)

    for host in host_list:
        host_ip = host.host_ip.private_ip
        logger.debug("host_ip %s" % host_ip)
        hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" + host_ip +
                                               "/system")  # @UndefinedVariable
        rrd_logger.debug(hypervisor_conn.getHostname())

        active_dom_ids = hypervisor_conn.listDomainsID()
        rrd_logger.info(active_dom_ids)
        all_dom_objs = hypervisor_conn.listAllDomains()
        rrd_logger.info(all_dom_objs)
        rrd_logger.info(type(all_dom_objs))
        if not all_dom_objs:
            logger.debug("Host1 %s is free...Shutting it down " %
                         str(host.host_ip.private_ip))
Beispiel #28
0
def update_rrd(host_ip,m_type=None):
    #UPDATE CONTROLLER AND NAT RRD
    if m_type is not None:
    
        rrd_logger.info("Startiing rrd updation for nat/controller %s" % (host_ip))
        update_host_rrd(host_ip,m_type)
        rrd_logger.info("Ending rrd updation for nat/controller %s" % (host_ip))
    
    #UPDATE HOST RRD
    else:
    
        rrd_logger.info("Startiing rrd updation for host %s" % (host_ip))
        update_host_rrd(host_ip)
        rrd_logger.info("Ending rrd updation for host %s" % (host_ip)) 


        rrd_logger.info("Startiing rrd updation for VMs on host %s" % (host_ip))
        #Adding the call for setting stat periods for all VM's on the host
        set_domain_memoryStatsperiod(host_ip)

        #UPDATE RRD for ALL VMs on GIVEN HOST
        hypervisor_conn = None
        
        try:
        
            hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" + host_ip + "/system")
            rrd_logger.debug(hypervisor_conn.getHostname())
            
            active_dom_ids  = hypervisor_conn.listDomainsID()
            rrd_logger.info(active_dom_ids)
            all_dom_objs    = hypervisor_conn.listAllDomains()
            rrd_logger.info(all_dom_objs)
            
            for dom_obj in all_dom_objs:
            
                try:
                
                    rrd_logger.info("Starting rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
                    update_vm_rrd(dom_obj, active_dom_ids, host_ip)
                
                except Exception, e:
                
                    rrd_logger.debug(e)
                    rrd_logger.debug("Error occured while creating/updating rrd for VM : %s" % dom_obj.name())
                
                finally:
                
                    rrd_logger.info("Ending rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
Beispiel #29
0
    rrd_logger.info("Checking temp on host"+str(type(host_ip)))
    command='ipmitool sdr elist full'
    try:
        ret=execute_remote_cmd(host_ip, 'root', command, None,  True)
    except Exception, e:
        temp=0
        ret=""
        rrd_logger.debug("ipmitool not installed")

    if str(ret).find("Inlet Temp")!=int(-1):
        rrd_logger.debug("Entering")
        temp=get_impi_output('ipmitool sdr elist full | grep "Inlet Temp"',host_ip)
    if str(ret).find("Ambient Temp")!=int(-1):
        rrd_logger.debug("Entering")
        temp=get_impi_output('ipmitool sdr elist full | grep "Ambient Temp"',host_ip)
    rrd_logger.info("temp stats" +str(type(temp)))
    rrd_logger.info("temp stats" +str(temp))
    return temp # returning temperature in degree Celsius

"""Uses IMPITOOL to get host power consumption"""
def get_host_power_usage(host_ip):
    rrd_logger.info("Checking power on host"+str(host_ip))
    command='ipmitool sdr elist full'
    try:
        ret=execute_remote_cmd(host_ip, 'root', command, None,  True)
    except Exception, e:
        pwr_usage=0
        ret=""
        rrd_logger.debug("ipmitool not installed")

    if str(ret).find("System Level")!=int(-1):
Beispiel #30
0
def update_rrd(host_ip,m_type=None):
    #UPDATE CONTROLLER AND NAT RRD
    if m_type is not None:
    
        rrd_logger.info("Startiing rrd updation for nat/controller %s" % (host_ip))
        update_host_rrd(host_ip,m_type)
        rrd_logger.info("Ending rrd updation for nat/controller %s" % (host_ip))
    
    #UPDATE HOST RRD
    else:
    
        rrd_logger.info("Startiing rrd updation for host %s" % (host_ip))
        update_host_rrd(host_ip)
        rrd_logger.info("Ending rrd updation for host %s" % (host_ip)) 


        rrd_logger.info("Startiing rrd updation for VMs on host %s" % (host_ip))
        #Adding the call for setting stat periods for all VM's on the host
        set_domain_memoryStatsperiod(host_ip)

        #UPDATE RRD for ALL VMs on GIVEN HOST
        hypervisor_conn = None
        
        try:
        
            hypervisor_conn = libvirt.openReadOnly("qemu+ssh://root@" + host_ip + "/system")
            rrd_logger.debug(hypervisor_conn.getHostname())
            
            active_dom_ids  = hypervisor_conn.listDomainsID()
            rrd_logger.info(active_dom_ids)
            all_dom_objs    = hypervisor_conn.listAllDomains()
            rrd_logger.info(all_dom_objs)
            
            for dom_obj in all_dom_objs:
            
                try:
                
                    rrd_logger.info("Starting rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
                    update_vm_rrd(dom_obj, active_dom_ids, host_ip)
                
                except Exception, e:
                
                    rrd_logger.debug(e)
                    rrd_logger.debug("Error occured while creating/updating rrd for VM : %s" % dom_obj.name())
                
                finally:
                
                    rrd_logger.info("Ending rrd updation for vm %s on host %s" % (dom_obj.name(), host_ip))
Beispiel #31
0
                rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, vm_usage['cpu'], vm_usage['ram'], vm_usage['dr'], vm_usage['dw'], vm_usage['tx'], vm_usage['rx']))
            else:
                rrdtool.update(rrd_file, "%s:0:0:0:0:0:0" % (timestamp_now))


def set_domain_memoryStatsperiod(host_ip):
    try:
        hypervisor_conn = libvirt.open("qemu+ssh://root@" + host_ip + "/system")
        active_dom_ids  = hypervisor_conn.listDomainsID()
        for dom_id in active_dom_ids:
            dom_info = hypervisor_conn.lookupByID(dom_id)
            dom_info.setMemoryStatsPeriod(2)
    except Exception, e:
        rrd_logger.debug(e)
    finally:
        rrd_logger.info("Ending setting memory stats periods for VM's %s" % host_ip)
        if hypervisor_conn:
            hypervisor_conn.close()


def update_rrd(host_ip,m_type=None):
    #UPDATE CONTROLLER AND NAT RRD
    if m_type is not None:
    
        rrd_logger.info("Startiing rrd updation for nat/controller %s" % (host_ip))
        update_host_rrd(host_ip,m_type)
        rrd_logger.info("Ending rrd updation for nat/controller %s" % (host_ip))
    
    #UPDATE HOST RRD
    else:
    
Beispiel #32
0
def compare_rrd_data_with_threshold(rrd_file_name, thresholdcontext):

    start_time = 'now - ' + str(VM_UTIL_20_DAYS * 24 * 60 * 60)
    ''' start_time = 'now - ' + str(60*60) '''
    end_time = 'now'
    rrd_file = get_rrd_file(rrd_file_name)

    cpu_data = []
    mem_data = []
    dskr_data = []
    dskw_data = []
    nwr_data = []
    nww_data = []

    rrd_logger.info(
        "inside compare_rrd_data_with_threshold function rrd file is :" +
        str(rrd_file_name))
    if os.path.exists(rrd_file):
        rrd_ret = rrdtool.fetch(rrd_file, 'MIN', '--start', start_time,
                                '--end', end_time)

        fld_info = rrd_ret[1]
        data_info = rrd_ret[2]
        cpu_idx = fld_info.index('cpu')
        mem_idx = fld_info.index('ram')
        dskr_idx = fld_info.index('dr')
        dskw_idx = fld_info.index('dw')
        nwr_idx = fld_info.index('tx')
        nww_idx = fld_info.index('rx')

        cpu_threshold = 'N'
        read_threshold = 'N'
        write_threshold = 'N'
        CPUNoneIdentifier = 'Y'
        NWRNoneIdentifier = 'Y'
        NWWNoneIdentifier = 'Y'

        for row in data_info:
            if (row[cpu_idx] != None):
                CPUNoneIdentifier = 'N'
                if int(row[cpu_idx]) > int(thresholdcontext['CPUThreshold']):
                    cpu_threshold = 'Y'

            if (row[nwr_idx] != None):
                NWRNoneIdentifier = 'N'
                if int(row[nwr_idx]) > int(thresholdcontext['ReadThreshold']):
                    read_threshold = 'Y'

            if (row[nww_idx] != None):
                NWWNoneIdentifier = 'N'
                if int(row[nww_idx]) > int(thresholdcontext['WriteThreshold']):
                    write_threshold = 'Y'

            if (cpu_threshold == 'Y' or read_threshold == 'Y'
                    or write_threshold == 'Y'):
                rrd_logger.info(
                    "Info about the VM:%s,row[cpu_idx]:%s,row[nwr_idx]:%s,row[nww_idx]:%s"
                    %
                    (rrd_file_name, row[cpu_idx], row[nwr_idx], row[nww_idx]))
                rrd_logger.info("Threshold is reached once.. VM:" +
                                str(rrd_file_name) + " is in use")
                return False

        ## If only none values are read from the rrd .. do not send a warning email
        if (CPUNoneIdentifier == 'N' and NWRNoneIdentifier == 'N'
                and NWWNoneIdentifier == 'N'):
            rrd_logger.info(
                "Returning true to send warning email as threshold is never reached for VM:%s"
                % (rrd_file_name))
            return True
        else:
            rrd_logger.info(
                "RRD capturing is not correct... returning all null values only for VM:%s"
                % (rrd_file_name))
            return False
Beispiel #33
0
                rrdtool.update(rrd_file, "%s:%s:%s:%s:%s:%s:%s" % (timestamp_now, vm_usage['cpu'], vm_usage['ram'], vm_usage['dr'], vm_usage['dw'], vm_usage['tx'], vm_usage['rx']))
            else:
                rrdtool.update(rrd_file, "%s:0:0:0:0:0:0" % (timestamp_now))


def set_domain_memoryStatsperiod(host_ip):
    try:
        hypervisor_conn = libvirt.open("qemu+ssh://root@" + host_ip + "/system")
        active_dom_ids  = hypervisor_conn.listDomainsID()
        for dom_id in active_dom_ids:
            dom_info = hypervisor_conn.lookupByID(dom_id)
            dom_info.setMemoryStatsPeriod(2)
    except Exception, e:
        rrd_logger.debug(e)
    finally:
        rrd_logger.info("Ending setting memory stats periods for VM's %s" % host_ip)
        if hypervisor_conn:
           hypervisor_conn.close()


def update_rrd(host_ip,m_type=None):
    #UPDATE CONTROLLER AND NAT RRD
    if m_type is not None:
    
        rrd_logger.info("Startiing rrd updation for nat/controller %s" % (host_ip))
        update_host_rrd(host_ip,m_type)
        rrd_logger.info("Ending rrd updation for nat/controller %s" % (host_ip))
    
    #UPDATE HOST RRD
    else: