def overload_memory(): logger.debug("Executing overload memory task") file_path_row = db( db.constants.name == "memory_overload_file_path").select( db.constants.value).first() file_path = file_path_row.value logger.debug(type(file_path)) host_ips_rows = db((db.host.status == HOST_STATUS_UP) & (db.host.host_ip == db.private_ip_pool.id)).select( db.private_ip_pool.private_ip) logger.debug(host_ips_rows) command2 = '/memhog >/memoryhog.out &' command3 = "ps -ef | grep memhog | grep -v grep | awk 'END{print FNR}'" for host_ip_row in host_ips_rows: logger.debug("overloading memory of" + str(host_ip_row)) logger.debug(type(host_ip_row['private_ip'])) _check_compile_folder(file_path) command1 = 'scp ' + str(file_path) + '/memhog root@' + str( host_ip_row['private_ip']) + ':/' logger.debug('executing' + command1) ret = os.system(command1) logger.debug('os.system return value' + str(ret)) output = execute_remote_cmd(host_ip_row['private_ip'], 'root', command3) ret1 = int(output[0]) if (ret1 == 0): ret = execute_remote_cmd(host_ip_row['private_ip'], 'root', command2) logger.debug(ret) logger.debug("Completed overload memory task")
def create_dhcp_bulk_entry(dhcp_info_list): if len(dhcp_info_list) == 0: return entry_cmd = "" for dhcp_info in dhcp_info_list: host_name = dhcp_info[0] if dhcp_info[0] != None else ( 'IP_' + dhcp_info[2].replace(".", '_')) dhcp_cmd = ''' file_name="/etc/dhcp/dhcp.d/1_%s.conf" if [ -e "$file_name" ] then echo $file_name else echo "host %s {\n\thardware ethernet %s;\n\tfixed-address %s;\n}\n" > $file_name fi ''' % (host_name, host_name, dhcp_info[1], dhcp_info[2]) entry_cmd += dhcp_cmd restart_cmd = ''' cat /etc/dhcp/dhcp.d/*.conf > /etc/dhcp/dhcpd.conf /etc/init.d/isc-dhcp-server restart ''' execute_remote_cmd(dhcp_ip, 'root', entry_cmd) execute_remote_cmd(dhcp_ip, 'root', restart_cmd)
def host_power_down(host_data): try: host_ip = host_data.host_ip.private_ip if host_data.host_type == HOST_TYPE_VIRTUAL: output = execute_remote_cmd(host_ip, 'root', 'virsh destroy' + get_host_name[host_ip]) else: setup_type = config.get("GENERAL_CONF","setup_type") if setup_type == "nic": ucs_management_ip = config.get("UCS_MANAGER_DETAILS","ucs_management_ip") logger.debug(ucs_management_ip) logger.debug(type(ucs_management_ip)) ucs_user = config.get("UCS_MANAGER_DETAILS","ucs_user") logger.debug(ucs_user) logger.debug(type(ucs_user)) ucs_password = config.get("UCS_MANAGER_DETAILS","ucs_password") logger.debug(ucs_password) host_ip=str(host_ip) server_num=host_ip.split('.') ucs_server_num=str(int(server_num[3])-20) logger.debug("ucs server number is :"+ucs_server_num) ssh = paramiko.SSHClient() ssh.load_system_host_keys() ssh.connect(ucs_management_ip,username=ucs_user,password=ucs_password) stdin, stdout,stderr=ssh.exec_command("scope org / ; scope org IIT-Delhi ; scope service-profile Badal-Host"+ str(ucs_server_num) + " ; power down ; commit-buffer") # @UnusedVariable output=stdout.readlines() if len(output)!= 0: logger.debug("Host not powered up . Command not run properly ") else: output = execute_remote_cmd(host_ip, 'root', 'init 0') logger.debug(str(output) + ' ,Host shut down successfully !!!') except: log_exception()
def create_dhcp_bulk_entry(dhcp_info_list): if len(dhcp_info_list) == 0: return entry_cmd = "" for dhcp_info in dhcp_info_list: host_name = dhcp_info[0] if dhcp_info[0] != None else ('IP_' + dhcp_info[2].replace(".", '_')) dhcp_cmd = ''' file_name="/etc/dhcp/dhcp.d/1_%s.conf" if [ -e "$file_name" ] then echo $file_name else echo "host %s {\n\thardware ethernet %s;\n\tfixed-address %s;\n}\n" > $file_name fi ''' %(host_name, host_name, dhcp_info[1], dhcp_info[2]) entry_cmd += dhcp_cmd restart_cmd = ''' cat /etc/dhcp/dhcp.d/*.conf > /etc/dhcp/dhcpd.conf /etc/init.d/isc-dhcp-server restart ''' execute_remote_cmd(dhcp_ip, 'root', entry_cmd) execute_remote_cmd(dhcp_ip, 'root', restart_cmd)
def show_cont_performance(): ip_cmd="/sbin/ifconfig | grep 'inet addr' | sed -n '1p' | awk '{print $2}' | cut -d ':' -f 2" cpu_cmd="nproc" mem_cmd = "free -m | grep 'Mem:' | awk '{print $2}'" cont_cpu=execute_remote_cmd("localhost", 'root', cpu_cmd, None, True).strip() cont_ram=execute_remote_cmd("localhost", 'root', mem_cmd, None, True).strip() cont_ip=execute_remote_cmd("localhost", 'root', ip_cmd, None, True).replace(".","_").strip() return dict(host_identity=cont_ip ,host_ram=cont_ram, m_type="host",host_cpu=cont_cpu)
def show_nat_performance(): ip_cmd="/sbin/route -n | sed -n '3p' | awk '{print $2}'" cpu_cmd="nproc" mem_cmd = "free -m | grep 'Mem:' | awk '{print $2}'" nat_ip=execute_remote_cmd("localhost", 'root', ip_cmd, None, True).strip() nat_cpu=execute_remote_cmd(nat_ip, 'root', cpu_cmd, None, True)[0].strip() nat_ram=execute_remote_cmd(nat_ip, 'root', mem_cmd, None, True)[0].strip() nat_identity=nat_ip.replace(".","_") return dict(host_identity= nat_identity ,host_ram=nat_ram, m_type="host",host_cpu=nat_cpu)
def task_rrd(): list_host=[] cont_ip_cmd="/sbin/ifconfig | grep 'inet addr' | sed -n '1p' | awk '{print $2}' | cut -d ':' -f 2" nat_ip_cmd="/sbin/route -n | sed -n '3p' | awk '{print $2}'" cont_ip=execute_remote_cmd("localhost", 'root',cont_ip_cmd, None, True).strip() nat_ip=execute_remote_cmd("localhost", 'root',nat_ip_cmd, None, True).strip() list_host.append(cont_ip) list_host.append(nat_ip) for ip in list_host: m_type="controller"if cont_ip==ip else "nat" vm_utilization_rrd(ip,m_type)
def task_rrd(): list_host=[] ip_cmd="/sbin/ifconfig | grep 'inet addr' | sed -n '1p' | awk '{print $2}' | cut -d ':' -f 2" controller_ip = execute_remote_cmd("localhost", 'root', ip_cmd, None, True).strip() rrd_logger.info(controller_ip[0].strip("\n")) nat_ip=execute_remote_cmd("localhost", 'root', ip_cmd, None, True).strip() list_host.append(controller_ip) list_host.append(nat_ip) for ip in list_host: m_type="controller" if controller_ip==ip else "nat" vm_utilization_rrd(ip,m_type)
def get_host_mem_usage(host_ip,m_type=None): command = "free -k | grep 'buffers/cache' | awk '{print $3}'" if m_type=="controller": command_output =execute_remote_cmd("localhost", 'root', command, None, True) else: command_output = execute_remote_cmd(host_ip, 'root', command, None, True) rrd_logger.info("command_output is %s"%(command_output[0])) used_mem_in_kb = int(command_output[0]) rrd_logger.info("Mem stats of host %s is %s" % (host_ip, used_mem_in_kb)) return used_mem_in_kb #return memory in KB
def get_host_disk_usage(host_ip,m_type=None): command = "iostat -d | sed '1,2d'" if m_type=="controller": command_output =execute_remote_cmd("localhost", 'root', command, None, True) else: command_output = execute_remote_cmd(host_ip, 'root', command, None, True) disk_stats = re.split('\s+', command_output[1]) rrd_logger.info("Disk stats of host %s is dr: %s dw: %s" % (host_ip, disk_stats[2], disk_stats[3])) return [float(disk_stats[2]), float(disk_stats[3])] #return memory in KB/sec
def remove_dhcp_entry(host_name, ip_addr): host_name = host_name if host_name != None else ('IP_' + ip_addr.replace(".", '_')) entry_cmd = "rm /etc/dhcp/dhcp.d/1_%s.conf" %(host_name) restart_cmd = ''' cat /etc/dhcp/dhcp.d/*.conf > /etc/dhcp/dhcpd.conf /etc/init.d/isc-dhcp-server restart ''' execute_remote_cmd(dhcp_ip, 'root', entry_cmd) execute_remote_cmd(dhcp_ip, 'root', restart_cmd)
def get_host_nw_usage(host_ip,m_type=None): command = "ifconfig baadal-br-int | grep 'RX bytes:'" if m_type=="controller": command_output =execute_remote_cmd("localhost", 'root', command, None, True) else: command_output = execute_remote_cmd(host_ip, 'root', command, None, True) nw_stats = re.split('\s+', command_output[0]) rx = int(re.split(':', nw_stats[2])[1]) tx = int(re.split(':', nw_stats[6])[1]) rrd_logger.info("Network stats of host %s is rx: %s tx: %s" % (host_ip, rx, tx)) return [rx, tx] # return network in Bytes
def remove_dhcp_entry(host_name, ip_addr): host_name = host_name if host_name != None else ('IP_' + ip_addr.replace(".", '_')) entry_cmd = "rm /etc/dhcp/dhcp.d/1_%s.conf" % (host_name) restart_cmd = ''' cat /etc/dhcp/dhcp.d/*.conf > /etc/dhcp/dhcpd.conf /etc/init.d/isc-dhcp-server restart ''' execute_remote_cmd(dhcp_ip, 'root', entry_cmd) execute_remote_cmd(dhcp_ip, 'root', restart_cmd)
def task_rrd(): list_host=[] cont_ip_cmd="/sbin/ifconfig | grep 'inet addr' | sed -n '1p' | awk '{print $2}' | cut -d ':' -f 2" nat_ip_cmd="/sbin/route -n | sed -n '3p' | awk '{print $2}'" cont_ip=execute_remote_cmd("localhost", 'root',cont_ip_cmd, None, True).strip() nat_ip=execute_remote_cmd("localhost", 'root',nat_ip_cmd, None, True).strip() list_host.append(cont_ip) list_host.append(nat_ip) for ip in list_host: if cont_ip==ip: m_type="controller" else: m_type="nat" vm_utilization_rrd(ip,m_type)
def show_cont_performance(): ip_cmd = "/sbin/ifconfig | grep 'inet addr' | sed -n '1p' | awk '{print $2}' | cut -d ':' -f 2" cpu_cmd = "nproc" mem_cmd = "free -m | grep 'Mem:' | awk '{print $2}'" cont_cpu = execute_remote_cmd("localhost", 'root', cpu_cmd, None, True).strip() cont_ram = execute_remote_cmd("localhost", 'root', mem_cmd, None, True).strip() cont_ip = execute_remote_cmd("localhost", 'root', ip_cmd, None, True).replace(".", "_").strip() return dict(host_identity=cont_ip, host_ram=cont_ram, m_type="host", host_cpu=cont_cpu)
def check_host_service_status(host_ip): #Check libvirt status command = "ps -ef | grep libvirtd | grep -v grep | wc -l" ret = execute_remote_cmd(host_ip, 'root',command) if ret == 0 : logger.error("Critical: Libvirt service is not running on host " + host_ip) return False #Check OVS status command = "service openvswitch-switch status | grep -w 'running' | wc -l" ret = execute_remote_cmd(host_ip, 'root',command) if ret == 0 : logger.error("Critical: OVS switch is not running on host " + host_ip) return False return True
def grant_novnc_access(vm_id): msg = "" active_vnc = db((db.vnc_access.vm_id == vm_id) & ( db.vnc_access.status == VNC_ACCESS_STATUS_ACTIVE)).count() if active_vnc > 0: vm_data = db(db.vnc_access.vm_id == vm_id).select().first() token = vm_data.token msg = 'VNC access already granted. Please check your mail for further details.' else: vnc_count = db((db.vnc_access.vm_id == vm_id) & (db.vnc_access.time_requested > (get_datetime() - timedelta(days=1)))).count() if vnc_count >= MAX_VNC_ALLOWED_IN_A_DAY: msg = 'VNC request has exceeded limit.' else: try: f = os.popen('openssl rand -hex 10') token = f.read() token = token.split("\n") token = token[0] create_novnc_mapping(vm_id, token) vm_data = db(db.vm_data.id == vm_id).select().first() host_ip = vm_data.host_id.host_ip.private_ip vnc_port = vm_data.vnc_port vnc = str(vnc_port) file_token = str(token) + ":" + " " + str(host_ip) + ":" + str( vnc) + "\n" myfile = get_file_append_mode("/home/www-data/token.list") myfile.write(file_token) command = "ps -ef | grep websockify|awk '{print $2}'" port = config.get("NOVNC_CONF", "port") server_ip = config.get("NOVNC_CONF", "server_ip") return_value = execute_remote_cmd(server_ip, 'root', command) return_value = return_value.split() if len(return_value) <= 2: command = "./noVNC/utils/websockify/run --web /root/noVNC --target-config /home/www-data/token.list " + str( server_ip) + ":" + str(port) + " > /dev/null 2>&1 &" return_value = execute_remote_cmd(server_ip, 'root', command) msg = 'VNC access granted. Please check your mail for further details.' except: logger.debug('Some Error Occurred. Please try later') log_exception() pass logger.debug(msg) return token
def show_nat_performance(): ip_cmd = "/sbin/route -n | sed -n '3p' | awk '{print $2}'" cpu_cmd = "nproc" mem_cmd = "free -m | grep 'Mem:' | awk '{print $2}'" nat_ip = execute_remote_cmd("localhost", 'root', ip_cmd, None, True).strip() nat_cpu = execute_remote_cmd(nat_ip, 'root', cpu_cmd, None, True)[0].strip() nat_ram = execute_remote_cmd(nat_ip, 'root', mem_cmd, None, True)[0].strip() nat_identity = nat_ip.replace(".", "_") return dict(host_identity=nat_identity, host_ram=nat_ram, m_type="host", host_cpu=nat_cpu)
def add_orphan_vm(vm_name, host_id): """ Add Orphan VM information to database. VM information is retrieved from the VM definition XML. 'System User' is added as owner of the VM. """ host_details = db.host[host_id] host_ip = host_details.host_ip.private_ip connection_object = libvirt.openReadOnly("qemu+ssh://root@" + host_ip + "/system") domain = connection_object.lookupByName(vm_name) vm_state = domain.info()[0] vm_status = vm_state_map[vm_state] # Parse domain XML to get information about VM root = etree.fromstring(domain.XMLDesc(0)) ram_elem = root.xpath('memory')[0] ram_in_kb = int(ram_elem.text) ram_in_mb = int(round(int(ram_in_kb)/(1024),0)) cpu_elem = root.xpath('vcpu')[0] cpu = int(cpu_elem.text) vnc_elem = root.xpath("devices/graphics[@type='vnc']")[0] vnc_port = vnc_elem.attrib['port'] mac_elem = root.xpath("devices/interface[@type='network']/mac")[0] mac_address = mac_elem.attrib['address'] ip_addr = db.private_ip_pool(mac_addr = mac_address) ip_address = None if ip_addr: ip_address = ip_addr['id'] template_elem = root.xpath("devices/disk[@type='file']/source")[0] template_file = template_elem.attrib['file'] command = "qemu-img info " + template_file + " | grep 'virtual size'" ret = execute_remote_cmd(host_ip, 'root', command) # Returns e.g. virtual size: 40G (42949672960 bytes) hdd = int(ret[ret.index(':')+1:ret.index('G ')].strip()) security_domain_row = db.security_domain(vlan=ip_addr['vlan']) db.vm_data.insert( vm_name = vm_name, vm_identity = (vm_name), RAM = ram_in_mb, HDD = hdd, extra_HDD = 0, vCPU = cpu, host_id = host_id, template_id = 1, #TBD datastore_id = 1, #TBD owner_id = SYSTEM_USER, requester_id = SYSTEM_USER, private_ip = ip_address, vnc_port = vnc_port, purpose = 'Added by System', security_domain = security_domain_row['id'], status = vm_status) return
def remove_dhcp_entry(host_name, mac_addr, ip_addr): host_name = host_name if host_name != None else ('IP_' + ip_addr.replace(".", '_')) if mac_addr != None: entry_cmd = "sed -i '/host.*%s.*{/ {N;N;N; s/host.*%s.*{.*hardware.*ethernet.*%s;.*fixed-address.*%s;.*}//g}' /etc/dhcp/dhcpd.conf" % ( host_name, host_name, mac_addr, ip_addr) else: entry_cmd = "sed -i '/host.*%s.*{/ {N;N;N; s/host.*%s.*{.*}//g}' /etc/dhcp/dhcpd.conf" % ( host_name, host_name) restart_cmd = "/etc/init.d/isc-dhcp-server restart" execute_remote_cmd(dhcp_ip, 'root', entry_cmd) execute_remote_cmd(dhcp_ip, 'root', restart_cmd)
def get_host_disk_usage(host_ip): command = "iostat -d | sed '1,2d'" command_output = execute_remote_cmd(host_ip, 'root', command, None, True) disk_stats = re.split('\s+', command_output[1]) rrd_logger.info("Disk stats of host %s is dr: %s dw: %s" % (host_ip, disk_stats[2], disk_stats[3])) return [float(disk_stats[2]), float(disk_stats[3])]
def add_orphan_vm(vm_name, host_id): host_details = db.host[host_id] host_ip = host_details.host_ip.private_ip connection_object = libvirt.openReadOnly("qemu+ssh://root@" + host_ip + "/system") domain = connection_object.lookupByName(vm_name) vm_state = domain.info()[0] vm_status = vm_state_map[vm_state] # Parse domain XML to get information about VM root = etree.fromstring(domain.XMLDesc(0)) ram_elem = root.xpath('memory')[0] ram_in_kb = int(ram_elem.text) ram_in_mb = int(round(int(ram_in_kb) / (1024), 0)) cpu_elem = root.xpath('vcpu')[0] cpu = int(cpu_elem.text) vnc_elem = root.xpath("devices/graphics[@type='vnc']")[0] vnc_port = vnc_elem.attrib['port'] mac_elem = root.xpath("devices/interface[@type='network']/mac")[0] mac_address = mac_elem.attrib['address'] ip_addr = db.private_ip_pool(mac_addr=mac_address) ip_address = None if ip_addr: ip_address = ip_addr['id'] template_elem = root.xpath("devices/disk[@type='file']/source")[0] template_file = template_elem.attrib['file'] command = "qemu-img info " + template_file + " | grep 'virtual size'" ret = execute_remote_cmd( host_ip, 'root', command) # Returns e.g. virtual size: 40G (42949672960 bytes) hdd = int(ret[ret.index(':') + 1:ret.index('G ')].strip()) security_domain_row = db.security_domain(vlan=ip_addr['vlan']) db.vm_data.insert( vm_name=vm_name, vm_identity=(vm_name), RAM=ram_in_mb, HDD=hdd, extra_HDD=0, vCPU=cpu, host_id=host_id, template_id=1, #TBD datastore_id=1, #TBD owner_id=SYSTEM_USER, requester_id=SYSTEM_USER, private_ip=ip_address, vnc_port=vnc_port, purpose='Added by System', security_domain=security_domain_row['id'], status=vm_status) return
def request_vpn(): user_info = get_vpn_user_details() logger.debug(type(user_info)) user_name = user_info['username'] cmd = "./vpn_client_creation.sh " + str(user_name) #vpn_ip="" vpn_ip = config.get("VPN_CONF", "vpn_server_ip") vpn_key_path = config.get("VPN_CONF", "vpn_key_path") #passwd="" # password=config.get("VPN_CONF","passwd") try: var = execute_remote_cmd(vpn_ip, 'root', cmd, ret_list=True) filepath = vpn_key_path + str(user_name) + "_baadalVPN.tar" localpath = os.path.join( get_context_path(), 'private/VPN/' + str(user_name) + "_baadalVPN.tar") sftp_files(vpn_ip, 'root', filepath, localpath) if "false" in str(var): return 1 elif "true" in str(var): return 3 #transfer_vpn_files(user_name,vpn_ip,password) except Exception: return 2
def get_host_hdd(host_ip): command = "fdisk -l | egrep 'Disk.*bytes' | awk '{ sub(/,/,\"\"); sum +=$3;} END {print sum}'" ret = execute_remote_cmd(host_ip, 'root',command)#Returns e.g. 500.1 kB logger.debug("Host HDD is %s" %ret) hdd_in_gb = int(math.ceil(float(ret))) return hdd_in_gb
def get_dom_mem_usage(dom_name, host): rrd_logger.debug("Fetching memory usage of domain %s defined on host %s" % (dom_name, host)) cmd = "output=`ps -ef --sort=start_time | grep '%s.qcow2' | grep -v grep | awk '{print $2}'`;smem -c 'pid pss'| grep $output | awk '{print $2}'" % dom_name #"ps aux | grep '\-name " + dom_name + " ' | grep kvm" output = execute_remote_cmd(host, "root", cmd, None, True) return (int(output[0]))*1024 #return memory in Bytes by default
def get_host_mem_usage(host_ip): command = "top -b -n1 | grep 'Mem'" command_output = execute_remote_cmd(host_ip, 'root', command, None, True) mem_stats = re.split('\s+', command_output[0])[3] used_mem_in_kb = int(mem_stats[:-1]) rrd_logger.info("Mem stats of host %s is %s" % (host_ip, used_mem_in_kb)) return used_mem_in_kb
def get_impi_output(cmd,host_ip): rrd_logger.debug("entering ipmi") output=execute_remote_cmd(host_ip,'root', cmd, None, True) rrd_logger.debug(output) rrd_logger.debug(output[0].split("|")[4].split(" ")[1]) ret=output[0].split("|")[4].split(" ")[1] rrd_logger.debug("cheecking ret"+str(ret)) return int(ret)
def get_bandwidth_of_host(host_ip): bandwidth_cmd='lshw | grep capacity | grep bit/s | sed -n "1 p" | tr -s " " | cut -f"3" -d" "' bandwidth=execute_remote_cmd(host_ip,'root',bandwidth_cmd) m=str(bandwidth) q=m.replace("['",' ') bandwidth=q.replace("']",' ') return bandwidth
def get_host_cpu_usage(host_ip, m_type=None): rrd_logger.info("getting cpu info") command = "cat /proc/stat | awk 'FNR == 1{print $2+$3+$4}'" if m_type == "controller": command_output = execute_remote_cmd("localhost", 'root', command, None, True) else: command_output = execute_remote_cmd(host_ip, 'root', command, None, True) rrd_logger.debug(command_output[0]) timestamp_now = time.time() cpu_stats = { 'cputicks': float(command_output[0]) } # (cpu time in clock ticks cpu_stats.update({'timestamp': timestamp_now}) prev_cpu_stats = current.cache.disk(str(host_ip), lambda: cpu_stats, 86400) # @UndefinedVariable rrd_logger.debug(prev_cpu_stats) timestamp = float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp']) rrd_logger.debug("timestamp %s" % str(timestamp)) #cpu_usage = cpu_stats - prev_cpu_stats # cpu usage in last 5 min (cputime-in ticks) if timestamp == 0: cpu_usage = float(cpu_stats['cputicks'] - prev_cpu_stats['cputicks']) else: cpu_usage = float( (cpu_stats['cputicks'] - prev_cpu_stats['cputicks']) * 300) / float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp'] ) # keerti uncomment above line and comment it current.cache.disk.clear(str(host_ip)) # @UndefinedVariable """Cache the current CPU utilization, so that difference can be calculated in next instance""" latest_cpu_stats = current.cache.disk(str(host_ip), lambda: cpu_stats, 86400) # @UndefinedVariable rrd_logger.debug(latest_cpu_stats) clock_ticks = os.sysconf( os.sysconf_names['SC_CLK_TCK']) # (ticks per second) cpu_usage = float(cpu_usage * 1000000000) / float( clock_ticks) # (cpu time in ns) rrd_logger.info("CPU stats of host %s is %s" % (host_ip, cpu_usage)) return (cpu_usage)
def get_host_nw_usage(host_ip): command = "ifconfig baadal-br-int | grep 'RX bytes:'" command_output = execute_remote_cmd(host_ip, 'root', command, None, True) nw_stats = re.split('\s+', command_output[0]) rx = int(re.split(':', nw_stats[2])[1]) tx = int(re.split(':', nw_stats[6])[1]) rrd_logger.info("Disk stats of host %s is rx: %s tx: %s" % (host_ip, rx, tx)) return [rx, tx]
def get_memhog_usage(host_ip): logger.debug("Entering into getmemhog_usage") cmd = "output=`ps -ef --sort=start_time | grep 'memhog' | grep -v grep | awk '{print $2}'`;smem -c 'pid pss'| grep $output | awk '{print $2}'" output = execute_remote_cmd(host_ip, "root", cmd, None, True) if not output : return 0 else: #logger.debug("For host:"+str(host_ip)+" PSS Value from memhog="+str(output)) return (int(output[0])*1024)
def get_host_cpu_usage(host_ip): command = "iostat -c | sed '1,2d'" command_output = execute_remote_cmd(host_ip, 'root', command, None, True) rrd_logger.debug(type(command_output)) cpu_stats = re.split('\s+', command_output[1]) rrd_logger.debug(cpu_stats) rrd_logger.info("CPU stats of host %s is %s" % ( host_ip, (cpu_stats[1] + cpu_stats[2] + cpu_stats[3]))) return (float(cpu_stats[1]) + float(cpu_stats[2]) + float(cpu_stats[3]))
def get_host_power_usage(host_ip): rrd_logger.info("Checking power on host"+str(host_ip)) command='ipmitool sdr elist full' try: ret=execute_remote_cmd(host_ip, 'root', command, None, True) except Exception, e: pwr_usage=0 ret="" rrd_logger.debug("ipmitool not installed")
def get_host_power_usage(host_ip): rrd_logger.info("Checking power on host"+str(host_ip)) command='ipmitool sdr elist full' ret=execute_remote_cmd(host_ip, 'root', command, None, True) if str(ret).find("System Level")!=int(-1): pwr_usage=get_impi_output('ipmitool sdr elist full | grep "System Level"',host_ip) if str(ret).find("Pwr Consumption")!=int(-1): pwr_usage=get_impi_output('ipmitool sdr elist full | grep "Pwr Consumption"',host_ip) rrd_logger.info("power stats of host %s is %s" % (host_ip, str(pwr_usage))) return pwr_usage # returining power in Watt
def grant_novnc_access(vm_id): msg = "" active_vnc = db((db.vnc_access.vm_id == vm_id) & (db.vnc_access.status == VNC_ACCESS_STATUS_ACTIVE)).count() if active_vnc > 0: vm_data = db(db.vnc_access.vm_id == vm_id).select().first() token = vm_data.token msg = 'VNC access already granted. Please check your mail for further details.' else: vnc_count = db((db.vnc_access.vm_id == vm_id) & (db.vnc_access.time_requested > (get_datetime() - timedelta(days=1)))).count() if vnc_count >= MAX_VNC_ALLOWED_IN_A_DAY : msg = 'VNC request has exceeded limit.' else: try: f = os.popen('openssl rand -hex 10') token = f.read() token = token.split("\n") token=token[0] create_novnc_mapping(vm_id,token) vm_data = db(db.vm_data.id == vm_id).select().first() host_ip = vm_data.host_id.host_ip.private_ip vnc_port = vm_data.vnc_port vnc = str(vnc_port) file_token =str(token) +":" + " " + str(host_ip)+ ":" + str(vnc) + "\n" myfile=get_file_append_mode("/home/www-data/token.list") myfile.write(file_token) command = "ps -ef | grep websockify|awk '{print $2}'" port = config.get("NOVNC_CONF","port") server_ip = config.get("NOVNC_CONF","server_ip") return_value = execute_remote_cmd(server_ip, 'root',command) return_value=return_value.split() if len(return_value) <=2: command = "./noVNC/utils/websockify/run --web /root/noVNC --target-config /home/www-data/token.list " +str(server_ip)+ ":"+str(port) + " > /dev/null 2>&1 &" return_value = execute_remote_cmd(server_ip, 'root',command) msg = 'VNC access granted. Please check your mail for further details.' except: logger.debug('Some Error Occurred. Please try later') log_exception() pass logger.debug(msg) return token
def get_memhog_usage(host_ip): """ Returns memory used by memhog process so that it can be added to available memory. """ logger.debug("Entering into getmemhog_usage") cmd = "output=`ps -ef --sort=start_time | grep 'memhog' | grep -v grep | awk '{print $2}'`;smem -c 'pid pss'| grep $output | awk '{print $2}'" output = execute_remote_cmd(host_ip, "root", cmd, None, True) if not output: return 0 else: # logger.debug("For host:"+str(host_ip)+" PSS Value from memhog="+str(output)) return int(output[0]) * 1024
def get_memhog_usage(host_ip): """ Returns memory used by memhog process so that it can be added to available memory. """ logger.debug("Entering into getmemhog_usage") cmd = "output=`ps -ef --sort=start_time | grep 'memhog' | grep -v grep | awk '{print $2}'`;smem -c 'pid pss'| grep $output | awk '{print $2}'" output = execute_remote_cmd(host_ip, "root", cmd, None, True) if not output: return 0 else: #logger.debug("For host:"+str(host_ip)+" PSS Value from memhog="+str(output)) return (int(output[0]) * 1024)
def get_host_cpu_usage(host_ip,m_type=None): """ Uses iostat tool to capture CPU statistics of host """ rrd_logger.info("getting cpu info") command = "cat /proc/stat | awk 'FNR == 1{print $2+$3+$4}'" if m_type=="controller": command_output =execute_remote_cmd("localhost", 'root', command, None, True) else: command_output = execute_remote_cmd(host_ip, 'root', command, None, True) rrd_logger.debug(command_output[0]) timestamp_now = time.time() cpu_stats = {'cputicks' : float(command_output[0])} # (cpu time in clock ticks cpu_stats.update({'timestamp' : timestamp_now}) prev_cpu_stats = current.cache.disk(str(host_ip), lambda:cpu_stats, 86400) # @UndefinedVariable rrd_logger.debug(prev_cpu_stats) timestamp = float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp']) rrd_logger.debug("timestamp %s" % str(timestamp)) #cpu_usage = cpu_stats - prev_cpu_stats # cpu usage in last 5 min (cputime-in ticks) if timestamp == 0: cpu_usage = float(cpu_stats['cputicks'] - prev_cpu_stats['cputicks']) else: cpu_usage = float((cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])*300) / float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp']) # keerti uncomment above line and comment it current.cache.disk.clear(str(host_ip)) # @UndefinedVariable """Cache the current CPU utilization, so that difference can be calculated in next instance""" latest_cpu_stats = current.cache.disk(str(host_ip), lambda:cpu_stats, 86400) # @UndefinedVariable rrd_logger.debug(latest_cpu_stats) clock_ticks = os.sysconf(os.sysconf_names['SC_CLK_TCK']) # (ticks per second) cpu_usage = float(cpu_usage*1000000000) / float(clock_ticks) # (cpu time in ns) rrd_logger.info("CPU stats of host %s is %s" % ( host_ip, cpu_usage)) return (cpu_usage)
def get_host_temp_usage(host_ip): rrd_logger.info("Checking temp on host"+str(type(host_ip))) command='ipmitool sdr elist full' ret=execute_remote_cmd(host_ip, 'root', command, None, True) if str(ret).find("Inlet Temp")!=int(-1): rrd_logger.debug("Entering") temp=get_impi_output('ipmitool sdr elist full | grep "Inlet Temp"',host_ip) if str(ret).find("Ambient Temp")!=int(-1): rrd_logger.debug("Entering") temp=get_impi_output('ipmitool sdr elist full | grep "Ambient Temp"',host_ip) rrd_logger.info("temp stats" +str(type(temp))) rrd_logger.info("temp stats" +str(temp)) return temp # returning temperature in degree Celsius
def respawn_dangling_vms(host_id): vms = current.db(current.db.vm_data.host_id == host_id).select(current.db.vm_data.ALL) vm_image_location = get_constant('vmfiles_path') + get_constant('vms') + '/%s/%s.qcow2' for vm_data in vms: logger.debug('Re-spawning VM ' + vm_data.vm_identity) #Create a copy of existing image and rename it with '_old' suffix storage_type = config.get("GENERAL_CONF","storage_type") copy_command = 'ndmpcopy ' if storage_type == current.STORAGE_NETAPP_NFS else 'cp ' ds_image_location = vm_data.datastore_id.path + get_constant('vms') + '/%s/%s.qcow2' command_to_execute = copy_command + ds_image_location%(vm_data.vm_identity, vm_data.vm_identity) + \ ' ' + ds_image_location%(vm_data.vm_identity, vm_data.vm_identity+'_old') execute_remote_cmd(vm_data.datastore_id.ds_ip, vm_data.datastore_id.username, command_to_execute, vm_data.datastore_id.password) logger.debug('Backup copy of the VM image cretaed successfully.') vm_properties = {} vm_properties['host'] = find_new_host(vm_data.RAM, vm_data.vCPU) vm_properties['ram'] = vm_data.RAM vm_properties['vcpus'] = vm_data.vCPU vm_properties['mac_addr'] = vm_data.private_ip.mac_addr vm_properties['vnc_port'] = vm_data.vnc_port vm_properties['template'] = current.db.template[vm_data.template_id] vm_properties['vlan_name'] = current.db(current.db.private_ip_pool.private_ip == vm_data.private_ip).select()[0].vlan.name # Re-spawn the VM on new host launch_vm_on_host(vm_data, vm_image_location%(vm_data.vm_identity, vm_data.vm_identity), vm_properties) vm_data.update_record(host_id = vm_properties['host']) #Find the most recent snapshot of the given VM; revert to the snapshot recent_snapshot = current.db(current.db.snapshot.vm_id == vm_data.id).select(orderby = ~current.db.snapshot.timestamp)[0] logger.debug('Reverting VM %s to snapshot %s' %(vm_data.vm_identity, recent_snapshot.snapshot_name)) revert(dict(vm_id = vm_data.id, snapshot_id = recent_snapshot.id))
def overload_memory(): logger.debug("Executing overload memory task") file_path_row = db(db.constants.name=="memory_overload_file_path").select(db.constants.value).first() file_path = file_path_row.value logger.debug(type(file_path)) host_ips_rows = db((db.host.status == HOST_STATUS_UP) & (db.host.host_ip == db.private_ip_pool.id)).select(db.private_ip_pool.private_ip) logger.debug(host_ips_rows) command2 = '/memhog >/memoryhog.out &' command3 = "ps -ef | grep memhog | grep -v grep | awk 'END{print FNR}'" for host_ip_row in host_ips_rows: logger.debug("overloading memory of"+str(host_ip_row)) logger.debug(type(host_ip_row['private_ip'])) check_compile_folder(file_path) command1 = 'scp '+ str(file_path) +'/memhog root@'+ str(host_ip_row['private_ip']) +':/' logger.debug('executing' + command1) ret = os.system(command1) logger.debug('os.system return value' + str(ret)) output = execute_remote_cmd(host_ip_row['private_ip'], 'root', command3) ret1 = int(output[0]) if(ret1 == 0): ret = execute_remote_cmd(host_ip_row['private_ip'], 'root', command2) logger.debug(ret) logger.debug("Completed overload memory task")
def get_latency_btw_hosts(next_host_ip,host_ip): latency_cmd="netperf -t TCP_RR -H"+str(next_host_ip)+ "| sed -n '7 p' | tr -s ' '| cut -f'6' -d' '" print("latency_cmd:" + str(latency_cmd)) l_output=execute_remote_cmd(host_ip, "root", latency_cmd) lat=l_output if lat: latency=1/float(lat) print("latency between host is :"+ str(latency)) ret=str(latency) else: ret=str(lat) return ret
def overload_memory(): logger.debug("Executing overload memory task") file_path_row = db(db.constants.name=="memory_overload_file_path").select(db.constants.value).first() file_path = file_path_row.value logger.debug(type(file_path)) host_ips_rows = db((db.host.status == HOST_STATUS_UP) & (db.host.host_ip == db.private_ip_pool.id)).select(db.private_ip_pool.private_ip) logger.debug(host_ips_rows) command2 = 'nohup /memhog >memoryhog.out 2>&1 &' for host_ip_row in host_ips_rows: logger.debug("overloading memory of") logger.debug(host_ip_row) logger.debug(type(host_ip_row['private_ip'])) check_compile_folder(file_path) command1 = 'scp '+ str(file_path) +'/memhog root@'+ str(host_ip_row['private_ip']) +':/' logger.debug('executing' + command1) ret = os.system(command1) logger.debug('os.system return value' + str(ret)) ret = execute_remote_cmd(host_ip_row['private_ip'], 'root', command2) logger.debug(ret) logger.debug("Completed overload memory task")
def request_vpn(): user_info=get_vpn_user_details() logger.debug(type(user_info)) user_name=user_info['username'] cmd="./vpn_client_creation.sh "+ str(user_name) #vpn_ip="" vpn_ip=config.get("VPN_CONF","vpn_server_ip") vpn_key_path=config.get("VPN_CONF","vpn_key_path") try: var = execute_remote_cmd(vpn_ip, 'root',cmd, ret_list=True) filepath=vpn_key_path+str(user_name)+"_baadalVPN.tar" localpath = os.path.join(get_context_path(), 'private/VPN/' + str(user_name) +"_baadalVPN.tar") sftp_files(vpn_ip, 'root', filepath, localpath) if "false" in str(var): return 1 elif "true" in str(var): return 3 #transfer_vpn_files(user_name,vpn_ip,password) except Exception: return 2
def get_host_type(host_ip): command="virt-what" ret=execute_remote_cmd(host_ip, 'root',command) return HOST_TYPE_VIRTUAL if ret else HOST_TYPE_PHYSICAL
def get_host_ram(host_ip): command = "cat /proc/meminfo | grep MemTotal" ret = execute_remote_cmd(host_ip, 'root',command)#Returns e.g. MemTotal: 32934972 kB ram_in_kb = ret[ret.index(' '):-3].strip() ram_in_gb = int(math.ceil(float(ram_in_kb)/(1024*1024))) return ram_in_gb
def get_host_cpu(host_ip): command = "grep -c processor /proc/cpuinfo" ret = execute_remote_cmd(host_ip, 'root',command) return int(ret)
def get_host_mac_address(host_ip): command = "ifconfig -a | grep baadal-br-int | head -n 1" ret = execute_remote_cmd(host_ip, 'root',command)#Returns e.g. eth0 Link encap:Ethernet HWaddr 18:03:73:0d:e4:49 ret=ret.strip() mac_addr = ret[ret.rindex(' '):].lstrip() return mac_addr