def overload_memory():
logger.debug("Executing overload memory task")
file_path_row = db(
db.constants.name == "memory_overload_file_path").select(
db.constants.value).first()
file_path = file_path_row.value
logger.debug(type(file_path))
host_ips_rows = db((db.host.status == HOST_STATUS_UP)
& (db.host.host_ip == db.private_ip_pool.id)).select(
db.private_ip_pool.private_ip)
logger.debug(host_ips_rows)
command2 = '/memhog >/memoryhog.out &'
command3 = "ps -ef | grep memhog | grep -v grep | awk 'END{print FNR}'"
for host_ip_row in host_ips_rows:
logger.debug("overloading memory of" + str(host_ip_row))
logger.debug(type(host_ip_row['private_ip']))
_check_compile_folder(file_path)
command1 = 'scp ' + str(file_path) + '/memhog root@' + str(
host_ip_row['private_ip']) + ':/'
logger.debug('executing' + command1)
ret = os.system(command1)
logger.debug('os.system return value' + str(ret))
output = execute_remote_cmd(host_ip_row['private_ip'], 'root',
command3)
ret1 = int(output[0])
if (ret1 == 0):
ret = execute_remote_cmd(host_ip_row['private_ip'], 'root',
command2)
logger.debug(ret)
logger.debug("Completed overload memory task")
def create_dhcp_bulk_entry(dhcp_info_list):
if len(dhcp_info_list) == 0: return
entry_cmd = ""
for dhcp_info in dhcp_info_list:
host_name = dhcp_info[0] if dhcp_info[0] != None else (
'IP_' + dhcp_info[2].replace(".", '_'))
dhcp_cmd = '''
file_name="/etc/dhcp/dhcp.d/1_%s.conf"
if [ -e "$file_name" ]
then
echo $file_name
else
echo "host %s {\n\thardware ethernet %s;\n\tfixed-address %s;\n}\n" > $file_name
fi
''' % (host_name, host_name, dhcp_info[1], dhcp_info[2])
entry_cmd += dhcp_cmd
restart_cmd = '''
cat /etc/dhcp/dhcp.d/*.conf > /etc/dhcp/dhcpd.conf
/etc/init.d/isc-dhcp-server restart
'''
execute_remote_cmd(dhcp_ip, 'root', entry_cmd)
execute_remote_cmd(dhcp_ip, 'root', restart_cmd)
def host_power_down(host_data):
try:
host_ip = host_data.host_ip.private_ip
if host_data.host_type == HOST_TYPE_VIRTUAL:
output = execute_remote_cmd(host_ip, 'root', 'virsh destroy' + get_host_name[host_ip])
else:
setup_type = config.get("GENERAL_CONF","setup_type")
if setup_type == "nic":
ucs_management_ip = config.get("UCS_MANAGER_DETAILS","ucs_management_ip")
logger.debug(ucs_management_ip)
logger.debug(type(ucs_management_ip))
ucs_user = config.get("UCS_MANAGER_DETAILS","ucs_user")
logger.debug(ucs_user)
logger.debug(type(ucs_user))
ucs_password = config.get("UCS_MANAGER_DETAILS","ucs_password")
logger.debug(ucs_password)
host_ip=str(host_ip)
server_num=host_ip.split('.')
ucs_server_num=str(int(server_num[3])-20)
logger.debug("ucs server number is :"+ucs_server_num)
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.connect(ucs_management_ip,username=ucs_user,password=ucs_password)
stdin, stdout,stderr=ssh.exec_command("scope org / ; scope org IIT-Delhi ; scope service-profile Badal-Host"+ str(ucs_server_num) + " ; power down ; commit-buffer") # @UnusedVariable
output=stdout.readlines()
if len(output)!= 0:
logger.debug("Host not powered up . Command not run properly ")
else:
output = execute_remote_cmd(host_ip, 'root', 'init 0')
logger.debug(str(output) + ' ,Host shut down successfully !!!')
except:
log_exception()
def create_dhcp_bulk_entry(dhcp_info_list):
if len(dhcp_info_list) == 0: return
entry_cmd = ""
for dhcp_info in dhcp_info_list:
host_name = dhcp_info[0] if dhcp_info[0] != None else ('IP_' + dhcp_info[2].replace(".", '_'))
dhcp_cmd = '''
file_name="/etc/dhcp/dhcp.d/1_%s.conf"
if [ -e "$file_name" ]
then
echo $file_name
else
echo "host %s {\n\thardware ethernet %s;\n\tfixed-address %s;\n}\n" > $file_name
fi
''' %(host_name, host_name, dhcp_info[1], dhcp_info[2])
entry_cmd += dhcp_cmd
restart_cmd = '''
cat /etc/dhcp/dhcp.d/*.conf > /etc/dhcp/dhcpd.conf
/etc/init.d/isc-dhcp-server restart
'''
execute_remote_cmd(dhcp_ip, 'root', entry_cmd)
execute_remote_cmd(dhcp_ip, 'root', restart_cmd)
def show_cont_performance():
ip_cmd="/sbin/ifconfig | grep 'inet addr' | sed -n '1p' | awk '{print $2}' | cut -d ':' -f 2"
cpu_cmd="nproc"
mem_cmd = "free -m | grep 'Mem:' | awk '{print $2}'"
cont_cpu=execute_remote_cmd("localhost", 'root', cpu_cmd, None, True).strip()
cont_ram=execute_remote_cmd("localhost", 'root', mem_cmd, None, True).strip()
cont_ip=execute_remote_cmd("localhost", 'root', ip_cmd, None, True).replace(".","_").strip()
return dict(host_identity=cont_ip ,host_ram=cont_ram, m_type="host",host_cpu=cont_cpu)
def show_nat_performance():
ip_cmd="/sbin/route -n | sed -n '3p' | awk '{print $2}'"
cpu_cmd="nproc"
mem_cmd = "free -m | grep 'Mem:' | awk '{print $2}'"
nat_ip=execute_remote_cmd("localhost", 'root', ip_cmd, None, True).strip()
nat_cpu=execute_remote_cmd(nat_ip, 'root', cpu_cmd, None, True)[0].strip()
nat_ram=execute_remote_cmd(nat_ip, 'root', mem_cmd, None, True)[0].strip()
nat_identity=nat_ip.replace(".","_")
return dict(host_identity= nat_identity ,host_ram=nat_ram, m_type="host",host_cpu=nat_cpu)
def task_rrd():
list_host=[]
cont_ip_cmd="/sbin/ifconfig | grep 'inet addr' | sed -n '1p' | awk '{print $2}' | cut -d ':' -f 2"
nat_ip_cmd="/sbin/route -n | sed -n '3p' | awk '{print $2}'"
cont_ip=execute_remote_cmd("localhost", 'root',cont_ip_cmd, None, True).strip()
nat_ip=execute_remote_cmd("localhost", 'root',nat_ip_cmd, None, True).strip()
list_host.append(cont_ip)
list_host.append(nat_ip)
for ip in list_host:
m_type="controller"if cont_ip==ip else "nat"
vm_utilization_rrd(ip,m_type)
def task_rrd():
list_host=[]
ip_cmd="/sbin/ifconfig | grep 'inet addr' | sed -n '1p' | awk '{print $2}' | cut -d ':' -f 2"
controller_ip = execute_remote_cmd("localhost", 'root', ip_cmd, None, True).strip()
rrd_logger.info(controller_ip[0].strip("\n"))
nat_ip=execute_remote_cmd("localhost", 'root', ip_cmd, None, True).strip()
list_host.append(controller_ip)
list_host.append(nat_ip)
for ip in list_host:
m_type="controller" if controller_ip==ip else "nat"
vm_utilization_rrd(ip,m_type)
def get_host_mem_usage(host_ip,m_type=None):
command = "free -k | grep 'buffers/cache' | awk '{print $3}'"
if m_type=="controller":
command_output =execute_remote_cmd("localhost", 'root', command, None, True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
rrd_logger.info("command_output is %s"%(command_output[0]))
used_mem_in_kb = int(command_output[0])
rrd_logger.info("Mem stats of host %s is %s" % (host_ip, used_mem_in_kb))
return used_mem_in_kb #return memory in KB
def get_host_disk_usage(host_ip,m_type=None):
command = "iostat -d | sed '1,2d'"
if m_type=="controller":
command_output =execute_remote_cmd("localhost", 'root', command, None, True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
disk_stats = re.split('\s+', command_output[1])
rrd_logger.info("Disk stats of host %s is dr: %s dw: %s" % (host_ip, disk_stats[2], disk_stats[3]))
return [float(disk_stats[2]), float(disk_stats[3])] #return memory in KB/sec
def get_host_disk_usage(host_ip,m_type=None):
command = "iostat -d | sed '1,2d'"
if m_type=="controller":
command_output =execute_remote_cmd("localhost", 'root', command, None, True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
disk_stats = re.split('\s+', command_output[1])
rrd_logger.info("Disk stats of host %s is dr: %s dw: %s" % (host_ip, disk_stats[2], disk_stats[3]))
return [float(disk_stats[2]), float(disk_stats[3])] #return memory in KB/sec
def get_host_mem_usage(host_ip,m_type=None):
command = "free -k | grep 'buffers/cache' | awk '{print $3}'"
if m_type=="controller":
command_output =execute_remote_cmd("localhost", 'root', command, None, True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
rrd_logger.info("command_output is %s"%(command_output[0]))
used_mem_in_kb = int(command_output[0])
rrd_logger.info("Mem stats of host %s is %s" % (host_ip, used_mem_in_kb))
return used_mem_in_kb #return memory in KB
def remove_dhcp_entry(host_name, ip_addr):
host_name = host_name if host_name != None else ('IP_' + ip_addr.replace(".", '_'))
entry_cmd = "rm /etc/dhcp/dhcp.d/1_%s.conf" %(host_name)
restart_cmd = '''
cat /etc/dhcp/dhcp.d/*.conf > /etc/dhcp/dhcpd.conf
/etc/init.d/isc-dhcp-server restart
'''
execute_remote_cmd(dhcp_ip, 'root', entry_cmd)
execute_remote_cmd(dhcp_ip, 'root', restart_cmd)
def get_host_nw_usage(host_ip,m_type=None):
command = "ifconfig baadal-br-int | grep 'RX bytes:'"
if m_type=="controller":
command_output =execute_remote_cmd("localhost", 'root', command, None, True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
nw_stats = re.split('\s+', command_output[0])
rx = int(re.split(':', nw_stats[2])[1])
tx = int(re.split(':', nw_stats[6])[1])
rrd_logger.info("Network stats of host %s is rx: %s tx: %s" % (host_ip, rx, tx))
return [rx, tx] # return network in Bytes
def remove_dhcp_entry(host_name, ip_addr):
host_name = host_name if host_name != None else ('IP_' +
ip_addr.replace(".", '_'))
entry_cmd = "rm /etc/dhcp/dhcp.d/1_%s.conf" % (host_name)
restart_cmd = '''
cat /etc/dhcp/dhcp.d/*.conf > /etc/dhcp/dhcpd.conf
/etc/init.d/isc-dhcp-server restart
'''
execute_remote_cmd(dhcp_ip, 'root', entry_cmd)
execute_remote_cmd(dhcp_ip, 'root', restart_cmd)
def get_host_nw_usage(host_ip,m_type=None):
command = "ifconfig baadal-br-int | grep 'RX bytes:'"
if m_type=="controller":
command_output =execute_remote_cmd("localhost", 'root', command, None, True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
nw_stats = re.split('\s+', command_output[0])
rx = int(re.split(':', nw_stats[2])[1])
tx = int(re.split(':', nw_stats[6])[1])
rrd_logger.info("Network stats of host %s is rx: %s tx: %s" % (host_ip, rx, tx))
return [rx, tx] # return network in Bytes
def task_rrd():
list_host=[]
cont_ip_cmd="/sbin/ifconfig | grep 'inet addr' | sed -n '1p' | awk '{print $2}' | cut -d ':' -f 2"
nat_ip_cmd="/sbin/route -n | sed -n '3p' | awk '{print $2}'"
cont_ip=execute_remote_cmd("localhost", 'root',cont_ip_cmd, None, True).strip()
nat_ip=execute_remote_cmd("localhost", 'root',nat_ip_cmd, None, True).strip()
list_host.append(cont_ip)
list_host.append(nat_ip)
for ip in list_host:
if cont_ip==ip:
m_type="controller"
else:
m_type="nat"
vm_utilization_rrd(ip,m_type)
def show_cont_performance():
ip_cmd = "/sbin/ifconfig | grep 'inet addr' | sed -n '1p' | awk '{print $2}' | cut -d ':' -f 2"
cpu_cmd = "nproc"
mem_cmd = "free -m | grep 'Mem:' | awk '{print $2}'"
cont_cpu = execute_remote_cmd("localhost", 'root', cpu_cmd, None,
True).strip()
cont_ram = execute_remote_cmd("localhost", 'root', mem_cmd, None,
True).strip()
cont_ip = execute_remote_cmd("localhost", 'root', ip_cmd, None,
True).replace(".", "_").strip()
return dict(host_identity=cont_ip,
host_ram=cont_ram,
m_type="host",
host_cpu=cont_cpu)
def check_host_service_status(host_ip):
#Check libvirt status
command = "ps -ef | grep libvirtd | grep -v grep | wc -l"
ret = execute_remote_cmd(host_ip, 'root',command)
if ret == 0 :
logger.error("Critical: Libvirt service is not running on host " + host_ip)
return False
#Check OVS status
command = "service openvswitch-switch status | grep -w 'running' | wc -l"
ret = execute_remote_cmd(host_ip, 'root',command)
if ret == 0 :
logger.error("Critical: OVS switch is not running on host " + host_ip)
return False
return True
def grant_novnc_access(vm_id):
msg = ""
active_vnc = db((db.vnc_access.vm_id == vm_id) & (
db.vnc_access.status == VNC_ACCESS_STATUS_ACTIVE)).count()
if active_vnc > 0:
vm_data = db(db.vnc_access.vm_id == vm_id).select().first()
token = vm_data.token
msg = 'VNC access already granted. Please check your mail for further details.'
else:
vnc_count = db((db.vnc_access.vm_id == vm_id)
& (db.vnc_access.time_requested >
(get_datetime() - timedelta(days=1)))).count()
if vnc_count >= MAX_VNC_ALLOWED_IN_A_DAY:
msg = 'VNC request has exceeded limit.'
else:
try:
f = os.popen('openssl rand -hex 10')
token = f.read()
token = token.split("\n")
token = token[0]
create_novnc_mapping(vm_id, token)
vm_data = db(db.vm_data.id == vm_id).select().first()
host_ip = vm_data.host_id.host_ip.private_ip
vnc_port = vm_data.vnc_port
vnc = str(vnc_port)
file_token = str(token) + ":" + " " + str(host_ip) + ":" + str(
vnc) + "\n"
myfile = get_file_append_mode("/home/www-data/token.list")
myfile.write(file_token)
command = "ps -ef | grep websockify|awk '{print $2}'"
port = config.get("NOVNC_CONF", "port")
server_ip = config.get("NOVNC_CONF", "server_ip")
return_value = execute_remote_cmd(server_ip, 'root', command)
return_value = return_value.split()
if len(return_value) <= 2:
command = "./noVNC/utils/websockify/run --web /root/noVNC --target-config /home/www-data/token.list " + str(
server_ip) + ":" + str(port) + " > /dev/null 2>&1 &"
return_value = execute_remote_cmd(server_ip, 'root',
command)
msg = 'VNC access granted. Please check your mail for further details.'
except:
logger.debug('Some Error Occurred. Please try later')
log_exception()
pass
logger.debug(msg)
return token
def show_nat_performance():
ip_cmd = "/sbin/route -n | sed -n '3p' | awk '{print $2}'"
cpu_cmd = "nproc"
mem_cmd = "free -m | grep 'Mem:' | awk '{print $2}'"
nat_ip = execute_remote_cmd("localhost", 'root', ip_cmd, None,
True).strip()
nat_cpu = execute_remote_cmd(nat_ip, 'root', cpu_cmd, None,
True)[0].strip()
nat_ram = execute_remote_cmd(nat_ip, 'root', mem_cmd, None,
True)[0].strip()
nat_identity = nat_ip.replace(".", "_")
return dict(host_identity=nat_identity,
host_ram=nat_ram,
m_type="host",
host_cpu=nat_cpu)
def add_orphan_vm(vm_name, host_id):
"""
Add Orphan VM information to database. VM information is retrieved from the VM definition XML.
'System User' is added as owner of the VM.
"""
host_details = db.host[host_id]
host_ip = host_details.host_ip.private_ip
connection_object = libvirt.openReadOnly("qemu+ssh://root@" + host_ip + "/system")
domain = connection_object.lookupByName(vm_name)
vm_state = domain.info()[0]
vm_status = vm_state_map[vm_state]
# Parse domain XML to get information about VM
root = etree.fromstring(domain.XMLDesc(0))
ram_elem = root.xpath('memory')[0]
ram_in_kb = int(ram_elem.text)
ram_in_mb = int(round(int(ram_in_kb)/(1024),0))
cpu_elem = root.xpath('vcpu')[0]
cpu = int(cpu_elem.text)
vnc_elem = root.xpath("devices/graphics[@type='vnc']")[0]
vnc_port = vnc_elem.attrib['port']
mac_elem = root.xpath("devices/interface[@type='network']/mac")[0]
mac_address = mac_elem.attrib['address']
ip_addr = db.private_ip_pool(mac_addr = mac_address)
ip_address = None
if ip_addr:
ip_address = ip_addr['id']
template_elem = root.xpath("devices/disk[@type='file']/source")[0]
template_file = template_elem.attrib['file']
command = "qemu-img info " + template_file + " | grep 'virtual size'"
ret = execute_remote_cmd(host_ip, 'root', command) # Returns e.g. virtual size: 40G (42949672960 bytes)
hdd = int(ret[ret.index(':')+1:ret.index('G ')].strip())
security_domain_row = db.security_domain(vlan=ip_addr['vlan'])
db.vm_data.insert(
vm_name = vm_name,
vm_identity = (vm_name),
RAM = ram_in_mb,
HDD = hdd,
extra_HDD = 0,
vCPU = cpu,
host_id = host_id,
template_id = 1, #TBD
datastore_id = 1, #TBD
owner_id = SYSTEM_USER,
requester_id = SYSTEM_USER,
private_ip = ip_address,
vnc_port = vnc_port,
purpose = 'Added by System',
security_domain = security_domain_row['id'],
status = vm_status)
return
def remove_dhcp_entry(host_name, mac_addr, ip_addr):
host_name = host_name if host_name != None else ('IP_' +
ip_addr.replace(".", '_'))
if mac_addr != None:
entry_cmd = "sed -i '/host.*%s.*{/ {N;N;N; s/host.*%s.*{.*hardware.*ethernet.*%s;.*fixed-address.*%s;.*}//g}' /etc/dhcp/dhcpd.conf" % (
host_name, host_name, mac_addr, ip_addr)
else:
entry_cmd = "sed -i '/host.*%s.*{/ {N;N;N; s/host.*%s.*{.*}//g}' /etc/dhcp/dhcpd.conf" % (
host_name, host_name)
restart_cmd = "/etc/init.d/isc-dhcp-server restart"
execute_remote_cmd(dhcp_ip, 'root', entry_cmd)
execute_remote_cmd(dhcp_ip, 'root', restart_cmd)
def get_host_disk_usage(host_ip):
command = "iostat -d | sed '1,2d'"
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
disk_stats = re.split('\s+', command_output[1])
rrd_logger.info("Disk stats of host %s is dr: %s dw: %s" % (host_ip, disk_stats[2], disk_stats[3]))
return [float(disk_stats[2]), float(disk_stats[3])]
def add_orphan_vm(vm_name, host_id):
host_details = db.host[host_id]
host_ip = host_details.host_ip.private_ip
connection_object = libvirt.openReadOnly("qemu+ssh://root@" + host_ip +
"/system")
domain = connection_object.lookupByName(vm_name)
vm_state = domain.info()[0]
vm_status = vm_state_map[vm_state]
# Parse domain XML to get information about VM
root = etree.fromstring(domain.XMLDesc(0))
ram_elem = root.xpath('memory')[0]
ram_in_kb = int(ram_elem.text)
ram_in_mb = int(round(int(ram_in_kb) / (1024), 0))
cpu_elem = root.xpath('vcpu')[0]
cpu = int(cpu_elem.text)
vnc_elem = root.xpath("devices/graphics[@type='vnc']")[0]
vnc_port = vnc_elem.attrib['port']
mac_elem = root.xpath("devices/interface[@type='network']/mac")[0]
mac_address = mac_elem.attrib['address']
ip_addr = db.private_ip_pool(mac_addr=mac_address)
ip_address = None
if ip_addr:
ip_address = ip_addr['id']
template_elem = root.xpath("devices/disk[@type='file']/source")[0]
template_file = template_elem.attrib['file']
command = "qemu-img info " + template_file + " | grep 'virtual size'"
ret = execute_remote_cmd(
host_ip, 'root',
command) # Returns e.g. virtual size: 40G (42949672960 bytes)
hdd = int(ret[ret.index(':') + 1:ret.index('G ')].strip())
security_domain_row = db.security_domain(vlan=ip_addr['vlan'])
db.vm_data.insert(
vm_name=vm_name,
vm_identity=(vm_name),
RAM=ram_in_mb,
HDD=hdd,
extra_HDD=0,
vCPU=cpu,
host_id=host_id,
template_id=1, #TBD
datastore_id=1, #TBD
owner_id=SYSTEM_USER,
requester_id=SYSTEM_USER,
private_ip=ip_address,
vnc_port=vnc_port,
purpose='Added by System',
security_domain=security_domain_row['id'],
status=vm_status)
return
def request_vpn():
user_info = get_vpn_user_details()
logger.debug(type(user_info))
user_name = user_info['username']
cmd = "./vpn_client_creation.sh " + str(user_name)
#vpn_ip=""
vpn_ip = config.get("VPN_CONF", "vpn_server_ip")
vpn_key_path = config.get("VPN_CONF", "vpn_key_path")
#passwd=""
# password=config.get("VPN_CONF","passwd")
try:
var = execute_remote_cmd(vpn_ip, 'root', cmd, ret_list=True)
filepath = vpn_key_path + str(user_name) + "_baadalVPN.tar"
localpath = os.path.join(
get_context_path(),
'private/VPN/' + str(user_name) + "_baadalVPN.tar")
sftp_files(vpn_ip, 'root', filepath, localpath)
if "false" in str(var):
return 1
elif "true" in str(var):
return 3
#transfer_vpn_files(user_name,vpn_ip,password)
except Exception:
return 2
def get_host_hdd(host_ip):
command = "fdisk -l | egrep 'Disk.*bytes' | awk '{ sub(/,/,\"\"); sum +=$3;} END {print sum}'"
ret = execute_remote_cmd(host_ip, 'root',command)#Returns e.g. 500.1 kB
logger.debug("Host HDD is %s" %ret)
hdd_in_gb = int(math.ceil(float(ret)))
return hdd_in_gb
def get_dom_mem_usage(dom_name, host):
rrd_logger.debug("Fetching memory usage of domain %s defined on host %s" % (dom_name, host))
cmd = "output=`ps -ef --sort=start_time | grep '%s.qcow2' | grep -v grep | awk '{print $2}'`;smem -c 'pid pss'| grep $output | awk '{print $2}'" % dom_name
#"ps aux | grep '\-name " + dom_name + " ' | grep kvm"
output = execute_remote_cmd(host, "root", cmd, None, True)
return (int(output[0]))*1024 #return memory in Bytes by default
def get_host_mem_usage(host_ip):
command = "top -b -n1 | grep 'Mem'"
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
mem_stats = re.split('\s+', command_output[0])[3]
used_mem_in_kb = int(mem_stats[:-1])
rrd_logger.info("Mem stats of host %s is %s" % (host_ip, used_mem_in_kb))
return used_mem_in_kb
def get_impi_output(cmd,host_ip):
rrd_logger.debug("entering ipmi")
output=execute_remote_cmd(host_ip,'root', cmd, None, True)
rrd_logger.debug(output)
rrd_logger.debug(output[0].split("|")[4].split(" ")[1])
ret=output[0].split("|")[4].split(" ")[1]
rrd_logger.debug("cheecking ret"+str(ret))
return int(ret)
def get_impi_output(cmd,host_ip):
rrd_logger.debug("entering ipmi")
output=execute_remote_cmd(host_ip,'root', cmd, None, True)
rrd_logger.debug(output)
rrd_logger.debug(output[0].split("|")[4].split(" ")[1])
ret=output[0].split("|")[4].split(" ")[1]
rrd_logger.debug("cheecking ret"+str(ret))
return int(ret)
def get_bandwidth_of_host(host_ip):
bandwidth_cmd='lshw | grep capacity | grep bit/s | sed -n "1 p" | tr -s " " | cut -f"3" -d" "'
bandwidth=execute_remote_cmd(host_ip,'root',bandwidth_cmd)
m=str(bandwidth)
q=m.replace("['",' ')
bandwidth=q.replace("']",' ')
return bandwidth
def get_host_cpu_usage(host_ip, m_type=None):
rrd_logger.info("getting cpu info")
command = "cat /proc/stat | awk 'FNR == 1{print $2+$3+$4}'"
if m_type == "controller":
command_output = execute_remote_cmd("localhost", 'root', command, None,
True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None,
True)
rrd_logger.debug(command_output[0])
timestamp_now = time.time()
cpu_stats = {
'cputicks': float(command_output[0])
} # (cpu time in clock ticks
cpu_stats.update({'timestamp': timestamp_now})
prev_cpu_stats = current.cache.disk(str(host_ip), lambda: cpu_stats,
86400) # @UndefinedVariable
rrd_logger.debug(prev_cpu_stats)
timestamp = float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp'])
rrd_logger.debug("timestamp %s" % str(timestamp))
#cpu_usage = cpu_stats - prev_cpu_stats # cpu usage in last 5 min (cputime-in ticks)
if timestamp == 0:
cpu_usage = float(cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])
else:
cpu_usage = float(
(cpu_stats['cputicks'] - prev_cpu_stats['cputicks']) *
300) / float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp']
) # keerti uncomment above line and comment it
current.cache.disk.clear(str(host_ip)) # @UndefinedVariable
"""Cache the current CPU utilization, so that difference can be calculated in next instance"""
latest_cpu_stats = current.cache.disk(str(host_ip), lambda: cpu_stats,
86400) # @UndefinedVariable
rrd_logger.debug(latest_cpu_stats)
clock_ticks = os.sysconf(
os.sysconf_names['SC_CLK_TCK']) # (ticks per second)
cpu_usage = float(cpu_usage * 1000000000) / float(
clock_ticks) # (cpu time in ns)
rrd_logger.info("CPU stats of host %s is %s" % (host_ip, cpu_usage))
return (cpu_usage)
def get_host_nw_usage(host_ip):
command = "ifconfig baadal-br-int | grep 'RX bytes:'"
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
nw_stats = re.split('\s+', command_output[0])
rx = int(re.split(':', nw_stats[2])[1])
tx = int(re.split(':', nw_stats[6])[1])
rrd_logger.info("Disk stats of host %s is rx: %s tx: %s" % (host_ip, rx, tx))
return [rx, tx]
def get_memhog_usage(host_ip):
logger.debug("Entering into getmemhog_usage")
cmd = "output=`ps -ef --sort=start_time | grep 'memhog' | grep -v grep | awk '{print $2}'`;smem -c 'pid pss'| grep $output | awk '{print $2}'"
output = execute_remote_cmd(host_ip, "root", cmd, None, True)
if not output :
return 0
else:
#logger.debug("For host:"+str(host_ip)+" PSS Value from memhog="+str(output))
return (int(output[0])*1024)
def get_host_cpu_usage(host_ip):
command = "iostat -c | sed '1,2d'"
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
rrd_logger.debug(type(command_output))
cpu_stats = re.split('\s+', command_output[1])
rrd_logger.debug(cpu_stats)
rrd_logger.info("CPU stats of host %s is %s" % ( host_ip, (cpu_stats[1] + cpu_stats[2] + cpu_stats[3])))
return (float(cpu_stats[1]) + float(cpu_stats[2]) + float(cpu_stats[3]))
def get_host_power_usage(host_ip):
rrd_logger.info("Checking power on host"+str(host_ip))
command='ipmitool sdr elist full'
try:
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
except Exception, e:
pwr_usage=0
ret=""
rrd_logger.debug("ipmitool not installed")
def get_host_power_usage(host_ip):
rrd_logger.info("Checking power on host"+str(host_ip))
command='ipmitool sdr elist full'
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
if str(ret).find("System Level")!=int(-1):
pwr_usage=get_impi_output('ipmitool sdr elist full | grep "System Level"',host_ip)
if str(ret).find("Pwr Consumption")!=int(-1):
pwr_usage=get_impi_output('ipmitool sdr elist full | grep "Pwr Consumption"',host_ip)
rrd_logger.info("power stats of host %s is %s" % (host_ip, str(pwr_usage)))
return pwr_usage # returining power in Watt
def get_host_power_usage(host_ip):
rrd_logger.info("Checking power on host"+str(host_ip))
command='ipmitool sdr elist full'
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
if str(ret).find("System Level")!=int(-1):
pwr_usage=get_impi_output('ipmitool sdr elist full | grep "System Level"',host_ip)
if str(ret).find("Pwr Consumption")!=int(-1):
pwr_usage=get_impi_output('ipmitool sdr elist full | grep "Pwr Consumption"',host_ip)
rrd_logger.info("power stats of host %s is %s" % (host_ip, str(pwr_usage)))
return pwr_usage # returining power in Watt
def grant_novnc_access(vm_id):
msg = ""
active_vnc = db((db.vnc_access.vm_id == vm_id) & (db.vnc_access.status == VNC_ACCESS_STATUS_ACTIVE)).count()
if active_vnc > 0:
vm_data = db(db.vnc_access.vm_id == vm_id).select().first()
token = vm_data.token
msg = 'VNC access already granted. Please check your mail for further details.'
else:
vnc_count = db((db.vnc_access.vm_id == vm_id) & (db.vnc_access.time_requested > (get_datetime() - timedelta(days=1)))).count()
if vnc_count >= MAX_VNC_ALLOWED_IN_A_DAY :
msg = 'VNC request has exceeded limit.'
else:
try:
f = os.popen('openssl rand -hex 10')
token = f.read()
token = token.split("\n")
token=token[0]
create_novnc_mapping(vm_id,token)
vm_data = db(db.vm_data.id == vm_id).select().first()
host_ip = vm_data.host_id.host_ip.private_ip
vnc_port = vm_data.vnc_port
vnc = str(vnc_port)
file_token =str(token) +":" + " " + str(host_ip)+ ":" + str(vnc) + "\n"
myfile=get_file_append_mode("/home/www-data/token.list")
myfile.write(file_token)
command = "ps -ef | grep websockify|awk '{print $2}'"
port = config.get("NOVNC_CONF","port")
server_ip = config.get("NOVNC_CONF","server_ip")
return_value = execute_remote_cmd(server_ip, 'root',command)
return_value=return_value.split()
if len(return_value) <=2:
command = "./noVNC/utils/websockify/run --web /root/noVNC --target-config /home/www-data/token.list " +str(server_ip)+ ":"+str(port) + " > /dev/null 2>&1 &"
return_value = execute_remote_cmd(server_ip, 'root',command)
msg = 'VNC access granted. Please check your mail for further details.'
except:
logger.debug('Some Error Occurred. Please try later')
log_exception()
pass
logger.debug(msg)
return token
def get_memhog_usage(host_ip):
"""
Returns memory used by memhog process so that it can be added to available memory.
"""
logger.debug("Entering into getmemhog_usage")
cmd = "output=`ps -ef --sort=start_time | grep 'memhog' | grep -v grep | awk '{print $2}'`;smem -c 'pid pss'| grep $output | awk '{print $2}'"
output = execute_remote_cmd(host_ip, "root", cmd, None, True)
if not output:
return 0
else:
# logger.debug("For host:"+str(host_ip)+" PSS Value from memhog="+str(output))
return int(output[0]) * 1024
def get_memhog_usage(host_ip):
"""
Returns memory used by memhog process so that it can be added to available memory.
"""
logger.debug("Entering into getmemhog_usage")
cmd = "output=`ps -ef --sort=start_time | grep 'memhog' | grep -v grep | awk '{print $2}'`;smem -c 'pid pss'| grep $output | awk '{print $2}'"
output = execute_remote_cmd(host_ip, "root", cmd, None, True)
if not output:
return 0
else:
#logger.debug("For host:"+str(host_ip)+" PSS Value from memhog="+str(output))
return (int(output[0]) * 1024)
def get_host_cpu_usage(host_ip,m_type=None):
"""
Uses iostat tool to capture CPU statistics of host
"""
rrd_logger.info("getting cpu info")
command = "cat /proc/stat | awk 'FNR == 1{print $2+$3+$4}'"
if m_type=="controller":
command_output =execute_remote_cmd("localhost", 'root', command, None, True)
else:
command_output = execute_remote_cmd(host_ip, 'root', command, None, True)
rrd_logger.debug(command_output[0])
timestamp_now = time.time()
cpu_stats = {'cputicks' : float(command_output[0])} # (cpu time in clock ticks
cpu_stats.update({'timestamp' : timestamp_now})
prev_cpu_stats = current.cache.disk(str(host_ip), lambda:cpu_stats, 86400) # @UndefinedVariable
rrd_logger.debug(prev_cpu_stats)
timestamp = float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp'])
rrd_logger.debug("timestamp %s" % str(timestamp))
#cpu_usage = cpu_stats - prev_cpu_stats # cpu usage in last 5 min (cputime-in ticks)
if timestamp == 0:
cpu_usage = float(cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])
else:
cpu_usage = float((cpu_stats['cputicks'] - prev_cpu_stats['cputicks'])*300) / float(cpu_stats['timestamp'] - prev_cpu_stats['timestamp']) # keerti uncomment above line and comment it
current.cache.disk.clear(str(host_ip)) # @UndefinedVariable
"""Cache the current CPU utilization, so that difference can be calculated in next instance"""
latest_cpu_stats = current.cache.disk(str(host_ip), lambda:cpu_stats, 86400) # @UndefinedVariable
rrd_logger.debug(latest_cpu_stats)
clock_ticks = os.sysconf(os.sysconf_names['SC_CLK_TCK']) # (ticks per second)
cpu_usage = float(cpu_usage*1000000000) / float(clock_ticks) # (cpu time in ns)
rrd_logger.info("CPU stats of host %s is %s" % ( host_ip, cpu_usage))
return (cpu_usage)
def get_host_temp_usage(host_ip):
rrd_logger.info("Checking temp on host"+str(type(host_ip)))
command='ipmitool sdr elist full'
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
if str(ret).find("Inlet Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Inlet Temp"',host_ip)
if str(ret).find("Ambient Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Ambient Temp"',host_ip)
rrd_logger.info("temp stats" +str(type(temp)))
rrd_logger.info("temp stats" +str(temp))
return temp # returning temperature in degree Celsius
def get_host_temp_usage(host_ip):
rrd_logger.info("Checking temp on host"+str(type(host_ip)))
command='ipmitool sdr elist full'
ret=execute_remote_cmd(host_ip, 'root', command, None, True)
if str(ret).find("Inlet Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Inlet Temp"',host_ip)
if str(ret).find("Ambient Temp")!=int(-1):
rrd_logger.debug("Entering")
temp=get_impi_output('ipmitool sdr elist full | grep "Ambient Temp"',host_ip)
rrd_logger.info("temp stats" +str(type(temp)))
rrd_logger.info("temp stats" +str(temp))
return temp # returning temperature in degree Celsius
def respawn_dangling_vms(host_id):
vms = current.db(current.db.vm_data.host_id == host_id).select(current.db.vm_data.ALL)
vm_image_location = get_constant('vmfiles_path') + get_constant('vms') + '/%s/%s.qcow2'
for vm_data in vms:
logger.debug('Re-spawning VM ' + vm_data.vm_identity)
#Create a copy of existing image and rename it with '_old' suffix
storage_type = config.get("GENERAL_CONF","storage_type")
copy_command = 'ndmpcopy ' if storage_type == current.STORAGE_NETAPP_NFS else 'cp '
ds_image_location = vm_data.datastore_id.path + get_constant('vms') + '/%s/%s.qcow2'
command_to_execute = copy_command + ds_image_location%(vm_data.vm_identity, vm_data.vm_identity) + \
' ' + ds_image_location%(vm_data.vm_identity, vm_data.vm_identity+'_old')
execute_remote_cmd(vm_data.datastore_id.ds_ip,
vm_data.datastore_id.username,
command_to_execute,
vm_data.datastore_id.password)
logger.debug('Backup copy of the VM image cretaed successfully.')
vm_properties = {}
vm_properties['host'] = find_new_host(vm_data.RAM, vm_data.vCPU)
vm_properties['ram'] = vm_data.RAM
vm_properties['vcpus'] = vm_data.vCPU
vm_properties['mac_addr'] = vm_data.private_ip.mac_addr
vm_properties['vnc_port'] = vm_data.vnc_port
vm_properties['template'] = current.db.template[vm_data.template_id]
vm_properties['vlan_name'] = current.db(current.db.private_ip_pool.private_ip == vm_data.private_ip).select()[0].vlan.name
# Re-spawn the VM on new host
launch_vm_on_host(vm_data, vm_image_location%(vm_data.vm_identity, vm_data.vm_identity), vm_properties)
vm_data.update_record(host_id = vm_properties['host'])
#Find the most recent snapshot of the given VM; revert to the snapshot
recent_snapshot = current.db(current.db.snapshot.vm_id == vm_data.id).select(orderby = ~current.db.snapshot.timestamp)[0]
logger.debug('Reverting VM %s to snapshot %s' %(vm_data.vm_identity, recent_snapshot.snapshot_name))
revert(dict(vm_id = vm_data.id, snapshot_id = recent_snapshot.id))
def overload_memory():
logger.debug("Executing overload memory task")
file_path_row = db(db.constants.name=="memory_overload_file_path").select(db.constants.value).first()
file_path = file_path_row.value
logger.debug(type(file_path))
host_ips_rows = db((db.host.status == HOST_STATUS_UP) & (db.host.host_ip == db.private_ip_pool.id)).select(db.private_ip_pool.private_ip)
logger.debug(host_ips_rows)
command2 = '/memhog >/memoryhog.out &'
command3 = "ps -ef | grep memhog | grep -v grep | awk 'END{print FNR}'"
for host_ip_row in host_ips_rows:
logger.debug("overloading memory of"+str(host_ip_row))
logger.debug(type(host_ip_row['private_ip']))
check_compile_folder(file_path)
command1 = 'scp '+ str(file_path) +'/memhog root@'+ str(host_ip_row['private_ip']) +':/'
logger.debug('executing' + command1)
ret = os.system(command1)
logger.debug('os.system return value' + str(ret))
output = execute_remote_cmd(host_ip_row['private_ip'], 'root', command3)
ret1 = int(output[0])
if(ret1 == 0):
ret = execute_remote_cmd(host_ip_row['private_ip'], 'root', command2)
logger.debug(ret)
logger.debug("Completed overload memory task")
def get_latency_btw_hosts(next_host_ip,host_ip):
latency_cmd="netperf -t TCP_RR -H"+str(next_host_ip)+ "| sed -n '7 p' | tr -s ' '| cut -f'6' -d' '"
print("latency_cmd:" + str(latency_cmd))
l_output=execute_remote_cmd(host_ip, "root", latency_cmd)
lat=l_output
if lat:
latency=1/float(lat)
print("latency between host is :"+ str(latency))
ret=str(latency)
else:
ret=str(lat)
return ret
def overload_memory():
logger.debug("Executing overload memory task")
file_path_row = db(db.constants.name=="memory_overload_file_path").select(db.constants.value).first()
file_path = file_path_row.value
logger.debug(type(file_path))
host_ips_rows = db((db.host.status == HOST_STATUS_UP) & (db.host.host_ip == db.private_ip_pool.id)).select(db.private_ip_pool.private_ip)
logger.debug(host_ips_rows)
command2 = 'nohup /memhog >memoryhog.out 2>&1 &'
for host_ip_row in host_ips_rows:
logger.debug("overloading memory of")
logger.debug(host_ip_row)
logger.debug(type(host_ip_row['private_ip']))
check_compile_folder(file_path)
command1 = 'scp '+ str(file_path) +'/memhog root@'+ str(host_ip_row['private_ip']) +':/'
logger.debug('executing' + command1)
ret = os.system(command1)
logger.debug('os.system return value' + str(ret))
ret = execute_remote_cmd(host_ip_row['private_ip'], 'root', command2)
logger.debug(ret)
logger.debug("Completed overload memory task")
def request_vpn():
user_info=get_vpn_user_details()
logger.debug(type(user_info))
user_name=user_info['username']
cmd="./vpn_client_creation.sh "+ str(user_name)
#vpn_ip=""
vpn_ip=config.get("VPN_CONF","vpn_server_ip")
vpn_key_path=config.get("VPN_CONF","vpn_key_path")
try:
var = execute_remote_cmd(vpn_ip, 'root',cmd, ret_list=True)
filepath=vpn_key_path+str(user_name)+"_baadalVPN.tar"
localpath = os.path.join(get_context_path(), 'private/VPN/' + str(user_name) +"_baadalVPN.tar")
sftp_files(vpn_ip, 'root', filepath, localpath)
if "false" in str(var):
return 1
elif "true" in str(var):
return 3
#transfer_vpn_files(user_name,vpn_ip,password)
except Exception:
return 2
def get_host_type(host_ip):
command="virt-what"
ret=execute_remote_cmd(host_ip, 'root',command)
return HOST_TYPE_VIRTUAL if ret else HOST_TYPE_PHYSICAL
def get_host_ram(host_ip):
command = "cat /proc/meminfo | grep MemTotal"
ret = execute_remote_cmd(host_ip, 'root',command)#Returns e.g. MemTotal: 32934972 kB
ram_in_kb = ret[ret.index(' '):-3].strip()
ram_in_gb = int(math.ceil(float(ram_in_kb)/(1024*1024)))
return ram_in_gb
def get_host_cpu(host_ip):
command = "grep -c processor /proc/cpuinfo"
ret = execute_remote_cmd(host_ip, 'root',command)
return int(ret)
def get_host_mac_address(host_ip):
command = "ifconfig -a | grep baadal-br-int | head -n 1"
ret = execute_remote_cmd(host_ip, 'root',command)#Returns e.g. eth0 Link encap:Ethernet HWaddr 18:03:73:0d:e4:49
ret=ret.strip()
mac_addr = ret[ret.rindex(' '):].lstrip()
return mac_addr
