def copy_ucloud_image(self): print "Coping ubuntu cloud image to libvirt/images folder." waiting(10) global cloudimg cloudimg = CONF['DEFAULTS']['cloudimage'] command = 'cp %s/%s /var/lib/libvirt/images/' % (sk_img_path, cloudimg) execute(command, ignore_errors=False)
def check_reimage_vm_status(self, vmname): count = 40 expected_res = "reimage_completed" for _ in range(count): #Get reimage status res = self.sm_api.server_get(server_id=vmname, status=True) print "Waiting for %s VM to get re-imaged.." % vmname print "Expected status: %s :: Current status: %s " % ( expected_res, res) if res == expected_res: waiting(10) cmd = "virsh list --all" res = execute(cmd, ignore_errors=True) if re.search(r'shut off', res, re.M | re.I): print "Re-imaged VM is shut-off, starting now..." cmd = "virsh start " + vmname res = execute(cmd, ignore_errors=True) waiting(5) else: print "None of the re-imaged VM's are shut-off" return else: print "Retry after 30sec" waiting(30) print "\n" print "===========================================================" print "VM %s did not complete Re-imaging, hence exiting..." % vmname print "===========================================================" sys.exit(1)
def copy_seed_ks_to_websvr(self): print "Copying ubuntu seed and kickstart files on to jumphost webserver" cmd = 'cp {0}/contrail-ubuntu-sk.seed {1}/'.format( from_project_root('contrail'), sk_img_path) execute(cmd, ignore_errors=True) cmd = 'cp {0}/contrail-ubuntu-sk.ks {1}/'.format( from_project_root('contrail'), sk_img_path) execute(cmd, ignore_errors=True)
def delete_sm(self): sm_name = CONF['servermanager']['hostname'] print "Deleting SM-VM %s" % (sm_name) cmd = "virsh destroy %s" % sm_name execute(cmd, ignore_errors=True) waiting(5) cmd = "virsh undefine %s" % sm_name execute(cmd, ignore_errors=True) waiting(5)
def delete_vms(self): vm_list = CONF['DEFAULTS']['reimagevms'] print "Deleting %s VMS" % (vm_list) for vm in vm_list: cmd = "virsh destroy %s" % CONF[vm]['hostname'] execute(cmd, ignore_errors=True) waiting(5) cmd = "virsh undefine %s" % CONF[vm]['hostname'] execute(cmd, ignore_errors=True) waiting(5)
def check_connectivity_to_allnodes(self): print "Checking connectivity to ctrldata IPs of all nodes in cluster" all_list = CONF['DEFAULTS']['bms'] + CONF['DEFAULTS']['reimagevms'] for inventory in all_list: ctrld_ip = get_ip(CONF[inventory]['ctrldata_address']) cmd = 'ping -c 4 %s' % (ctrld_ip) res = execute(cmd, ignore_errors=True) if re.search(r'4 received', res, re.M | re.I): print "CTRLDATA IP(%s)of (%s) is reachable from jumphost." % ( ctrld_ip, inventory) else: print "\n" print "Command: %s" % (cmd) print "Response: %s" % (res) print "==========================================================================" print "CTRLDATA IP(%s)of (%s) is not reachable from jumphost.check configuration" % ( ctrld_ip, inventory) print "==========================================================================" sys.exit(1)
def cleanup_jumphost(self): svr1 = CONF['DEFAULTS']['bms'][0] print "Cleaning up jumphost server (%s) configuration" % (svr1) m_intf = CONF[svr1]['management_interface'] c_intf = CONF[svr1]['ctrldata_interface'] m_ip = get_ip(CONF[svr1]['management_address']) m_netmask = get_netmask(CONF[svr1]['management_address']) c_ip = get_ip(CONF[svr1]['ctrldata_address']) c_netmask = get_netmask(CONF[svr1]['ctrldata_address']) cmd = "/sbin/ifconfig %s 0.0.0.0" % m_intf execute(cmd, ignore_errors=True) cmd = "/sbin/ifconfig %s %s netmask %s up" % (m_intf, m_ip, m_netmask) execute(cmd, ignore_errors=False) cmd = "/sbin/ifconfig mgmtbr 0.0.0.0 down && brctl delif mgmtbr %s && brctl delbr mgmtbr" % ( m_intf) execute(cmd, ignore_errors=False) cmd = "/sbin/ifconfig %s 0.0.0.0" % c_intf execute(cmd, ignore_errors=False) cmd = "/sbin/ifconfig %s %s netmask %s up" % (c_intf, c_ip, c_netmask) execute(cmd, ignore_errors=False) cmd = "/sbin/ifconfig ctrldatabr 0.0.0.0 down && brctl delif ctrldatabr %s && brctl delbr ctrldatabr" % ( c_intf) execute(cmd, ignore_errors=False) cmd = "/sbin/ip route | grep default | awk '{print $3}'" ext_gw = execute(cmd, ignore_errors=False) cmd = "brctl show extbr | grep extbr | awk '{print $4}'" ext_intf = execute(cmd, ignore_errors=False) cmd = "/sbin/ifconfig extbr | grep inet | grep -oE \"\\b([0-9]{1,3}\.){3}[0-9]{1,3}\\b\"" ext_intf_ips = execute(cmd, ignore_errors=False) ext_intf_ip = ext_intf_ips.splitlines()[0] ext_intf_netmask = ext_intf_ips.splitlines()[2] cmd = "/sbin/ifconfig %s %s netmask %s up" % (ext_intf, ext_intf_ip, ext_intf_netmask) execute(cmd, ignore_errors=False) cmd = "/sbin/route add default gw %s dev %s" % (ext_gw, ext_intf) execute(cmd, ignore_errors=False) cmd = "/sbin/ifconfig extbr 0.0.0.0 down && brctl delif extbr %s && brctl delbr extbr" % ( ext_intf) execute(cmd, ignore_errors=False) cmd = "mv /etc/network/interfaces.bak /etc/network/interfaces" execute(cmd, ignore_errors=False) cmd = "/bin/rm -rf /var/www/html/pockit_images/" execute(cmd, ignore_errors=False) cmd = "/bin/rm -rf /root/sm_vm/" execute(cmd, ignore_errors=False)
def configure_and_verify_vmx(self): print "Configuring and verifying vMX management IP reachability" print "Waiting for vMX to boot up.." waiting(200) global vmx_folder self.svr2_bms.chdir(vmx_folder) cmd = 'expect vmx-init-cfg.exp' op = self.svr2_bms.execute_cmd(cmd, timeout=120) waiting(10) vmx_re_ipaddr = get_ip(CONF['DEFAULTS']['vmx_vcp_address']) cmd = 'ping -c 5 {}'.format(vmx_re_ipaddr) res = self.svr2_bms.execute_cmd(cmd, timeout=60) if re.search(r'5 received', res, re.M | re.I): print "vMX mgmt IP %s is reachable, its UP and RUNNING" % ( vmx_re_ipaddr) else: print "\n" print "Command: %s" % (cmd) print "Response: %s" % (res) print "============================================================" print "vMX mgmt IP %s is not reachable, please check configuration!" % ( vmx_re_ipaddr) print "============================================================" sys.exit(1) svr2 = self.svr2_bms_name print "Configuring ctrldata interface with IP on server(%s)" % ( self.svr2_bms_name) cmd = '/sbin/ifconfig {0} 0.0.0.0 up'.format( CONF[svr2]['ctrldata_interface']) res = self.svr2_bms.execute_cmd(cmd, timeout=20) ctl_ip = get_ip(CONF[svr2]['ctrldata_address']) ctl_netmask = get_netmask(CONF[svr2]['ctrldata_address']) cmd = '/sbin/ifconfig vmx_link1 {0} netmask {1} up'.format( ctl_ip, ctl_netmask) res = self.svr2_bms.execute_cmd(cmd, timeout=20) waiting(5) vmx_loopback_ip = CONF['DEFAULTS']['vmx_loopback_ip'] vmx_ctl_ip = get_ip(CONF['vmx']['ctrldata_address']) cmd = 'route add -host {0}/32 gw {1}'.format(vmx_loopback_ip, vmx_ctl_ip) res = self.svr2_bms.execute_cmd(cmd, timeout=40) waiting(10) #on jumphost add route to vmx loopback ip command = 'route add -host {0}/32 gw {1}'.format( vmx_loopback_ip, ctl_ip) res = execute(command, ignore_errors=False) waiting(5) #ping vMX loopback ip print "Checking vmx_loopback IP(%s) is reachable" % (vmx_loopback_ip) cmd = 'ping -c 5 {}'.format(vmx_loopback_ip) res = self.svr2_bms.execute_cmd(cmd, timeout=60) if re.search(r'5 received', res, re.M | re.I): print "vMX loopback IP %s is reachable from server(%s)" % ( vmx_loopback_ip, svr2) else: print "\n" print "Command: %s" % (cmd) print "Response: %s" % (res) print "==========================================================================" print "vMX loopback IP %s is not reachable from server(%s). Check configuration." % ( vmx_loopback_ip, svr2) print "==========================================================================" sys.exit(1)
def __init__(self): global jinja_env jinja_env = Environment( loader=FileSystemLoader(from_project_root('smgr'))) # configure server1 bridge ips svr1 = CONF['DEFAULTS']['bms'][0] print "Checking ntp server reachability from jumphost(%s)" % (svr1) sm_ntp_server = CONF['DEFAULTS']['ntp_servers'] res = os.system('ping -c 3 ' + sm_ntp_server) if res == 0: print "NTP server %s is reachable" % (sm_ntp_server) else: print "Response: %s" % (res) print "NTP server %s is not reachable" % (sm_ntp_server) sys.exit(1) print "Checking mgmtbr, ctrldatabr bridge configuration on jumphost server." m_intf = CONF[svr1]['management_interface'] c_intf = CONF[svr1]['ctrldata_interface'] m_ip = get_ip(CONF[svr1]['management_address']) m_netmask = get_netmask(CONF[svr1]['management_address']) c_ip = get_ip(CONF[svr1]['ctrldata_address']) c_netmask = get_netmask(CONF[svr1]['ctrldata_address']) command = 'brctl show mgmtbr' res1 = execute(command, ignore_errors=False) command = 'brctl show ctrldatabr' res2 = execute(command, ignore_errors=False) if None not in (re.search(r'%s' % m_intf, res1, re.M | re.I), re.search(r'%s' % c_intf, res2, re.M | re.I)): print "Jumphost server bridge interface configuration exists." else: print "Configuring jumphost server bridge interfaces. Begins.." command = '/sbin/ifconfig %s 0.0.0.0 up' % (m_intf) execute(command, ignore_errors=False) command = '/sbin/ifconfig mgmtbr %s netmask %s up' % (m_ip, m_netmask) execute(command, ignore_errors=False) command = 'brctl addif mgmtbr %s' % (m_intf) execute(command, ignore_errors=False) command = '/sbin/ifconfig %s 0.0.0.0 up' % (c_intf) execute(command, ignore_errors=False) command = '/sbin/ifconfig ctrldatabr %s netmask %s up' % ( c_ip, c_netmask) execute(command, ignore_errors=False) command = 'brctl addif ctrldatabr %s' % (c_intf) execute(command, ignore_errors=False) print "Configuring jumphost server bridge interfaces. Ends.." #update /etc/network/interfaces file with this ip cmd = 'cat /etc/network/interfaces' res = execute(cmd, ignore_errors=False) if re.search(r'mgmtbr|ctrldatabr', res, re.M | re.I): print "mgmtbr/ctrldatabr interface configuration exists" else: print "mgmtbr/ctrldatabr interface configuration does not exists, hence configuring" command = 'sudo echo -ne \"auto {}\n iface {} inet manual\n auto mgmtbr\n iface mgmtbr inet static\n address {}\n netmask {}\n bridge_ports {}\n bridge_stp off\n bridge_fd 0\n bridge_maxwait 0\n\" >> /etc/network/interfaces'.format( m_intf, m_intf, m_ip, m_netmask, m_intf) execute(command, ignore_errors=False) command = 'sudo echo -ne \"auto {}\n iface {} inet manual\n auto ctrldatabr\n iface ctrldatabr inet static\n address {}\n netmask {}\n bridge_ports {}\n bridge_stp off\n bridge_fd 0\n bridge_maxwait 0\n\" >> /etc/network/interfaces'.format( c_intf, c_intf, c_ip, c_netmask, c_intf) execute(command, ignore_errors=False)
def create_sm_vm(self): vm = 'servermanager' vm_name = CONF[vm]['hostname'] sm_m_ip = get_ip(CONF[vm]['management_address']) global sm_creation sm_creation = False #Check if the SM VM is created and service is UP command = 'virsh list | grep %s | awk \'{print $3}\'' % (vm_name) res = execute(command, ignore_errors=True) vm_running = False if res == 'running': print "SM VM is created and UP and RUNNING" vm_running = True #check if VM is reachable if so, check service is running #ping SM vm_pingable = False res = os.system('ping -c 5 ' + sm_m_ip) if res == 0: print "Servermanger VM is UP and reachable" vm_pingable = True sm_local_user = CONF[vm]['local_user'] sm_local_passwd = CONF[vm]['local_password'] sm_vm = RemoteConnection() sm_vm.connect(sm_m_ip, username=sm_local_user, password=sm_local_passwd) cmd = 'sudo service contrail-server-manager status' res = sm_vm.execute_cmd(cmd, timeout=10) if re.search(r'not|unrecognized', res, re.M | re.I): sm_vm_service = False else: print "contrail-server-manager service is running." sm_vm_service = True else: print "Servermanager VM is not reachable or does not exists" if vm_running and vm_pingable and sm_vm_service: print "The existing servermanager VM is UP and RUNNING, hence not deleting and creating a new one" #adding route to ipmi subnet print "Adding route to IPMI network to reach from servermanager" ipmi_network = CONF['DEFAULTS']['ipmi_network'] sm_gw = CONF[vm]['gateway'] cmd = 'route add -net {0} gw {1}'.format(ipmi_network, sm_gw) res = sm_vm.execute_cmd(cmd, timeout=10) else: sm_creation = True print "Creating servermanager VM with name " + vm_name print "Deleting existing servermanager VM" command = 'virsh destroy ' + vm_name execute(command, ignore_errors=True) waiting(3) command = 'virsh undefine ' + vm_name execute(command, ignore_errors=True) waiting(3) vncpasswd = CONF[vm]['local_password'] OS = '--os-variant=ubuntutrusty --os-type=linux' ARCH = '--arch=x86_64' NET1 = '--network bridge=mgmtbr' NET2 = '--network bridge=extbr' Disk = '--boot hd' Gr = '--graphics vnc,password={} --noautoconsole'.format(vncpasswd) Cpu = '--vcpus=%s' % CONF[vm]['vcpus'] Ram = '--ram=%s' % CONF[vm]['memory'] Src = '--disk path=/var/lib/libvirt/images/%s.qcow2,format=qcow2,bus=virtio --import --autostart --disk path=/var/lib/libvirt/images/%s_init.iso,bus=virtio,device=cdrom' % ( vm_name, vm_name) #print OS, ARCH, NET1, NET2, Disk, Gr, Cpu, Ram, Src command = 'cp -f /var/lib/libvirt/images/%s /var/lib/libvirt/images/%s.qcow2' % ( cloudimg, vm_name) execute(command, ignore_errors=False) command = 'qemu-img resize /var/lib/libvirt/images/%s.qcow2 +%sG' % ( vm_name, CONF[vm]['harddisk']) execute(command, ignore_errors=False) # create user-data and meta-data file print "Creating user-data and meta-data files to boot ubuntu cloud image. Begins.." vm_disk_path = '/root/sm_vm/{0}'.format(vm_name) command = 'rm -rf {0}'.format(vm_disk_path) execute(command, ignore_errors=False) waiting(3) command = 'mkdir -p {0}'.format(vm_disk_path) execute(command, ignore_errors=False) sm_root_password = CONF['DEFAULTS']['root_password'] sm_dns_search = CONF[vm]['dns_search'] sm_dns_server = CONF[vm]['dns_servers'] sm_local_user = CONF[vm]['local_user'] sm_local_passwd = CONF[vm]['local_password'] sm_m_ip = get_ip(CONF[vm]['management_address']) sm_m_netmask = get_netmask(CONF[vm]['management_address']) sm_ext_ip = get_ip(CONF['DEFAULTS']['sm_ext_address']) sm_ext_netmask = get_netmask(CONF['DEFAULTS']['sm_ext_address']) sm_ext_gateway = CONF['DEFAULTS']['sm_ext_gateway'] command = 'cat /home/{0}/.ssh/id_rsa.pub'.format(sm_local_user) sm_rsa_user = execute(command, ignore_errors=False) command = 'cat /root/.ssh/id_rsa.pub'.format(sm_local_user) sm_rsa_root = execute(command, ignore_errors=False) print "Creating user-data file required for cloud init image.." userdata_tmpl = jinja_env.get_template('user-data') userdata = userdata_tmpl.render(password=sm_root_password, local_user=sm_local_user, local_password=sm_local_passwd, dns_server=sm_dns_server, ssh_rsa_user=sm_rsa_user, ssh_rsa_root=sm_rsa_root, dns_search=sm_dns_search) fobj = open(vm_disk_path + '/user-data', 'w') if hasattr(userdata, '__iter__'): for data in userdata: fobj.write(data) fobj.close() else: fobj.write(userdata) fobj.close() print "Creating meta-data file required for cloud init image.." metadata_tmpl = jinja_env.get_template('meta-data') metadata = metadata_tmpl.render(hostname=vm_name, mgmt_ip=sm_m_ip, mgmt_netmask=sm_m_netmask, ext_ip=sm_ext_ip, ext_netmask=sm_ext_netmask, ext_gateway=sm_ext_gateway) fobj = open(vm_disk_path + '/meta-data', 'w') if hasattr(metadata, '__iter__'): for data in metadata: fobj.write(data) fobj.close() else: fobj.write(metadata) fobj.close() print "Creating user-data and meta-data files to boot ubuntu cloud image. Ends.." # Launch VM print "Launching servermanager VM..." cwddir = os.getcwd() command = 'cd {}'.format(vm_disk_path + '/') os.chdir(vm_disk_path) command = 'cloud-localds -v /var/lib/libvirt/images/{}_init.iso user-data meta-data'.format( vm_name) os.system(command) os.chdir(cwddir) command = 'virt-install {} {} {} {} {} {} {} {} {} --name={}'.format( OS, ARCH, NET1, NET2, Disk, Src, Gr, Cpu, Ram, vm_name) execute(command, ignore_errors=False) print "Launched servermanager VM, Waiting for the VM to come up...." self.verify_sm_vm(vm)
def reimage_vms(self): vm_list = CONF['DEFAULTS']['reimagevms'] print "Re-Imaging %s VM's. Begins.." % (vm_list) #add cluster, image and server1,2 print "Adding cluster.json to servermanager" cmd = 'server-manager delete cluster --cluster_id {}'.format( sk_cluster_id) self.sm_vm.execute_cmd(cmd, timeout=30) cmd = 'server-manager add cluster -f {}/json-files/cluster.json'.format( self.sm_home) self.sm_vm.execute_cmd(cmd, timeout=30) res = self.sm_api.cluster_get(sk_cluster_id) if res != 200: print "\n" print "Command: %s" % (cmd) print "Response: %s" % (res) print "======================================" print "Error in adding cluster.json to servermanager" print "======================================" sys.exit(1) else: print "Successfully added cluster.json to servermanager" print "Adding image.json to servermanager" cmd = 'server-manager delete image --image_id {}'.format(ubuntu_img_id) self.sm_vm.execute_cmd(cmd, timeout=120) cmd = 'server-manager add image -f {}/json-files/image.json'.format( self.sm_home) self.sm_vm.execute_cmd(cmd, timeout=120) res = self.sm_api.image_get(ubuntu_img_id) if res != 200: print "\n" print "Command: %s" % (cmd) print "Response: %s" % (res) print "======================================" print "Error in adding image.json to servermanager" print "======================================" sys.exit(1) else: print "Successfully added image.json to servermanager" vm_list = CONF['DEFAULTS']['reimagevms'] i = 1 for vm in vm_list: print "Adding server{}.json to servermanager".format(i) cmd = 'server-manager delete server --server_id {}'.format( CONF[vm]['hostname']) self.sm_vm.execute_cmd(cmd, timeout=30) cmd = 'server-manager add server -f {0}/json-files/server{1}.json'.format( self.sm_home, i) self.sm_vm.execute_cmd(cmd, timeout=30) res = self.sm_api.server_get(CONF[vm]['hostname']) if res != 200: print "\n" print "Command: %s" % (cmd) print "Response: %s" % (res) print "=============================================" print "Error in adding server{}.json to servermanager".format( i) print "=============================================" sys.exit(1) else: print "Successfully added server{}.json to servermanager".format( i) print "Issuing reimage server{}".format(i) cmd = 'yes y | server-manager reimage --server_id {}'.format( CONF[vm]['hostname']) self.sm_vm.execute_cmd(cmd, timeout=10) i += 1 for vm in vm_list: print "Re-imaging %s VM" % CONF[vm]['hostname'] cmd = "virsh destroy %s" % CONF[vm]['hostname'] execute(cmd, ignore_errors=True) waiting(5) cmd = "virsh undefine %s" % CONF[vm]['hostname'] execute(cmd, ignore_errors=True) waiting(5) cmd = "virt-install --os-variant=ubuntutrusty --os-type=linux --arch=x86_64 --network mac=%s,bridge=mgmtbr --network mac=%s,bridge=ctrldatabr --file=/var/lib/libvirt/images/%s.qcow2 --graphics vnc,password=%s --noautoconsole --vcpus=%s --ram=%s --pxe --name %s --autostart --file-size=%s" % ( CONF[vm]['management_mac'], CONF[vm]['ctrldata_mac'], CONF[vm]['hostname'], CONF[vm]['local_password'], CONF[vm]['vcpus'], CONF[vm]['memory'], CONF[vm]['hostname'], CONF[vm]['harddisk']) execute(cmd, ignore_errors=False) waiting(10) print "Waiting for %s VM to reimage" % vm self.check_reimage_status(vm_list) print "Re-Imaging %s VM's. Ends.." % vm_list