def copy_ucloud_image(self):
print "Coping ubuntu cloud image to libvirt/images folder."
waiting(10)
global cloudimg
cloudimg = CONF['DEFAULTS']['cloudimage']
command = 'cp %s/%s /var/lib/libvirt/images/' % (sk_img_path, cloudimg)
execute(command, ignore_errors=False)
def check_reimage_vm_status(self, vmname):
count = 40
expected_res = "reimage_completed"
for _ in range(count):
#Get reimage status
res = self.sm_api.server_get(server_id=vmname, status=True)
print "Waiting for %s VM to get re-imaged.." % vmname
print "Expected status: %s :: Current status: %s " % (
expected_res, res)
if res == expected_res:
waiting(10)
cmd = "virsh list --all"
res = execute(cmd, ignore_errors=True)
if re.search(r'shut off', res, re.M | re.I):
print "Re-imaged VM is shut-off, starting now..."
cmd = "virsh start " + vmname
res = execute(cmd, ignore_errors=True)
waiting(5)
else:
print "None of the re-imaged VM's are shut-off"
return
else:
print "Retry after 30sec"
waiting(30)
print "\n"
print "==========================================================="
print "VM %s did not complete Re-imaging, hence exiting..." % vmname
print "==========================================================="
sys.exit(1)
def copy_seed_ks_to_websvr(self):
print "Copying ubuntu seed and kickstart files on to jumphost webserver"
cmd = 'cp {0}/contrail-ubuntu-sk.seed {1}/'.format(
from_project_root('contrail'), sk_img_path)
execute(cmd, ignore_errors=True)
cmd = 'cp {0}/contrail-ubuntu-sk.ks {1}/'.format(
from_project_root('contrail'), sk_img_path)
execute(cmd, ignore_errors=True)
def delete_sm(self):
sm_name = CONF['servermanager']['hostname']
print "Deleting SM-VM %s" % (sm_name)
cmd = "virsh destroy %s" % sm_name
execute(cmd, ignore_errors=True)
waiting(5)
cmd = "virsh undefine %s" % sm_name
execute(cmd, ignore_errors=True)
waiting(5)
def delete_vms(self):
vm_list = CONF['DEFAULTS']['reimagevms']
print "Deleting %s VMS" % (vm_list)
for vm in vm_list:
cmd = "virsh destroy %s" % CONF[vm]['hostname']
execute(cmd, ignore_errors=True)
waiting(5)
cmd = "virsh undefine %s" % CONF[vm]['hostname']
execute(cmd, ignore_errors=True)
waiting(5)
def check_connectivity_to_allnodes(self):
print "Checking connectivity to ctrldata IPs of all nodes in cluster"
all_list = CONF['DEFAULTS']['bms'] + CONF['DEFAULTS']['reimagevms']
for inventory in all_list:
ctrld_ip = get_ip(CONF[inventory]['ctrldata_address'])
cmd = 'ping -c 4 %s' % (ctrld_ip)
res = execute(cmd, ignore_errors=True)
if re.search(r'4 received', res, re.M | re.I):
print "CTRLDATA IP(%s)of (%s) is reachable from jumphost." % (
ctrld_ip, inventory)
else:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "=========================================================================="
print "CTRLDATA IP(%s)of (%s) is not reachable from jumphost.check configuration" % (
ctrld_ip, inventory)
print "=========================================================================="
sys.exit(1)
def cleanup_jumphost(self):
svr1 = CONF['DEFAULTS']['bms'][0]
print "Cleaning up jumphost server (%s) configuration" % (svr1)
m_intf = CONF[svr1]['management_interface']
c_intf = CONF[svr1]['ctrldata_interface']
m_ip = get_ip(CONF[svr1]['management_address'])
m_netmask = get_netmask(CONF[svr1]['management_address'])
c_ip = get_ip(CONF[svr1]['ctrldata_address'])
c_netmask = get_netmask(CONF[svr1]['ctrldata_address'])
cmd = "/sbin/ifconfig %s 0.0.0.0" % m_intf
execute(cmd, ignore_errors=True)
cmd = "/sbin/ifconfig %s %s netmask %s up" % (m_intf, m_ip, m_netmask)
execute(cmd, ignore_errors=False)
cmd = "/sbin/ifconfig mgmtbr 0.0.0.0 down && brctl delif mgmtbr %s && brctl delbr mgmtbr" % (
m_intf)
execute(cmd, ignore_errors=False)
cmd = "/sbin/ifconfig %s 0.0.0.0" % c_intf
execute(cmd, ignore_errors=False)
cmd = "/sbin/ifconfig %s %s netmask %s up" % (c_intf, c_ip, c_netmask)
execute(cmd, ignore_errors=False)
cmd = "/sbin/ifconfig ctrldatabr 0.0.0.0 down && brctl delif ctrldatabr %s && brctl delbr ctrldatabr" % (
c_intf)
execute(cmd, ignore_errors=False)
cmd = "/sbin/ip route | grep default | awk '{print $3}'"
ext_gw = execute(cmd, ignore_errors=False)
cmd = "brctl show extbr | grep extbr | awk '{print $4}'"
ext_intf = execute(cmd, ignore_errors=False)
cmd = "/sbin/ifconfig extbr | grep inet | grep -oE \"\\b([0-9]{1,3}\.){3}[0-9]{1,3}\\b\""
ext_intf_ips = execute(cmd, ignore_errors=False)
ext_intf_ip = ext_intf_ips.splitlines()[0]
ext_intf_netmask = ext_intf_ips.splitlines()[2]
cmd = "/sbin/ifconfig %s %s netmask %s up" % (ext_intf, ext_intf_ip,
ext_intf_netmask)
execute(cmd, ignore_errors=False)
cmd = "/sbin/route add default gw %s dev %s" % (ext_gw, ext_intf)
execute(cmd, ignore_errors=False)
cmd = "/sbin/ifconfig extbr 0.0.0.0 down && brctl delif extbr %s && brctl delbr extbr" % (
ext_intf)
execute(cmd, ignore_errors=False)
cmd = "mv /etc/network/interfaces.bak /etc/network/interfaces"
execute(cmd, ignore_errors=False)
cmd = "/bin/rm -rf /var/www/html/pockit_images/"
execute(cmd, ignore_errors=False)
cmd = "/bin/rm -rf /root/sm_vm/"
execute(cmd, ignore_errors=False)
def configure_and_verify_vmx(self):
print "Configuring and verifying vMX management IP reachability"
print "Waiting for vMX to boot up.."
waiting(200)
global vmx_folder
self.svr2_bms.chdir(vmx_folder)
cmd = 'expect vmx-init-cfg.exp'
op = self.svr2_bms.execute_cmd(cmd, timeout=120)
waiting(10)
vmx_re_ipaddr = get_ip(CONF['DEFAULTS']['vmx_vcp_address'])
cmd = 'ping -c 5 {}'.format(vmx_re_ipaddr)
res = self.svr2_bms.execute_cmd(cmd, timeout=60)
if re.search(r'5 received', res, re.M | re.I):
print "vMX mgmt IP %s is reachable, its UP and RUNNING" % (
vmx_re_ipaddr)
else:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "============================================================"
print "vMX mgmt IP %s is not reachable, please check configuration!" % (
vmx_re_ipaddr)
print "============================================================"
sys.exit(1)
svr2 = self.svr2_bms_name
print "Configuring ctrldata interface with IP on server(%s)" % (
self.svr2_bms_name)
cmd = '/sbin/ifconfig {0} 0.0.0.0 up'.format(
CONF[svr2]['ctrldata_interface'])
res = self.svr2_bms.execute_cmd(cmd, timeout=20)
ctl_ip = get_ip(CONF[svr2]['ctrldata_address'])
ctl_netmask = get_netmask(CONF[svr2]['ctrldata_address'])
cmd = '/sbin/ifconfig vmx_link1 {0} netmask {1} up'.format(
ctl_ip, ctl_netmask)
res = self.svr2_bms.execute_cmd(cmd, timeout=20)
waiting(5)
vmx_loopback_ip = CONF['DEFAULTS']['vmx_loopback_ip']
vmx_ctl_ip = get_ip(CONF['vmx']['ctrldata_address'])
cmd = 'route add -host {0}/32 gw {1}'.format(vmx_loopback_ip,
vmx_ctl_ip)
res = self.svr2_bms.execute_cmd(cmd, timeout=40)
waiting(10)
#on jumphost add route to vmx loopback ip
command = 'route add -host {0}/32 gw {1}'.format(
vmx_loopback_ip, ctl_ip)
res = execute(command, ignore_errors=False)
waiting(5)
#ping vMX loopback ip
print "Checking vmx_loopback IP(%s) is reachable" % (vmx_loopback_ip)
cmd = 'ping -c 5 {}'.format(vmx_loopback_ip)
res = self.svr2_bms.execute_cmd(cmd, timeout=60)
if re.search(r'5 received', res, re.M | re.I):
print "vMX loopback IP %s is reachable from server(%s)" % (
vmx_loopback_ip, svr2)
else:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "=========================================================================="
print "vMX loopback IP %s is not reachable from server(%s). Check configuration." % (
vmx_loopback_ip, svr2)
print "=========================================================================="
sys.exit(1)
def __init__(self):
global jinja_env
jinja_env = Environment(
loader=FileSystemLoader(from_project_root('smgr')))
# configure server1 bridge ips
svr1 = CONF['DEFAULTS']['bms'][0]
print "Checking ntp server reachability from jumphost(%s)" % (svr1)
sm_ntp_server = CONF['DEFAULTS']['ntp_servers']
res = os.system('ping -c 3 ' + sm_ntp_server)
if res == 0:
print "NTP server %s is reachable" % (sm_ntp_server)
else:
print "Response: %s" % (res)
print "NTP server %s is not reachable" % (sm_ntp_server)
sys.exit(1)
print "Checking mgmtbr, ctrldatabr bridge configuration on jumphost server."
m_intf = CONF[svr1]['management_interface']
c_intf = CONF[svr1]['ctrldata_interface']
m_ip = get_ip(CONF[svr1]['management_address'])
m_netmask = get_netmask(CONF[svr1]['management_address'])
c_ip = get_ip(CONF[svr1]['ctrldata_address'])
c_netmask = get_netmask(CONF[svr1]['ctrldata_address'])
command = 'brctl show mgmtbr'
res1 = execute(command, ignore_errors=False)
command = 'brctl show ctrldatabr'
res2 = execute(command, ignore_errors=False)
if None not in (re.search(r'%s' % m_intf, res1, re.M | re.I),
re.search(r'%s' % c_intf, res2, re.M | re.I)):
print "Jumphost server bridge interface configuration exists."
else:
print "Configuring jumphost server bridge interfaces. Begins.."
command = '/sbin/ifconfig %s 0.0.0.0 up' % (m_intf)
execute(command, ignore_errors=False)
command = '/sbin/ifconfig mgmtbr %s netmask %s up' % (m_ip,
m_netmask)
execute(command, ignore_errors=False)
command = 'brctl addif mgmtbr %s' % (m_intf)
execute(command, ignore_errors=False)
command = '/sbin/ifconfig %s 0.0.0.0 up' % (c_intf)
execute(command, ignore_errors=False)
command = '/sbin/ifconfig ctrldatabr %s netmask %s up' % (
c_ip, c_netmask)
execute(command, ignore_errors=False)
command = 'brctl addif ctrldatabr %s' % (c_intf)
execute(command, ignore_errors=False)
print "Configuring jumphost server bridge interfaces. Ends.."
#update /etc/network/interfaces file with this ip
cmd = 'cat /etc/network/interfaces'
res = execute(cmd, ignore_errors=False)
if re.search(r'mgmtbr|ctrldatabr', res, re.M | re.I):
print "mgmtbr/ctrldatabr interface configuration exists"
else:
print "mgmtbr/ctrldatabr interface configuration does not exists, hence configuring"
command = 'sudo echo -ne \"auto {}\n iface {} inet manual\n auto mgmtbr\n iface mgmtbr inet static\n address {}\n netmask {}\n bridge_ports {}\n bridge_stp off\n bridge_fd 0\n bridge_maxwait 0\n\" >> /etc/network/interfaces'.format(
m_intf, m_intf, m_ip, m_netmask, m_intf)
execute(command, ignore_errors=False)
command = 'sudo echo -ne \"auto {}\n iface {} inet manual\n auto ctrldatabr\n iface ctrldatabr inet static\n address {}\n netmask {}\n bridge_ports {}\n bridge_stp off\n bridge_fd 0\n bridge_maxwait 0\n\" >> /etc/network/interfaces'.format(
c_intf, c_intf, c_ip, c_netmask, c_intf)
execute(command, ignore_errors=False)
def create_sm_vm(self):
vm = 'servermanager'
vm_name = CONF[vm]['hostname']
sm_m_ip = get_ip(CONF[vm]['management_address'])
global sm_creation
sm_creation = False
#Check if the SM VM is created and service is UP
command = 'virsh list | grep %s | awk \'{print $3}\'' % (vm_name)
res = execute(command, ignore_errors=True)
vm_running = False
if res == 'running':
print "SM VM is created and UP and RUNNING"
vm_running = True
#check if VM is reachable if so, check service is running
#ping SM
vm_pingable = False
res = os.system('ping -c 5 ' + sm_m_ip)
if res == 0:
print "Servermanger VM is UP and reachable"
vm_pingable = True
sm_local_user = CONF[vm]['local_user']
sm_local_passwd = CONF[vm]['local_password']
sm_vm = RemoteConnection()
sm_vm.connect(sm_m_ip,
username=sm_local_user,
password=sm_local_passwd)
cmd = 'sudo service contrail-server-manager status'
res = sm_vm.execute_cmd(cmd, timeout=10)
if re.search(r'not|unrecognized', res, re.M | re.I):
sm_vm_service = False
else:
print "contrail-server-manager service is running."
sm_vm_service = True
else:
print "Servermanager VM is not reachable or does not exists"
if vm_running and vm_pingable and sm_vm_service:
print "The existing servermanager VM is UP and RUNNING, hence not deleting and creating a new one"
#adding route to ipmi subnet
print "Adding route to IPMI network to reach from servermanager"
ipmi_network = CONF['DEFAULTS']['ipmi_network']
sm_gw = CONF[vm]['gateway']
cmd = 'route add -net {0} gw {1}'.format(ipmi_network, sm_gw)
res = sm_vm.execute_cmd(cmd, timeout=10)
else:
sm_creation = True
print "Creating servermanager VM with name " + vm_name
print "Deleting existing servermanager VM"
command = 'virsh destroy ' + vm_name
execute(command, ignore_errors=True)
waiting(3)
command = 'virsh undefine ' + vm_name
execute(command, ignore_errors=True)
waiting(3)
vncpasswd = CONF[vm]['local_password']
OS = '--os-variant=ubuntutrusty --os-type=linux'
ARCH = '--arch=x86_64'
NET1 = '--network bridge=mgmtbr'
NET2 = '--network bridge=extbr'
Disk = '--boot hd'
Gr = '--graphics vnc,password={} --noautoconsole'.format(vncpasswd)
Cpu = '--vcpus=%s' % CONF[vm]['vcpus']
Ram = '--ram=%s' % CONF[vm]['memory']
Src = '--disk path=/var/lib/libvirt/images/%s.qcow2,format=qcow2,bus=virtio --import --autostart --disk path=/var/lib/libvirt/images/%s_init.iso,bus=virtio,device=cdrom' % (
vm_name, vm_name)
#print OS, ARCH, NET1, NET2, Disk, Gr, Cpu, Ram, Src
command = 'cp -f /var/lib/libvirt/images/%s /var/lib/libvirt/images/%s.qcow2' % (
cloudimg, vm_name)
execute(command, ignore_errors=False)
command = 'qemu-img resize /var/lib/libvirt/images/%s.qcow2 +%sG' % (
vm_name, CONF[vm]['harddisk'])
execute(command, ignore_errors=False)
# create user-data and meta-data file
print "Creating user-data and meta-data files to boot ubuntu cloud image. Begins.."
vm_disk_path = '/root/sm_vm/{0}'.format(vm_name)
command = 'rm -rf {0}'.format(vm_disk_path)
execute(command, ignore_errors=False)
waiting(3)
command = 'mkdir -p {0}'.format(vm_disk_path)
execute(command, ignore_errors=False)
sm_root_password = CONF['DEFAULTS']['root_password']
sm_dns_search = CONF[vm]['dns_search']
sm_dns_server = CONF[vm]['dns_servers']
sm_local_user = CONF[vm]['local_user']
sm_local_passwd = CONF[vm]['local_password']
sm_m_ip = get_ip(CONF[vm]['management_address'])
sm_m_netmask = get_netmask(CONF[vm]['management_address'])
sm_ext_ip = get_ip(CONF['DEFAULTS']['sm_ext_address'])
sm_ext_netmask = get_netmask(CONF['DEFAULTS']['sm_ext_address'])
sm_ext_gateway = CONF['DEFAULTS']['sm_ext_gateway']
command = 'cat /home/{0}/.ssh/id_rsa.pub'.format(sm_local_user)
sm_rsa_user = execute(command, ignore_errors=False)
command = 'cat /root/.ssh/id_rsa.pub'.format(sm_local_user)
sm_rsa_root = execute(command, ignore_errors=False)
print "Creating user-data file required for cloud init image.."
userdata_tmpl = jinja_env.get_template('user-data')
userdata = userdata_tmpl.render(password=sm_root_password,
local_user=sm_local_user,
local_password=sm_local_passwd,
dns_server=sm_dns_server,
ssh_rsa_user=sm_rsa_user,
ssh_rsa_root=sm_rsa_root,
dns_search=sm_dns_search)
fobj = open(vm_disk_path + '/user-data', 'w')
if hasattr(userdata, '__iter__'):
for data in userdata:
fobj.write(data)
fobj.close()
else:
fobj.write(userdata)
fobj.close()
print "Creating meta-data file required for cloud init image.."
metadata_tmpl = jinja_env.get_template('meta-data')
metadata = metadata_tmpl.render(hostname=vm_name,
mgmt_ip=sm_m_ip,
mgmt_netmask=sm_m_netmask,
ext_ip=sm_ext_ip,
ext_netmask=sm_ext_netmask,
ext_gateway=sm_ext_gateway)
fobj = open(vm_disk_path + '/meta-data', 'w')
if hasattr(metadata, '__iter__'):
for data in metadata:
fobj.write(data)
fobj.close()
else:
fobj.write(metadata)
fobj.close()
print "Creating user-data and meta-data files to boot ubuntu cloud image. Ends.."
# Launch VM
print "Launching servermanager VM..."
cwddir = os.getcwd()
command = 'cd {}'.format(vm_disk_path + '/')
os.chdir(vm_disk_path)
command = 'cloud-localds -v /var/lib/libvirt/images/{}_init.iso user-data meta-data'.format(
vm_name)
os.system(command)
os.chdir(cwddir)
command = 'virt-install {} {} {} {} {} {} {} {} {} --name={}'.format(
OS, ARCH, NET1, NET2, Disk, Src, Gr, Cpu, Ram, vm_name)
execute(command, ignore_errors=False)
print "Launched servermanager VM, Waiting for the VM to come up...."
self.verify_sm_vm(vm)
def reimage_vms(self):
vm_list = CONF['DEFAULTS']['reimagevms']
print "Re-Imaging %s VM's. Begins.." % (vm_list)
#add cluster, image and server1,2
print "Adding cluster.json to servermanager"
cmd = 'server-manager delete cluster --cluster_id {}'.format(
sk_cluster_id)
self.sm_vm.execute_cmd(cmd, timeout=30)
cmd = 'server-manager add cluster -f {}/json-files/cluster.json'.format(
self.sm_home)
self.sm_vm.execute_cmd(cmd, timeout=30)
res = self.sm_api.cluster_get(sk_cluster_id)
if res != 200:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "======================================"
print "Error in adding cluster.json to servermanager"
print "======================================"
sys.exit(1)
else:
print "Successfully added cluster.json to servermanager"
print "Adding image.json to servermanager"
cmd = 'server-manager delete image --image_id {}'.format(ubuntu_img_id)
self.sm_vm.execute_cmd(cmd, timeout=120)
cmd = 'server-manager add image -f {}/json-files/image.json'.format(
self.sm_home)
self.sm_vm.execute_cmd(cmd, timeout=120)
res = self.sm_api.image_get(ubuntu_img_id)
if res != 200:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "======================================"
print "Error in adding image.json to servermanager"
print "======================================"
sys.exit(1)
else:
print "Successfully added image.json to servermanager"
vm_list = CONF['DEFAULTS']['reimagevms']
i = 1
for vm in vm_list:
print "Adding server{}.json to servermanager".format(i)
cmd = 'server-manager delete server --server_id {}'.format(
CONF[vm]['hostname'])
self.sm_vm.execute_cmd(cmd, timeout=30)
cmd = 'server-manager add server -f {0}/json-files/server{1}.json'.format(
self.sm_home, i)
self.sm_vm.execute_cmd(cmd, timeout=30)
res = self.sm_api.server_get(CONF[vm]['hostname'])
if res != 200:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "============================================="
print "Error in adding server{}.json to servermanager".format(
i)
print "============================================="
sys.exit(1)
else:
print "Successfully added server{}.json to servermanager".format(
i)
print "Issuing reimage server{}".format(i)
cmd = 'yes y | server-manager reimage --server_id {}'.format(
CONF[vm]['hostname'])
self.sm_vm.execute_cmd(cmd, timeout=10)
i += 1
for vm in vm_list:
print "Re-imaging %s VM" % CONF[vm]['hostname']
cmd = "virsh destroy %s" % CONF[vm]['hostname']
execute(cmd, ignore_errors=True)
waiting(5)
cmd = "virsh undefine %s" % CONF[vm]['hostname']
execute(cmd, ignore_errors=True)
waiting(5)
cmd = "virt-install --os-variant=ubuntutrusty --os-type=linux --arch=x86_64 --network mac=%s,bridge=mgmtbr --network mac=%s,bridge=ctrldatabr --file=/var/lib/libvirt/images/%s.qcow2 --graphics vnc,password=%s --noautoconsole --vcpus=%s --ram=%s --pxe --name %s --autostart --file-size=%s" % (
CONF[vm]['management_mac'], CONF[vm]['ctrldata_mac'],
CONF[vm]['hostname'], CONF[vm]['local_password'],
CONF[vm]['vcpus'], CONF[vm]['memory'], CONF[vm]['hostname'],
CONF[vm]['harddisk'])
execute(cmd, ignore_errors=False)
waiting(10)
print "Waiting for %s VM to reimage" % vm
self.check_reimage_status(vm_list)
print "Re-Imaging %s VM's. Ends.." % vm_list
