def check_contrail_status(self):
print "Checking contrail-status on the contrail controller node"
cc_name = CONF['DEFAULTS']['reimagevms'][0]
os_name = CONF['DEFAULTS']['reimagevms'][1]
cc_ip = get_ip(CONF[cc_name]['management_address'])
cc_ctrld_ip = get_ip(CONF[cc_name]['ctrldata_address'])
os_ctrld_ip = get_ip(CONF[os_name]['ctrldata_address'])
user = CONF['DEFAULTS']['root_username']
passwd = CONF['DEFAULTS']['root_password']
reconnect = RemoteConnection()
reconnect.connect(cc_ip, username=user, password=passwd)
cmd = 'contrail-status'
res = reconnect.execute_cmd(cmd, timeout=20)
if re.search(r'NTP state|inactive', res, re.M | re.I):
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "========================================================================================="
print "contrail services on controller show in NTP state unsynchronized/inactive. please check."
print "========================================================================================="
sys.exit(1)
else:
print "All the services on contrail controller shows active. Installation successfully done."
print res
print "Access to Contrail web-ui from jumphost({}): https://{}:8143/".format(
svr1, cc_ctrld_ip)
print "Access to Openstack Horizon web-ui from jumphost({}): http://{}/horizon".format(
svr1, os_ctrld_ip)
def verify_sm_on_vm(self):
sm_creation = True
if sm_creation:
print "\nVerifying contrail-server-manager service is running on the VM"
vm = 'servermanager'
sm_m_ip = get_ip(CONF[vm]['management_address'])
sm_local_user = CONF[vm]['local_user']
sm_local_passwd = CONF[vm]['local_password']
svr1 = CONF['DEFAULTS']['bms'][0]
sm_vm = RemoteConnection()
sm_vm.connect(sm_m_ip,
username=sm_local_user,
password=sm_local_passwd)
cmd = 'sudo service contrail-server-manager status'
res = sm_vm.execute_cmd(cmd, timeout=10)
#print res
if re.search(r'not', res, re.M | re.I):
print "\n"
print "Command:%s" % (cmd)
print "Response:%s" % (res)
print "======================================================================"
print "contrail-server-manager service is not running...please check the logs"
print "======================================================================"
sys.exit(1)
else:
print "contrail-server-manager service is running."
sm_ext_ip = get_ip(CONF['DEFAULTS']['sm_ext_address'])
print "Access to ServerManager web-ui from jumphost({}): https://{}:9143/".format(
svr1, sm_m_ip)
print "Do ssh tunnel command, ssh -f -N -L 9143:{}:9143 root@{}".format(
sm_m_ip, sm_ext_ip)
print "Access to ServerManager web-ui from laptop to monitor provisioning \
and health of servers: https://127.0.0.1:9143/"
#adding route to ipmi subnet
print "Adding route to IPMI network to reach from servermanager"
ipmi_network = CONF['DEFAULTS']['ipmi_network']
sm_gw = CONF[vm]['gateway']
cmd = 'sudo route add -net {0} gw {1}'.format(
ipmi_network, sm_gw)
res = sm_vm.execute_cmd(cmd, timeout=10)
def remove_server2_from_cluster(self):
bms2 = CONF['DEFAULTS']['bms'][1]
print "\nRemoving %s from contrail cluster, as it will be used to deploy vMX on it" % (
bms2)
svr2_ip = get_ip(CONF[bms2]['management_address'])
svr2_netmask = get_netmask(CONF[bms2]['management_address'])
svr2_usr = CONF['DEFAULTS']['root_username']
svr2_passwd = CONF['DEFAULTS']['root_password']
svr2_connect = RemoteConnection()
svr2_connect.connect(svr2_ip, username=svr2_usr, password=svr2_passwd)
#reboot
bms2_name = CONF[bms2]['hostname']
print "Rebooting %s" % (bms2_name)
cmd = 'reboot'
svr2_connect.execute_cmd(cmd, timeout=10)
waiting(10)
print "Deleting %s from the cluster" % (bms2_name)
cmd = 'server-manager delete server --server_id %s' % (bms2_name)
res = self.sm_vm.execute_cmd(cmd, timeout=10)
#wait till the bms to comes up
count = 40
for _ in range(count):
#ping server2
res = os.system('ping -c 3 ' + svr2_ip)
if res == 0:
print "%s is reachable and is UP" % (bms2_name)
waiting(60)
return
else:
print "Waiting for %s to come UP.." % (bms2_name)
print "Retry after 30sec"
waiting(30)
print "\n"
print "========================================================================="
print "{0} is not reachable, did not boot up with correct mgmt_address {1}".format(
bms2_anme, svr2_ip)
print "========================================================================="
sys.exit(1)
class vmxDeploy(object):
def __init__(self):
global jinja_env
jinja_env = Environment(
loader=FileSystemLoader(from_project_root('vmx')))
svr1name = CONF['DEFAULTS']['bms'][0]
self.svr2_bms_name = CONF['DEFAULTS']['bms'][1]
self.server1_ip = get_ip(CONF[svr1name]['management_address'])
server2_ip = get_ip(CONF[self.svr2_bms_name]['management_address'])
server2_user = CONF['DEFAULTS']['root_username']
server2_passwd = CONF['DEFAULTS']['root_password']
self.svr2_bms = RemoteConnection()
self.svr2_bms.connect(server2_ip,
username=server2_user,
password=server2_passwd)
cmd = 'eval echo ~$USER'
self.svr2_bms_home = self.svr2_bms.execute_cmd(cmd, timeout=10)
def copy_vmx_images(self):
print "Copying vMX image onto %s" % self.svr2_bms_name
vmximage = CONF['DEFAULTS']['vmximage']
cmd = 'rm -rf images/'
self.svr2_bms.execute_cmd(cmd, timeout=10)
cmd = 'mkdir -p images'
self.svr2_bms.execute_cmd(cmd, timeout=10)
cmd = '{0}/images/'.format(self.svr2_bms_home)
self.svr2_bms.chdir(cmd)
cmd = 'wget -q http://{0}/pockit_images/{1} -O {2}/images/{1}'.format(
self.server1_ip, vmximage, self.svr2_bms_home)
if self.svr2_bms.status_command(cmd) != 0:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "=================================================="
print "Error in downloading vMX image on to %s" % (
self.svr2_bms_name)
print "=================================================="
sys.exit(1)
else:
print "Downloaded vMX image on to %s successfully" % (
self.svr2_bms_name)
def configure_server_uplink_intf(self):
print "Configuring external/internet facing interface and routes"
vm = 'vmx'
intf = CONF[vm]['management_interface']
server_ext_ip = get_ip(CONF[vm]['management_address'])
server_ext_mask = get_netmask(CONF[vm]['management_address'])
gateway = CONF['DEFAULTS']['vmx_ext_gateway']
dns_server = CONF[vm]['dns_servers']
dns_search = CONF[vm]['dns_search']
cmd = '/sbin/ifconfig {0} {1} netmask {2} up'.format(
intf, server_ext_ip, server_ext_mask)
self.svr2_bms.execute_cmd(cmd, timeout=10)
cmd = 'route delete default'
self.svr2_bms.execute_cmd(cmd, timeout=10)
cmd = 'route add default gw {}'.format(gateway)
self.svr2_bms.execute_cmd(cmd, timeout=10)
#update /etc/network/interfaces file with this ip
cmd = 'cat /etc/network/interfaces'
res = self.svr2_bms.execute_cmd(cmd, timeout=10)
if re.search(r'%s' % intf, res, re.M | re.I):
print "external interface configuration exists."
else:
print "external interface configuration does not exists, hence configuring"
cmd = '(echo ;echo auto {}; echo iface {} inet static; echo address {}; echo netmask {}; echo gateway {}; echo dns-nameservers {}; echo dns-search {}) >> /etc/network/interfaces'.format(
intf, intf, server_ext_ip, server_ext_mask, gateway,
dns_server, dns_search)
self.svr2_bms.execute_cmd(cmd, timeout=30)
#add nameserver in /etc/resolv.conf
cmd = '(echo nameserver {}) >> /etc/resolv.conf'.format(dns_server)
res = self.svr2_bms.execute_cmd(cmd, timeout=10)
def update_vmx_packages(self):
#check if internet is reachable(8.8.8.8, www.google.com"
waiting(5)
cmd = 'ping -c 5 %s' % (internet_www)
if self.svr2_bms.status_command(cmd) != 0:
print "\n"
print "Command: %s" % (cmd)
print "========================================================================="
print "Internet URL({0}) is not reachable from {1}, please check route and DNS".format(
internet_www, self.svr2_bms_name)
print "========================================================================="
sys.exit(1)
else:
print "Internet URL({0}) is reachable from {1}".format(
internet_www, self.svr2_bms_name)
#apt-get update, upgrade kernel
print "Upgrading packages to install vMX"
cmd = 'cp /etc/apt/sources.list.save /etc/apt/sources.list'
self.svr2_bms.execute_cmd(cmd, timeout=10)
cmd = 'sed -i \'/dl.google/s/^/#/\' /etc/apt/sources.list'
self.svr2_bms.execute_cmd(cmd, timeout=10)
cmd = 'sed -i \'/medibuntu/s/^/#/\' /etc/apt/sources.list'
self.svr2_bms.execute_cmd(cmd, timeout=10)
cmd = 'apt-get update'
self.svr2_bms.execute_cmd(cmd, timeout=240)
cmd = 'apt-get install -y linux-firmware linux-image-3.13.0.32-generic linux-image-extra-3.13.0.32-generic linux-headers-3.13.0.32-generic'
self.svr2_bms.execute_cmd(cmd, timeout=300)
cmd = 'update-grub'
self.svr2_bms.execute_cmd(cmd, timeout=120)
cmd = 'apt-get install -y expect bridge-utils qemu-kvm libvirt-bin python-netifaces vnc4server libyaml-dev python-yaml numactl libparted0-dev libpciaccess-dev libnuma-dev libyajl-dev libxml2-dev libglib2.0-dev libnl-dev python-pip python-dev libxml2-dev libxslt-dev'
self.svr2_bms.execute_cmd(cmd, timeout=600)
#install upgrade libvirt 1.2.19
print "Upgrading libvirt version to 1.2.19 on %s" % (
self.svr2_bms_name)
self.svr2_bms.chdir('/tmp')
cmd = 'rm -rf libvirt-1.2.19/'
self.svr2_bms.execute_cmd(cmd, timeout=10)
cmd = 'rm -rf libvirt-1.2.19.*'
self.svr2_bms.execute_cmd(cmd, timeout=10)
cmd = 'wget -q http://{0}/pockit_images/{1} -O {1}'.format(
self.server1_ip, sk_libvirt_img)
if self.svr2_bms.status_command(cmd) != 0:
print "\n"
print "Command: %s" % (cmd)
print "=================================================="
print "Error in downloading libvirt image on to %s" % (
self.svr2_bms_name)
print "=================================================="
sys.exit(1)
else:
print "Downloaded libvirt image on to %s successfully" % (
self.svr2_bms_name)
cmd = 'sudo service libvirt-bin stop'
res = self.svr2_bms.execute_cmd(cmd, timeout=40)
cmd = 'tar -xzf libvirt-1.2.19.tar.gz'
self.svr2_bms.execute_cmd(cmd, timeout=40)
self.svr2_bms.chdir('/tmp/libvirt-1.2.19/')
cmd = './configure --prefix=/usr --localstatedir=/ --with-numactl'
self.svr2_bms.execute_cmd(cmd, timeout=300)
self.svr2_bms.chdir('/tmp/libvirt-1.2.19/')
cmd = 'make'
self.svr2_bms.execute_cmd(cmd, timeout=300)
self.svr2_bms.chdir('/tmp/libvirt-1.2.19/')
cmd = 'sudo make install && sudo ldconfig'
self.svr2_bms.execute_cmd(cmd, timeout=300)
#check if service is running
cmd = 'sudo service libvirt-bin start'
self.svr2_bms.execute_cmd(cmd, timeout=10)
cmd = 'sudo service libvirt-bin status'
res = self.svr2_bms.execute_cmd(cmd, timeout=10)
if re.search(r'running', res, re.M | re.I):
print "libvirt-bin service is running after upgrading to 1.2.19 version"
else:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "===================================================="
print "libvirt-bin service is not running! please check it"
print "===================================================="
sys.exit(1)
cmd = 'sudo libvirtd --version'
res = self.svr2_bms.execute_cmd(cmd, timeout=10)
if re.search(r'19', res, re.M | re.I):
print "Upgraded libvirt-bin version 1.2.19 is running"
else:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "==================================================================="
print "Upgraded libvirt-bin version 1.2.19 is not running! please check it"
print "==================================================================="
sys.exit(1)
def create_vmx_conf(self, re_img, fpc_img, hdd_img):
print 'Creating vMX confguration file'
vmx_conf_template = jinja_env.get_template('vmx.conf')
user_data = vmx_conf_template.render(
vmx_hostname=CONF['vmx']['hostname'],
vmx_ext_intf=CONF['vmx']['management_interface'],
vmx_re_img=re_img,
vmx_re_hdd=hdd_img,
vmx_pfe_img=fpc_img,
vmx_ctl_ip_addr=get_ip(CONF['DEFAULTS']['vmx_vcp_address']),
vmx_pfe_ip_addr=get_ip(CONF['DEFAULTS']['vmx_vfp_address']))
fobj = open(sk_img_path + '/vmx.conf', 'w')
if hasattr(user_data, '__iter__'):
for data in user_data:
fobj.write(data)
fobj.close()
else:
fobj.write(user_data)
fobj.close()
def create_vmx_junosdev_conf(self):
print 'Creating vmx-junosdev file'
dev_conf_template = jinja_env.get_template('vmx-junosdev.conf')
user_data = dev_conf_template.render(
vmx_hostname=CONF['vmx']['hostname'],
ctrldata_intf=CONF['vmx']['ctrldata_interface'])
fobj = open(sk_img_path + '/vmx-junosdev.conf', 'w')
if hasattr(user_data, '__iter__'):
for data in user_data:
fobj.write(data)
fobj.close()
else:
fobj.write(user_data)
fobj.close()
def create_vmx_default_config(self):
print 'Creating vMX init configuration expect script file'
init_conf_template = jinja_env.get_template('vmx-init-cfg.exp')
user_data = init_conf_template.render(
vmx_hostname=CONF['vmx']['hostname'],
vmx_root_passwd=CONF['DEFAULTS']['root_password'],
vmx_dns_server=CONF['vmx']['dns_servers'],
vmx_ntp_server=CONF['DEFAULTS']['ntp_servers'],
vmx_ctrldata_ip=CONF['vmx']['ctrldata_address'],
vmx_re_ipaddr=CONF['DEFAULTS']['vmx_vcp_address'],
vmx_default_gateway=CONF['DEFAULTS']['vmx_ext_gateway'],
vmx_loopback_ip=CONF['DEFAULTS']['vmx_loopback_ip'])
fobj = open(sk_img_path + '/vmx-init-cfg.exp', 'w')
if hasattr(user_data, '__iter__'):
for data in user_data:
fobj.write(data)
fobj.close()
else:
fobj.write(user_data)
fobj.close()
def pre_up_vmx(self):
print 'Preparing the infra-structure for installing vMX'
global vmx_folder
vmximage = CONF['DEFAULTS']['vmximage']
vmx_base_folder = '{0}/images/'.format(self.svr2_bms_home)
self.svr2_bms.chdir(vmx_base_folder)
cmd = 'tar -xzf %s' % (vmximage)
self.svr2_bms.execute_cmd(cmd, timeout=60)
fname, fext = os.path.splitext(vmximage)
vmx_folder_name = 'vmx-' + fname.split('-')[2]
vmx_img_folder = vmx_base_folder + vmx_folder_name + '/images/'
vmx_folder = vmx_base_folder + vmx_folder_name + '/'
self.svr2_bms.chdir(vmx_img_folder)
cmd = 'ls | grep -E \'qcow2|vFPC\''
op = self.svr2_bms.execute_cmd(cmd, timeout=60)
vmx_img_list = op.splitlines()
re_img = vmx_img_folder + vmx_img_list[0]
fpc_img = vmx_img_folder + vmx_img_list[1]
hdd_img = vmx_img_folder + 'vmxhdd.img'
self.create_vmx_conf(re_img, fpc_img, hdd_img)
self.create_vmx_junosdev_conf()
self.create_vmx_default_config()
self.svr2_bms.chdir(vmx_folder + 'config/')
jumphost_img_url = 'http://{}/pockit_images/'.format(self.server1_ip)
cmd = 'wget -O vmx.conf {0}vmx.conf'.format(jumphost_img_url)
if self.svr2_bms.status_command(cmd) != 0:
print "\n"
print "Command: %s" % (cmd)
print "===================================================="
print "Error in downloading vmx.conf file onto server %s" % (
self.svr2_bms_name)
print "===================================================="
sys.exit(1)
else:
print "Successfully downloaded vmx.conf file onto server %s" % (
self.svr2_bms_name)
cmd = 'wget -O vmx-junosdev.conf {0}vmx-junosdev.conf'.format(
jumphost_img_url)
if self.svr2_bms.status_command(cmd) != 0:
print "\n"
print "Command: %s" % (cmd)
print "============================================================"
print "Error in downloading vmx-junosdev.conf file onto server %s" % (
self.svr2_bms_name)
print "============================================================"
sys.exit(1)
else:
print "Successfully downloaded vmx-junosdev.conf file onto server %s" % (
self.svr2_bms_name)
self.svr2_bms.chdir(vmx_folder)
cmd = 'wget -O vmx-init-cfg.exp {0}vmx-init-cfg.exp'.format(
jumphost_img_url)
if self.svr2_bms.status_command(cmd) != 0:
print "\n"
print "Command: %s" % (cmd)
print "============================================================"
print "Error in downloading vmx-init-cfg.exp file onto server %s" % (
self.svr2_bms_name)
print "============================================================"
sys.exit(1)
else:
print "Successfully downloaded vmx-init-cfg.exp file onto server %s" % (
self.svr2_bms_name)
def deploy_vmx(self):
print "Deploying vMX. Begins.."
global vmx_folder
self.svr2_bms.chdir(vmx_folder)
#delete virbr0 bridge
cmd = '/sbin/ifconfig virbr0 down'
self.svr2_bms.execute_cmd(cmd, timeout=10)
cmd = 'brctl delbr virbr0'
self.svr2_bms.execute_cmd(cmd, timeout=10)
cmd = 'sudo ./vmx.sh -lv --install'
res = self.svr2_bms.execute_cmd(cmd, timeout=360)
if 'error(s)' in res:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "======================================"
print "Error in deploying vMX onto server %s" % (
self.svr2_bms_name)
print "======================================"
sys.exit(1)
else:
print "Successfully deployed vMX onto server %s" % (
self.svr2_bms_name)
cmd = 'sudo ./vmx.sh --bind-dev'
op = self.svr2_bms.execute_cmd(cmd, timeout=40)
cmd = 'sudo ./vmx.sh --bind-check'
op = self.svr2_bms.execute_cmd(cmd, timeout=40)
print "Deploying vMX. Ends.."
def configure_and_verify_vmx(self):
print "Configuring and verifying vMX management IP reachability"
print "Waiting for vMX to boot up.."
waiting(200)
global vmx_folder
self.svr2_bms.chdir(vmx_folder)
cmd = 'expect vmx-init-cfg.exp'
op = self.svr2_bms.execute_cmd(cmd, timeout=120)
waiting(10)
vmx_re_ipaddr = get_ip(CONF['DEFAULTS']['vmx_vcp_address'])
cmd = 'ping -c 5 {}'.format(vmx_re_ipaddr)
res = self.svr2_bms.execute_cmd(cmd, timeout=60)
if re.search(r'5 received', res, re.M | re.I):
print "vMX mgmt IP %s is reachable, its UP and RUNNING" % (
vmx_re_ipaddr)
else:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "============================================================"
print "vMX mgmt IP %s is not reachable, please check configuration!" % (
vmx_re_ipaddr)
print "============================================================"
sys.exit(1)
svr2 = self.svr2_bms_name
print "Configuring ctrldata interface with IP on server(%s)" % (
self.svr2_bms_name)
cmd = '/sbin/ifconfig {0} 0.0.0.0 up'.format(
CONF[svr2]['ctrldata_interface'])
res = self.svr2_bms.execute_cmd(cmd, timeout=20)
ctl_ip = get_ip(CONF[svr2]['ctrldata_address'])
ctl_netmask = get_netmask(CONF[svr2]['ctrldata_address'])
cmd = '/sbin/ifconfig vmx_link1 {0} netmask {1} up'.format(
ctl_ip, ctl_netmask)
res = self.svr2_bms.execute_cmd(cmd, timeout=20)
waiting(5)
vmx_loopback_ip = CONF['DEFAULTS']['vmx_loopback_ip']
vmx_ctl_ip = get_ip(CONF['vmx']['ctrldata_address'])
cmd = 'route add -host {0}/32 gw {1}'.format(vmx_loopback_ip,
vmx_ctl_ip)
res = self.svr2_bms.execute_cmd(cmd, timeout=40)
waiting(10)
#on jumphost add route to vmx loopback ip
command = 'route add -host {0}/32 gw {1}'.format(
vmx_loopback_ip, ctl_ip)
res = execute(command, ignore_errors=False)
waiting(5)
#ping vMX loopback ip
print "Checking vmx_loopback IP(%s) is reachable" % (vmx_loopback_ip)
cmd = 'ping -c 5 {}'.format(vmx_loopback_ip)
res = self.svr2_bms.execute_cmd(cmd, timeout=60)
if re.search(r'5 received', res, re.M | re.I):
print "vMX loopback IP %s is reachable from server(%s)" % (
vmx_loopback_ip, svr2)
else:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "=========================================================================="
print "vMX loopback IP %s is not reachable from server(%s). Check configuration." % (
vmx_loopback_ip, svr2)
print "=========================================================================="
sys.exit(1)
def deploy_sm_on_vm(self):
if sm_creation:
print "Servermanager software deployment is started on VM..."
vm = 'servermanager'
sm_m_ip = get_ip(CONF[vm]['management_address'])
sm_local_user = CONF[vm]['local_user']
sm_local_passwd = CONF[vm]['local_password']
#create dhcp.template file
self.create_dhcp_template(vm)
sm_img = CONF['DEFAULTS']['smimage']
svr1 = CONF['DEFAULTS']['bms'][0]
server1_mgmt_ip = get_ip(CONF[svr1]['management_address'])
sm_vm = RemoteConnection()
sm_vm.connect(sm_m_ip,
username=sm_local_user,
password=sm_local_passwd)
cmd = 'eval echo ~$USER'
sm_home_dir = sm_vm.execute_cmd(cmd, timeout=10)
#check internet reachability from SM
cmd = 'ping -c 4 %s' % internet_ip
res = sm_vm.execute_cmd(cmd, timeout=10)
if re.search(r'4 received', res, re.M | re.I):
print "Internet IP (%s) is reachable from servermanager VM." % internet_ip
else:
print "\n"
print "Command:%s" % (cmd)
print "Response:%s" % (res)
print "========================================================================="
print "Internet IP (%s) is not reachable from servermanager...please check configuration" % internet_ip
print "========================================================================="
sys.exit(1)
cmd = 'ping -c 4 %s' % internet_www
res = sm_vm.execute_cmd(cmd, timeout=10)
if re.search(r'4 received', res, re.M | re.I):
print "Internet URL (%s) is reachable from servermanager VM." % internet_www
else:
print "\n"
print "Command:%s" % (cmd)
print "Response:%s" % (res)
print "========================================================================="
print "Internet URL (%s) is not reachable from servermanager...please check configuration" % internet_www
print "========================================================================="
sys.exit(1)
#create dir images
cmd = 'rm -rf images/; mkdir -p images'
sm_vm.execute_cmd(cmd, timeout=10)
cmd = 'rm -rf json-files/; mkdir -p json-files'
sm_vm.execute_cmd(cmd, timeout=10)
sm_vm.chdir(sm_home_dir + '/images/')
jumphost_img_url = 'http://{}/pockit_images/'.format(
server1_mgmt_ip)
cmd = 'wget -q {0}{1} -O {1}'.format(jumphost_img_url, sm_img)
if sm_vm.status_command(cmd) != 0:
print "\n"
print "Command:%s" % (cmd)
print "=============================================================="
print "Error in downloading servermanager build onto servermanager VM"
print "=============================================================="
sys.exit(1)
else:
print "Successfully downloaded servermanager build onto servermanager VM"
cmd = 'sudo dpkg -i {0}/images/{1}'.format(sm_home_dir, sm_img)
if sm_vm.status_command(cmd) != 0:
print "\n"
print "Command:%s" % (cmd)
print "====================================================="
print "Error in depackaging the servermanager debian package"
print "====================================================="
sys.exit(1)
else:
print "Successfully Depackaged the servermanager debian package"
print "Starting setup.sh --all command, this will take a while"
global kill
global stop
sm_vm.chdir('/opt/contrail/contrail_server_manager/')
queue = Queue.Queue()
t = threading.Thread(target=self.progress_bar_print, args=())
t.start()
try:
#cmd = 'sudo ./setup.sh --sm={0}'.format(sm_img)
cmd = 'sudo ./setup.sh --all'
if sm_vm.status_command(cmd) != 0:
kill = True
stop = True
print "\n"
print "Command:%s" % (cmd)
print "======================================="
print "Error in running setup.sh --all command"
print "======================================="
sys.exit(1)
else:
stop = True
print "\nSuccessfully completed running setup.sh --all command"
except KeyboardInterrupt or EOFError:
print "entering keyboard error"
kill = True
stop = True
sm_vm.chdir(sm_home_dir + '/json-files/')
cmd = 'sudo wget -q {0}dhcp.template -O /etc/cobbler/dhcp.template'.format(
jumphost_img_url)
if sm_vm.status_command(cmd, timeout=90) != 0:
print "\n"
print "Command:%s" % (cmd)
print "=========================================================="
print "Error in copying dhcp.template file on to servermanager VM"
print "=========================================================="
sys.exit(1)
else:
print "Successfully copied dhcp.template file on to servermanager VM"
cmd = 'sudo sed -i \'s/= authn_testing/= authn_configfile/g\' /etc/cobbler/modules.conf'
sm_vm.execute_cmd(cmd, timeout=10)
cmd = 'sudo sed -i \'/^cobbler_username/c\cobbler_username \t= cobbler\' /opt/contrail/server_manager/sm-config.ini'
sm_vm.execute_cmd(cmd, timeout=10)
cmd = 'sudo sed -i \'/^cobbler_password/c\cobbler_password \t= cobbler\' /opt/contrail/server_manager/sm-config.ini'
sm_vm.execute_cmd(cmd, timeout=10)
cmd = 'sudo cp /etc/contrail_smgr/cobbler/named.conf.options.u.sample /etc/bind/named.conf.options'
sm_vm.execute_cmd(cmd, timeout=10)
cmd = 'sudo cp /etc/contrail_smgr/cobbler/named.template.u.sample /etc/cobbler/named.template'
sm_vm.execute_cmd(cmd, timeout=10)
#comment ubuntu ntp servers in /etc/ntp.conf and add ntp_server
sm_ntp_server = CONF['DEFAULTS']['ntp_servers']
cmd = 'sudo sed -i \'/ubunt/s/^/#/\' /etc/ntp.conf'
sm_vm.execute_cmd(cmd, timeout=10)
cmd = 'echo server {} | sudo tee --append /etc/ntp.conf > /dev/null'.format(
sm_ntp_server)
sm_vm.execute_cmd(cmd, timeout=10)
#restart ntp service
cmd = 'sudo service ntp stop; sudo ntpdate {};sudo service ntp start'.format(
sm_ntp_server)
res = sm_vm.execute_cmd(cmd, timeout=30)
waiting(10)
cmd = 'sudo service contrail-server-manager status'
res = sm_vm.execute_cmd(cmd, timeout=10)
print "Servermanager software deployment ended."
print "\n"
print "Starting servermanager service on the VM"
cmd = 'sudo service contrail-server-manager start'
res = sm_vm.execute_cmd(cmd, timeout=10)
waiting(20)
def create_sm_vm(self):
vm = 'servermanager'
vm_name = CONF[vm]['hostname']
sm_m_ip = get_ip(CONF[vm]['management_address'])
global sm_creation
sm_creation = False
#Check if the SM VM is created and service is UP
command = 'virsh list | grep %s | awk \'{print $3}\'' % (vm_name)
res = execute(command, ignore_errors=True)
vm_running = False
if res == 'running':
print "SM VM is created and UP and RUNNING"
vm_running = True
#check if VM is reachable if so, check service is running
#ping SM
vm_pingable = False
res = os.system('ping -c 5 ' + sm_m_ip)
if res == 0:
print "Servermanger VM is UP and reachable"
vm_pingable = True
sm_local_user = CONF[vm]['local_user']
sm_local_passwd = CONF[vm]['local_password']
sm_vm = RemoteConnection()
sm_vm.connect(sm_m_ip,
username=sm_local_user,
password=sm_local_passwd)
cmd = 'sudo service contrail-server-manager status'
res = sm_vm.execute_cmd(cmd, timeout=10)
if re.search(r'not|unrecognized', res, re.M | re.I):
sm_vm_service = False
else:
print "contrail-server-manager service is running."
sm_vm_service = True
else:
print "Servermanager VM is not reachable or does not exists"
if vm_running and vm_pingable and sm_vm_service:
print "The existing servermanager VM is UP and RUNNING, hence not deleting and creating a new one"
#adding route to ipmi subnet
print "Adding route to IPMI network to reach from servermanager"
ipmi_network = CONF['DEFAULTS']['ipmi_network']
sm_gw = CONF[vm]['gateway']
cmd = 'route add -net {0} gw {1}'.format(ipmi_network, sm_gw)
res = sm_vm.execute_cmd(cmd, timeout=10)
else:
sm_creation = True
print "Creating servermanager VM with name " + vm_name
print "Deleting existing servermanager VM"
command = 'virsh destroy ' + vm_name
execute(command, ignore_errors=True)
waiting(3)
command = 'virsh undefine ' + vm_name
execute(command, ignore_errors=True)
waiting(3)
vncpasswd = CONF[vm]['local_password']
OS = '--os-variant=ubuntutrusty --os-type=linux'
ARCH = '--arch=x86_64'
NET1 = '--network bridge=mgmtbr'
NET2 = '--network bridge=extbr'
Disk = '--boot hd'
Gr = '--graphics vnc,password={} --noautoconsole'.format(vncpasswd)
Cpu = '--vcpus=%s' % CONF[vm]['vcpus']
Ram = '--ram=%s' % CONF[vm]['memory']
Src = '--disk path=/var/lib/libvirt/images/%s.qcow2,format=qcow2,bus=virtio --import --autostart --disk path=/var/lib/libvirt/images/%s_init.iso,bus=virtio,device=cdrom' % (
vm_name, vm_name)
#print OS, ARCH, NET1, NET2, Disk, Gr, Cpu, Ram, Src
command = 'cp -f /var/lib/libvirt/images/%s /var/lib/libvirt/images/%s.qcow2' % (
cloudimg, vm_name)
execute(command, ignore_errors=False)
command = 'qemu-img resize /var/lib/libvirt/images/%s.qcow2 +%sG' % (
vm_name, CONF[vm]['harddisk'])
execute(command, ignore_errors=False)
# create user-data and meta-data file
print "Creating user-data and meta-data files to boot ubuntu cloud image. Begins.."
vm_disk_path = '/root/sm_vm/{0}'.format(vm_name)
command = 'rm -rf {0}'.format(vm_disk_path)
execute(command, ignore_errors=False)
waiting(3)
command = 'mkdir -p {0}'.format(vm_disk_path)
execute(command, ignore_errors=False)
sm_root_password = CONF['DEFAULTS']['root_password']
sm_dns_search = CONF[vm]['dns_search']
sm_dns_server = CONF[vm]['dns_servers']
sm_local_user = CONF[vm]['local_user']
sm_local_passwd = CONF[vm]['local_password']
sm_m_ip = get_ip(CONF[vm]['management_address'])
sm_m_netmask = get_netmask(CONF[vm]['management_address'])
sm_ext_ip = get_ip(CONF['DEFAULTS']['sm_ext_address'])
sm_ext_netmask = get_netmask(CONF['DEFAULTS']['sm_ext_address'])
sm_ext_gateway = CONF['DEFAULTS']['sm_ext_gateway']
command = 'cat /home/{0}/.ssh/id_rsa.pub'.format(sm_local_user)
sm_rsa_user = execute(command, ignore_errors=False)
command = 'cat /root/.ssh/id_rsa.pub'.format(sm_local_user)
sm_rsa_root = execute(command, ignore_errors=False)
print "Creating user-data file required for cloud init image.."
userdata_tmpl = jinja_env.get_template('user-data')
userdata = userdata_tmpl.render(password=sm_root_password,
local_user=sm_local_user,
local_password=sm_local_passwd,
dns_server=sm_dns_server,
ssh_rsa_user=sm_rsa_user,
ssh_rsa_root=sm_rsa_root,
dns_search=sm_dns_search)
fobj = open(vm_disk_path + '/user-data', 'w')
if hasattr(userdata, '__iter__'):
for data in userdata:
fobj.write(data)
fobj.close()
else:
fobj.write(userdata)
fobj.close()
print "Creating meta-data file required for cloud init image.."
metadata_tmpl = jinja_env.get_template('meta-data')
metadata = metadata_tmpl.render(hostname=vm_name,
mgmt_ip=sm_m_ip,
mgmt_netmask=sm_m_netmask,
ext_ip=sm_ext_ip,
ext_netmask=sm_ext_netmask,
ext_gateway=sm_ext_gateway)
fobj = open(vm_disk_path + '/meta-data', 'w')
if hasattr(metadata, '__iter__'):
for data in metadata:
fobj.write(data)
fobj.close()
else:
fobj.write(metadata)
fobj.close()
print "Creating user-data and meta-data files to boot ubuntu cloud image. Ends.."
# Launch VM
print "Launching servermanager VM..."
cwddir = os.getcwd()
command = 'cd {}'.format(vm_disk_path + '/')
os.chdir(vm_disk_path)
command = 'cloud-localds -v /var/lib/libvirt/images/{}_init.iso user-data meta-data'.format(
vm_name)
os.system(command)
os.chdir(cwddir)
command = 'virt-install {} {} {} {} {} {} {} {} {} --name={}'.format(
OS, ARCH, NET1, NET2, Disk, Src, Gr, Cpu, Ram, vm_name)
execute(command, ignore_errors=False)
print "Launched servermanager VM, Waiting for the VM to come up...."
self.verify_sm_vm(vm)
class reImageAndDeploy(object):
def __init__(self):
global jinja_env
jinja_env = Environment(
loader=FileSystemLoader(from_project_root('contrail')))
self.sm_vm = None
self.sm_api = None
self.sm_home = None
def check_sm_status(self):
print "Checking servermanager reachablility."
sm_m_ip = get_ip(CONF['servermanager']['management_address'])
sm_local_user = CONF['servermanager']['local_user']
sm_local_passwd = CONF['servermanager']['local_password']
count = 5
for _ in range(count):
#ping SM
res = os.system('ping -c 3 ' + sm_m_ip)
if res == 0:
print "servermanager is reachable"
self.sm_vm = RemoteConnection()
self.sm_vm.connect(sm_m_ip,
username=sm_local_user,
password=sm_local_passwd)
self.sm_api = serverManager(sm_m_ip)
cmd = 'eval echo ~$USER'
self.sm_home = self.sm_vm.execute_cmd(cmd, timeout=10)
return
else:
print "Waiting for servermanager to respond."
print "Retry after 30sec"
waiting(30)
print "\n"
print "Response: %s" % (res)
print "=================================================================================="
print "servermanager is not reachable, please check or run ./smgr_provision.sh if not done"
print "=================================================================================="
sys.exit(1)
def copy_iso_contrail_images(self):
print "Copying ubuntu OS ISO and contrail installer package onto servermanager VM"
svr1 = CONF['DEFAULTS']['bms'][0]
server1_mgmt_ip = get_ip(CONF[svr1]['management_address'])
sm_m_ip = get_ip(CONF['servermanager']['management_address'])
ubuntuiso = CONF['DEFAULTS']['ubuntuimage']
contrailimg = CONF['DEFAULTS']['contrailimage']
jumphost_img_url = 'http://{}/pockit_images/'.format(server1_mgmt_ip)
self.sm_vm.chdir('{}/images/'.format(self.sm_home))
cmd = 'wget -q {0}{1} -O {1}'.format(jumphost_img_url, ubuntuiso)
if self.sm_vm.status_command(cmd) != 0:
print "\n"
print "Command: %s" % (cmd)
print "=========================================================="
print "Error in downloading ubuntu ISO image onto server-manager"
print "=========================================================="
sys.exit(1)
else:
print "Downloaded ubuntu ISO image successfully onto server-manager"
cmd = 'wget -q {0}{1} -O {1}'.format(jumphost_img_url, contrailimg)
if self.sm_vm.status_command(cmd) != 0:
print "\n"
print "Command: %s" % (cmd)
print "==================================================================="
print "Error in downloading contrail installer package onto server-manager"
print "==================================================================="
sys.exit(1)
else:
print "Downloaded contrail installer package successfully onto server-manager"
def create_cluster_json_file(self):
print "Creating contrail cluster json file"
cluster_tmpl = jinja_env.get_template('cluster.json')
userdata = cluster_tmpl.render(
dns_search=CONF['DEFAULTS']['dns_search'],
cluster_gateway=CONF['DEFAULTS']['cluster_gateway'],
contrail_asn=CONF['DEFAULTS']['contrail_asn'],
openstack_password=CONF['DEFAULTS']['contrail_os_webui_passwd'],
mysql_password=CONF['DEFAULTS']['contrail_mysql_passwd'],
cassandra_db_size=CONF['DEFAULTS']['cassandra_db_size'],
cluster_netmask=CONF['DEFAULTS']['cluster_netmask'])
with open(sk_img_path + '/cluster.json', 'w+') as fobj:
fobj.write(userdata)
def create_image_json_file(self):
print "Creating cluster host os image and contrail installer package file"
img_tmpl = jinja_env.get_template('image.json')
userdata = img_tmpl.render(
kickseed_file=self.sm_home + '/json-files/contrail-ubuntu-sk.seed',
kickstart_file=self.sm_home + '/json-files/contrail-ubuntu-sk.ks',
ubuntu_iso_file=self.sm_home + '/images/' +
CONF['DEFAULTS']['ubuntuimage'],
contrail_pkg_file=self.sm_home + '/images/' +
CONF['DEFAULTS']['contrailimage'])
with open(sk_img_path + '/image.json', 'w+') as fobj:
fobj.write(userdata)
def create_vm_server_json_files(self):
vm_list = CONF['DEFAULTS']['reimagevms']
print "Creating VM %s json files" % (vm_list)
filenum = 1
for vm in vm_list:
vm_tmpl = jinja_env.get_template('server.json')
userdata = vm_tmpl.render(
ctrldata_intf=CONF[vm]['ctrldata_interface'],
dns_search=CONF[vm]['dns_search'],
server_name=CONF[vm]['hostname'],
ipmi_address="",
ipmi_password="",
ipmi_username="",
mgmt_address=CONF[vm]['management_address'],
mgmt_mac=CONF[vm]['management_mac'],
mgmt_intf=CONF[vm]['management_interface'],
mgmt_gateway=CONF[vm]['gateway'],
ctrldata_address=CONF[vm]['ctrldata_address'],
ctrldata_mac=CONF[vm]['ctrldata_mac'],
disk_partition=CONF[vm]['partition'],
root_password=CONF['DEFAULTS']['root_password'],
roles=CONF[vm]['roles'])
with open(sk_img_path + '/server{}.json'.format(filenum),
'w+') as fobj:
fobj.write(userdata)
filenum += 1
def create_bms_server_json_files(self):
vm_list = CONF['DEFAULTS']['reimagevms']
bms_list = CONF['DEFAULTS']['bms']
print "Creating BMS %s json files" % (bms_list)
filenum = len(vm_list) + 1
for vm in bms_list[1:]:
vm_tmpl = jinja_env.get_template('server.json')
userdata = vm_tmpl.render(
ctrldata_intf=CONF[vm]['ctrldata_interface'],
dns_search=CONF[vm]['dns_search'],
server_name=CONF[vm]['hostname'],
ipmi_address=CONF[vm]['ipmi_address'],
ipmi_password=CONF[vm]['ipmi_password'],
ipmi_username=CONF[vm]['ipmi_username'],
mgmt_address=CONF[vm]['management_address'],
mgmt_mac=CONF[vm]['management_mac'],
mgmt_intf=CONF[vm]['management_interface'],
mgmt_gateway=CONF[vm]['gateway'],
ctrldata_address=CONF[vm]['ctrldata_address'],
ctrldata_mac=CONF[vm]['ctrldata_mac'],
disk_partition=CONF[vm]['partition'],
root_password=CONF['DEFAULTS']['root_password'],
roles=CONF[vm]['roles'])
with open(sk_img_path + '/server{}.json'.format(filenum),
'w+') as fobj:
fobj.write(userdata)
filenum += 1
def copy_seed_ks_to_websvr(self):
print "Copying ubuntu seed and kickstart files on to jumphost webserver"
cmd = 'cp {0}/contrail-ubuntu-sk.seed {1}/'.format(
from_project_root('contrail'), sk_img_path)
execute(cmd, ignore_errors=True)
cmd = 'cp {0}/contrail-ubuntu-sk.ks {1}/'.format(
from_project_root('contrail'), sk_img_path)
execute(cmd, ignore_errors=True)
def copy_json_files_to_sm(self):
print "Copying cluster, image, VM and BMS json files on to server-manager VM"
svr1 = CONF['DEFAULTS']['bms'][0]
server1_mgmt_ip = get_ip(CONF[svr1]['management_address'])
jumphost_img_url = 'http://{}/pockit_images/'.format(server1_mgmt_ip)
cmd = 'cd {0}/json-files/ && wget -O cluster.json {1}/cluster.json'.format(
self.sm_home, jumphost_img_url)
if self.sm_vm.status_command(cmd) != 0:
print "\n"
print "Command: %s" % (cmd)
print "============================================================"
print "Error in downloading cluster.json file onto servermanager VM"
print "============================================================"
sys.exit(1)
else:
print "Downloaded cluster.json file successfully onto servermanager VM"
cmd = 'cd {0}/json-files/ && wget -O image.json {1}/image.json'.format(
self.sm_home, jumphost_img_url)
if self.sm_vm.status_command(cmd) != 0:
print "\n"
print "Command: %s" % (cmd)
print "============================================================"
print "Error in downloading image.json file onto servermanager VM"
print "============================================================"
sys.exit(1)
else:
print "Downloaded image.json file successfully onto servermanager VM"
cluster_list = CONF['DEFAULTS']['reimagevms'] + CONF['DEFAULTS'][
'bms'][1:]
for i in range(1, len(cluster_list) + 1):
cmd = 'cd {0}/json-files/ && wget -O server{1}.json {2}/server{1}.json'.format(
self.sm_home, i, jumphost_img_url)
if self.sm_vm.status_command(cmd) != 0:
print "\n"
print "Command: %s" % (cmd)
print "============================================================="
print "Error in downloading server{}.json file onto servermanager VM".format(
i)
print "============================================================"
sys.exit(1)
else:
print "Downloaded server{}.json file successfully onto servermanager VM".format(
i)
self.copy_seed_ks_to_websvr()
cmd = 'cd {0}/json-files/ && wget -O contrail-ubuntu-sk.ks {1}/contrail-ubuntu-sk.ks'.format(
self.sm_home, jumphost_img_url)
if self.sm_vm.status_command(cmd) != 0:
print "\n"
print "Command: %s" % (cmd)
print "============================================================"
print "Error in downloading kickstart file onto servermanager VM"
print "============================================================"
sys.exit(1)
else:
print "Downloaded kickstart file successfully onto servermanager VM"
cmd = 'cd {0}/json-files/ && wget -O contrail-ubuntu-sk.seed {1}/contrail-ubuntu-sk.seed'.format(
self.sm_home, jumphost_img_url)
if self.sm_vm.status_command(cmd) != 0:
print "\n"
print "Command: %s" % (cmd)
print "============================================================"
print "Error in downloading kickseed file onto servermanager VM"
print "============================================================"
sys.exit(1)
else:
print "Downloaded kickseed file successfully onto servermanager VM"
def reimage_vms(self):
vm_list = CONF['DEFAULTS']['reimagevms']
print "Re-Imaging %s VM's. Begins.." % (vm_list)
#add cluster, image and server1,2
print "Adding cluster.json to servermanager"
cmd = 'server-manager delete cluster --cluster_id {}'.format(
sk_cluster_id)
self.sm_vm.execute_cmd(cmd, timeout=30)
cmd = 'server-manager add cluster -f {}/json-files/cluster.json'.format(
self.sm_home)
self.sm_vm.execute_cmd(cmd, timeout=30)
res = self.sm_api.cluster_get(sk_cluster_id)
if res != 200:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "======================================"
print "Error in adding cluster.json to servermanager"
print "======================================"
sys.exit(1)
else:
print "Successfully added cluster.json to servermanager"
print "Adding image.json to servermanager"
cmd = 'server-manager delete image --image_id {}'.format(ubuntu_img_id)
self.sm_vm.execute_cmd(cmd, timeout=120)
cmd = 'server-manager add image -f {}/json-files/image.json'.format(
self.sm_home)
self.sm_vm.execute_cmd(cmd, timeout=120)
res = self.sm_api.image_get(ubuntu_img_id)
if res != 200:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "======================================"
print "Error in adding image.json to servermanager"
print "======================================"
sys.exit(1)
else:
print "Successfully added image.json to servermanager"
vm_list = CONF['DEFAULTS']['reimagevms']
i = 1
for vm in vm_list:
print "Adding server{}.json to servermanager".format(i)
cmd = 'server-manager delete server --server_id {}'.format(
CONF[vm]['hostname'])
self.sm_vm.execute_cmd(cmd, timeout=30)
cmd = 'server-manager add server -f {0}/json-files/server{1}.json'.format(
self.sm_home, i)
self.sm_vm.execute_cmd(cmd, timeout=30)
res = self.sm_api.server_get(CONF[vm]['hostname'])
if res != 200:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "============================================="
print "Error in adding server{}.json to servermanager".format(
i)
print "============================================="
sys.exit(1)
else:
print "Successfully added server{}.json to servermanager".format(
i)
print "Issuing reimage server{}".format(i)
cmd = 'yes y | server-manager reimage --server_id {}'.format(
CONF[vm]['hostname'])
self.sm_vm.execute_cmd(cmd, timeout=10)
i += 1
for vm in vm_list:
print "Re-imaging %s VM" % CONF[vm]['hostname']
cmd = "virsh destroy %s" % CONF[vm]['hostname']
execute(cmd, ignore_errors=True)
waiting(5)
cmd = "virsh undefine %s" % CONF[vm]['hostname']
execute(cmd, ignore_errors=True)
waiting(5)
cmd = "virt-install --os-variant=ubuntutrusty --os-type=linux --arch=x86_64 --network mac=%s,bridge=mgmtbr --network mac=%s,bridge=ctrldatabr --file=/var/lib/libvirt/images/%s.qcow2 --graphics vnc,password=%s --noautoconsole --vcpus=%s --ram=%s --pxe --name %s --autostart --file-size=%s" % (
CONF[vm]['management_mac'], CONF[vm]['ctrldata_mac'],
CONF[vm]['hostname'], CONF[vm]['local_password'],
CONF[vm]['vcpus'], CONF[vm]['memory'], CONF[vm]['hostname'],
CONF[vm]['harddisk'])
execute(cmd, ignore_errors=False)
waiting(10)
print "Waiting for %s VM to reimage" % vm
self.check_reimage_status(vm_list)
print "Re-Imaging %s VM's. Ends.." % vm_list
def check_reimage_status(self, svr_list):
queue = Queue.Queue()
threads = []
for vm in svr_list:
svr = CONF[vm]['hostname']
t = threading.Thread(target=self.check_reimage_vm_status,
args=(svr, ),
name=svr)
threads.append(t)
t.start()
for t in threads:
t.join()
print "Completed VM re-image status verification"
def check_reimage_vm_status(self, vmname):
count = 40
expected_res = "reimage_completed"
for _ in range(count):
#Get reimage status
res = self.sm_api.server_get(server_id=vmname, status=True)
print "Waiting for %s VM to get re-imaged.." % vmname
print "Expected status: %s :: Current status: %s " % (
expected_res, res)
if res == expected_res:
waiting(10)
cmd = "virsh list --all"
res = execute(cmd, ignore_errors=True)
if re.search(r'shut off', res, re.M | re.I):
print "Re-imaged VM is shut-off, starting now..."
cmd = "virsh start " + vmname
res = execute(cmd, ignore_errors=True)
waiting(5)
else:
print "None of the re-imaged VM's are shut-off"
return
else:
print "Retry after 30sec"
waiting(30)
print "\n"
print "==========================================================="
print "VM %s did not complete Re-imaging, hence exiting..." % vmname
print "==========================================================="
sys.exit(1)
def check_ipmi_connectivity(self):
print "Checking IPMI reachability before re-imaging BMS"
waiting(10)
bms_list = CONF['DEFAULTS']['bms']
for bms in bms_list:
ipmi_ip = CONF[bms]['ipmi_address']
cmd = 'ping -c 4 %s' % (ipmi_ip)
res = self.sm_vm.execute_cmd(cmd, timeout=10)
if re.search(r'4 received', res, re.M | re.I):
print "IPMI IP (%s) of BMS (%s) is reachable from servermanager." % (
ipmi_ip, bms)
else:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "=============================================================================="
print "IPMI IP (%s) of BMS (%s) is not reachable from servermanager.check IPMI config" % (
ipmi_ip, bms)
print "=============================================================================="
sys.exit(1)
def reimage_bms(self):
print "Re-Imaging BMS's. Begins.."
#check ipmi
self.check_ipmi_connectivity()
bms_list = CONF['DEFAULTS']['bms']
vms_list = CONF['DEFAULTS']['reimagevms']
i = len(vms_list) + 1
for bms in bms_list[1:]:
print "Adding server{}.json to servermanager".format(i)
cmd = 'server-manager delete server --server_id {}'.format(
CONF[bms]['hostname'])
self.sm_vm.execute_cmd(cmd, timeout=30)
cmd = 'server-manager add server -f {0}/json-files/server{1}.json'.format(
self.sm_home, i)
self.sm_vm.execute_cmd(cmd, timeout=30)
res = self.sm_api.server_get(CONF[bms]['hostname'])
if res != 200:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "============================================="
print "Error in adding server{}.json to servermanager".format(
i)
print "============================================="
sys.exit(1)
else:
print "Successfully added server{}.json to servermanager".format(
i)
print "Issuing reimage server{}".format(i)
cmd = 'yes y | server-manager reimage --server_id {}'.format(
CONF[bms]['hostname'])
self.sm_vm.execute_cmd(cmd, timeout=10)
i += 1
print "Waiting for BMS's to get reimaged"
self.check_bms_reimage_status(bms_list[1:])
print "Re-Imaging %s BMS's. Ends.." % bms_list[1:]
def check_bms_reimage_status(self, svr_list):
queue = Queue.Queue()
threads = []
for bms in svr_list:
svr = CONF[bms]['hostname']
t = threading.Thread(target=self.check_reimage_bms_status,
args=(svr, ),
name=svr)
threads.append(t)
t.start()
for t in threads:
t.join()
print "Completed reimage status verification for BMS"
def check_reimage_bms_status(self, bmsname):
count = 80
expected_res = "reimage_completed"
for _ in range(count):
#Get reimage status
res = self.sm_api.server_get(server_id=bmsname, status=True)
print "Waiting for BMS %s to re-image.." % bmsname
print "Expected status: %s :: Current status: %s " % (
expected_res, res)
if res == expected_res:
print "BMS %s is re-imaged" % bmsname
return
else:
print "Retry after 30sec"
waiting(30)
if count == 30 and res == "restart_issued":
print "Looks like some issue with BMS (%s) powering on, please check" % (
bmsname)
continue
print "\n"
print "==============================================================="
print "BMS server %s did not complete Re-imaging, hence exiting..." % bmsname
print "==============================================================="
sys.exit(1)
def wait_bms_reboot(self):
bms_list = CONF['DEFAULTS']['bms']
for bms in bms_list[1:]:
count = 40
for _ in range(count):
bms_ip = get_ip(CONF[bms]['management_address'])
cmd = 'ping -c 4 %s' % (bms_ip)
res = self.sm_vm.execute_cmd(cmd, timeout=30)
if re.search(r'4 received', res, re.M | re.I):
print "MGMT IP(%s)of (%s) is reachable from servermanager." % (
bms_ip, bms)
continue
else:
print "Waiting for the BMS %s to reboot and come up" % bms
print "Retry after 30sec"
waiting(30)
def check_topology_connectivity(self):
print "Checking the management IP and ctrldata IP reachability from servermanager VM"
waiting(5)
all_list = CONF['DEFAULTS']['bms'] + CONF['DEFAULTS']['reimagevms']
for inventory in all_list:
mgmt_ip = get_ip(CONF[inventory]['management_address'])
cmd = 'ping -c 4 %s' % (mgmt_ip)
res = self.sm_vm.execute_cmd(cmd, timeout=30)
if re.search(r'4 received', res, re.M | re.I):
print "MGMT IP(%s)of (%s) is reachable from servermanager." % (
mgmt_ip, inventory)
else:
print "\n"
print "Command: %s" % (cmd)
print "Response: %s" % (res)
print "=============================================================================="
print "MGMT IP(%s)of (%s) is not reachable from servermanager.check configuration" % (
mgmt_ip, inventory)
print "=============================================================================="
sys.exit(1)
#TO DO check connectivity from cc-host VM, os-host, BMS
def remove_server2_from_cluster(self):
bms2 = CONF['DEFAULTS']['bms'][1]
print "\nRemoving %s from contrail cluster, as it will be used to deploy vMX on it" % (
bms2)
svr2_ip = get_ip(CONF[bms2]['management_address'])
svr2_netmask = get_netmask(CONF[bms2]['management_address'])
svr2_usr = CONF['DEFAULTS']['root_username']
svr2_passwd = CONF['DEFAULTS']['root_password']
svr2_connect = RemoteConnection()
svr2_connect.connect(svr2_ip, username=svr2_usr, password=svr2_passwd)
#reboot
bms2_name = CONF[bms2]['hostname']
print "Rebooting %s" % (bms2_name)
cmd = 'reboot'
svr2_connect.execute_cmd(cmd, timeout=10)
waiting(10)
print "Deleting %s from the cluster" % (bms2_name)
cmd = 'server-manager delete server --server_id %s' % (bms2_name)
res = self.sm_vm.execute_cmd(cmd, timeout=10)
#wait till the bms to comes up
count = 40
for _ in range(count):
#ping server2
res = os.system('ping -c 3 ' + svr2_ip)
if res == 0:
print "%s is reachable and is UP" % (bms2_name)
waiting(60)
return
else:
print "Waiting for %s to come UP.." % (bms2_name)
print "Retry after 30sec"
waiting(30)
print "\n"
print "========================================================================="
print "{0} is not reachable, did not boot up with correct mgmt_address {1}".format(
bms2_anme, svr2_ip)
print "========================================================================="
sys.exit(1)
def provision_contrail(self):
print "Started provisioning contrail on to the cluster"
waiting(5)
provision_list = CONF['DEFAULTS']['reimagevms'] + CONF['DEFAULTS'][
'bms'][2:]
self.check_topology_connectivity()
self.remove_server2_from_cluster()
cmd = 'yes y | server-manager provision --cluster_id {0} {1}'.format(
sk_cluster_id, contrail_pkg_id)
res = self.sm_vm.execute_cmd(cmd, timeout=30)
self.check_provision_status(provision_list)
sys.exit(0)
def check_provision_status(self, provision_list):
queue = Queue.Queue()
threads = []
for svr in provision_list:
svr = CONF[svr]['hostname']
t = threading.Thread(target=self.provision_status,
args=(svr, ),
name=svr)
threads.append(t)
t.start()
for t in threads:
t.join()
print "Completed contrail provision status verification"
def provision_status(self, bmsname):
count = 300
expected_res = "provision_completed"
for _ in range(count):
#Get provision status
res = self.sm_api.server_get(server_id=bmsname, status=True)
print "Waiting for BMS %s to get provisioned on %s." % (
bmsname, sk_cluster_id)
print "Expected status: %s :: Current status: %s " % (
expected_res, res)
if res == expected_res:
print "BMS %s is provisioned" % bmsname
return
else:
print "Retry after 30sec"
waiting(30)
print "\n"
print "================================================="
print "BMS %s did not get provisioned, hence exiting..." % bmsname
print "================================================="
sys.exit(1)