def check_contrail_status(self):
     print "Checking contrail-status on the contrail controller node"
     cc_name = CONF['DEFAULTS']['reimagevms'][0]
     os_name = CONF['DEFAULTS']['reimagevms'][1]
     cc_ip = get_ip(CONF[cc_name]['management_address'])
     cc_ctrld_ip = get_ip(CONF[cc_name]['ctrldata_address'])
     os_ctrld_ip = get_ip(CONF[os_name]['ctrldata_address'])
     user = CONF['DEFAULTS']['root_username']
     passwd = CONF['DEFAULTS']['root_password']
     reconnect = RemoteConnection()
     reconnect.connect(cc_ip, username=user, password=passwd)
     cmd = 'contrail-status'
     res = reconnect.execute_cmd(cmd, timeout=20)
     if re.search(r'NTP state|inactive', res, re.M | re.I):
         print "\n"
         print "Command: %s" % (cmd)
         print "Response: %s" % (res)
         print "========================================================================================="
         print "contrail services on controller show in NTP state unsynchronized/inactive. please check."
         print "========================================================================================="
         sys.exit(1)
     else:
         print "All the services on contrail controller shows active. Installation successfully done."
         print res
         print "Access to Contrail web-ui from jumphost({}): https://{}:8143/".format(
             svr1, cc_ctrld_ip)
         print "Access to Openstack Horizon web-ui from jumphost({}): http://{}/horizon".format(
             svr1, os_ctrld_ip)
    def verify_sm_on_vm(self):
        sm_creation = True
        if sm_creation:
            print "\nVerifying contrail-server-manager service is running on the VM"

            vm = 'servermanager'
            sm_m_ip = get_ip(CONF[vm]['management_address'])
            sm_local_user = CONF[vm]['local_user']
            sm_local_passwd = CONF[vm]['local_password']
            svr1 = CONF['DEFAULTS']['bms'][0]

            sm_vm = RemoteConnection()
            sm_vm.connect(sm_m_ip,
                          username=sm_local_user,
                          password=sm_local_passwd)
            cmd = 'sudo service contrail-server-manager status'
            res = sm_vm.execute_cmd(cmd, timeout=10)
            #print res
            if re.search(r'not', res, re.M | re.I):
                print "\n"
                print "Command:%s" % (cmd)
                print "Response:%s" % (res)
                print "======================================================================"
                print "contrail-server-manager service is not running...please check the logs"
                print "======================================================================"
                sys.exit(1)
            else:
                print "contrail-server-manager service is running."
                sm_ext_ip = get_ip(CONF['DEFAULTS']['sm_ext_address'])
                print "Access to ServerManager web-ui from jumphost({}): https://{}:9143/".format(
                    svr1, sm_m_ip)
                print "Do ssh tunnel command, ssh -f -N -L 9143:{}:9143 root@{}".format(
                    sm_m_ip, sm_ext_ip)
                print "Access to ServerManager web-ui from laptop to monitor provisioning \
                        and health of servers: https://127.0.0.1:9143/"

                #adding route to ipmi subnet
                print "Adding route to IPMI network to reach from servermanager"
                ipmi_network = CONF['DEFAULTS']['ipmi_network']
                sm_gw = CONF[vm]['gateway']
                cmd = 'sudo route add -net {0} gw {1}'.format(
                    ipmi_network, sm_gw)
                res = sm_vm.execute_cmd(cmd, timeout=10)
    def remove_server2_from_cluster(self):

        bms2 = CONF['DEFAULTS']['bms'][1]
        print "\nRemoving %s from contrail cluster, as it will be used to deploy vMX on it" % (
            bms2)

        svr2_ip = get_ip(CONF[bms2]['management_address'])
        svr2_netmask = get_netmask(CONF[bms2]['management_address'])
        svr2_usr = CONF['DEFAULTS']['root_username']
        svr2_passwd = CONF['DEFAULTS']['root_password']
        svr2_connect = RemoteConnection()
        svr2_connect.connect(svr2_ip, username=svr2_usr, password=svr2_passwd)

        #reboot
        bms2_name = CONF[bms2]['hostname']
        print "Rebooting %s" % (bms2_name)
        cmd = 'reboot'
        svr2_connect.execute_cmd(cmd, timeout=10)
        waiting(10)

        print "Deleting %s from the cluster" % (bms2_name)
        cmd = 'server-manager delete server --server_id %s' % (bms2_name)
        res = self.sm_vm.execute_cmd(cmd, timeout=10)

        #wait till the bms to comes up
        count = 40
        for _ in range(count):
            #ping server2
            res = os.system('ping -c 3 ' + svr2_ip)
            if res == 0:
                print "%s is reachable and is UP" % (bms2_name)
                waiting(60)
                return
            else:
                print "Waiting for %s to come UP.." % (bms2_name)
                print "Retry after 30sec"
                waiting(30)
        print "\n"
        print "========================================================================="
        print "{0} is not reachable, did not boot up with correct mgmt_address {1}".format(
            bms2_anme, svr2_ip)
        print "========================================================================="
        sys.exit(1)
class vmxDeploy(object):
    def __init__(self):
        global jinja_env
        jinja_env = Environment(
            loader=FileSystemLoader(from_project_root('vmx')))

        svr1name = CONF['DEFAULTS']['bms'][0]
        self.svr2_bms_name = CONF['DEFAULTS']['bms'][1]

        self.server1_ip = get_ip(CONF[svr1name]['management_address'])
        server2_ip = get_ip(CONF[self.svr2_bms_name]['management_address'])
        server2_user = CONF['DEFAULTS']['root_username']
        server2_passwd = CONF['DEFAULTS']['root_password']

        self.svr2_bms = RemoteConnection()
        self.svr2_bms.connect(server2_ip,
                              username=server2_user,
                              password=server2_passwd)
        cmd = 'eval echo ~$USER'
        self.svr2_bms_home = self.svr2_bms.execute_cmd(cmd, timeout=10)

    def copy_vmx_images(self):
        print "Copying vMX image onto %s" % self.svr2_bms_name

        vmximage = CONF['DEFAULTS']['vmximage']
        cmd = 'rm -rf images/'
        self.svr2_bms.execute_cmd(cmd, timeout=10)
        cmd = 'mkdir -p images'
        self.svr2_bms.execute_cmd(cmd, timeout=10)

        cmd = '{0}/images/'.format(self.svr2_bms_home)
        self.svr2_bms.chdir(cmd)
        cmd = 'wget -q http://{0}/pockit_images/{1} -O {2}/images/{1}'.format(
            self.server1_ip, vmximage, self.svr2_bms_home)
        if self.svr2_bms.status_command(cmd) != 0:
            print "\n"
            print "Command: %s" % (cmd)
            print "Response: %s" % (res)
            print "=================================================="
            print "Error in downloading vMX image on to %s" % (
                self.svr2_bms_name)
            print "=================================================="
            sys.exit(1)
        else:
            print "Downloaded vMX image on to %s successfully" % (
                self.svr2_bms_name)

    def configure_server_uplink_intf(self):

        print "Configuring external/internet facing interface and routes"

        vm = 'vmx'
        intf = CONF[vm]['management_interface']
        server_ext_ip = get_ip(CONF[vm]['management_address'])
        server_ext_mask = get_netmask(CONF[vm]['management_address'])
        gateway = CONF['DEFAULTS']['vmx_ext_gateway']
        dns_server = CONF[vm]['dns_servers']
        dns_search = CONF[vm]['dns_search']

        cmd = '/sbin/ifconfig {0} {1} netmask {2} up'.format(
            intf, server_ext_ip, server_ext_mask)
        self.svr2_bms.execute_cmd(cmd, timeout=10)
        cmd = 'route delete default'
        self.svr2_bms.execute_cmd(cmd, timeout=10)
        cmd = 'route add default gw {}'.format(gateway)
        self.svr2_bms.execute_cmd(cmd, timeout=10)

        #update /etc/network/interfaces file with this ip
        cmd = 'cat /etc/network/interfaces'
        res = self.svr2_bms.execute_cmd(cmd, timeout=10)
        if re.search(r'%s' % intf, res, re.M | re.I):
            print "external interface configuration exists."
        else:
            print "external interface configuration does not exists, hence configuring"
            cmd = '(echo ;echo auto {}; echo iface {} inet static; echo address {}; echo netmask {}; echo gateway {}; echo dns-nameservers {}; echo dns-search {}) >> /etc/network/interfaces'.format(
                intf, intf, server_ext_ip, server_ext_mask, gateway,
                dns_server, dns_search)
            self.svr2_bms.execute_cmd(cmd, timeout=30)
            #add nameserver in /etc/resolv.conf
            cmd = '(echo nameserver {}) >> /etc/resolv.conf'.format(dns_server)
            res = self.svr2_bms.execute_cmd(cmd, timeout=10)

    def update_vmx_packages(self):
        #check if internet is reachable(8.8.8.8, www.google.com"
        waiting(5)
        cmd = 'ping -c 5 %s' % (internet_www)
        if self.svr2_bms.status_command(cmd) != 0:
            print "\n"
            print "Command: %s" % (cmd)
            print "========================================================================="
            print "Internet URL({0}) is not reachable from {1}, please check route and DNS".format(
                internet_www, self.svr2_bms_name)
            print "========================================================================="
            sys.exit(1)
        else:
            print "Internet URL({0}) is reachable from {1}".format(
                internet_www, self.svr2_bms_name)

        #apt-get update, upgrade kernel
        print "Upgrading packages to install vMX"
        cmd = 'cp /etc/apt/sources.list.save /etc/apt/sources.list'
        self.svr2_bms.execute_cmd(cmd, timeout=10)
        cmd = 'sed -i \'/dl.google/s/^/#/\' /etc/apt/sources.list'
        self.svr2_bms.execute_cmd(cmd, timeout=10)
        cmd = 'sed -i \'/medibuntu/s/^/#/\' /etc/apt/sources.list'
        self.svr2_bms.execute_cmd(cmd, timeout=10)
        cmd = 'apt-get update'
        self.svr2_bms.execute_cmd(cmd, timeout=240)
        cmd = 'apt-get install -y linux-firmware linux-image-3.13.0.32-generic linux-image-extra-3.13.0.32-generic linux-headers-3.13.0.32-generic'
        self.svr2_bms.execute_cmd(cmd, timeout=300)
        cmd = 'update-grub'
        self.svr2_bms.execute_cmd(cmd, timeout=120)
        cmd = 'apt-get install -y expect bridge-utils qemu-kvm libvirt-bin python-netifaces vnc4server libyaml-dev python-yaml numactl libparted0-dev libpciaccess-dev libnuma-dev libyajl-dev libxml2-dev libglib2.0-dev libnl-dev python-pip python-dev libxml2-dev libxslt-dev'
        self.svr2_bms.execute_cmd(cmd, timeout=600)

        #install upgrade libvirt 1.2.19
        print "Upgrading libvirt version to 1.2.19 on %s" % (
            self.svr2_bms_name)
        self.svr2_bms.chdir('/tmp')
        cmd = 'rm -rf libvirt-1.2.19/'
        self.svr2_bms.execute_cmd(cmd, timeout=10)
        cmd = 'rm -rf libvirt-1.2.19.*'
        self.svr2_bms.execute_cmd(cmd, timeout=10)
        cmd = 'wget -q http://{0}/pockit_images/{1} -O {1}'.format(
            self.server1_ip, sk_libvirt_img)
        if self.svr2_bms.status_command(cmd) != 0:
            print "\n"
            print "Command: %s" % (cmd)
            print "=================================================="
            print "Error in downloading libvirt image on to %s" % (
                self.svr2_bms_name)
            print "=================================================="
            sys.exit(1)
        else:
            print "Downloaded libvirt image on to %s successfully" % (
                self.svr2_bms_name)
        cmd = 'sudo service libvirt-bin stop'
        res = self.svr2_bms.execute_cmd(cmd, timeout=40)
        cmd = 'tar -xzf libvirt-1.2.19.tar.gz'
        self.svr2_bms.execute_cmd(cmd, timeout=40)
        self.svr2_bms.chdir('/tmp/libvirt-1.2.19/')
        cmd = './configure --prefix=/usr --localstatedir=/ --with-numactl'
        self.svr2_bms.execute_cmd(cmd, timeout=300)
        self.svr2_bms.chdir('/tmp/libvirt-1.2.19/')
        cmd = 'make'
        self.svr2_bms.execute_cmd(cmd, timeout=300)
        self.svr2_bms.chdir('/tmp/libvirt-1.2.19/')
        cmd = 'sudo make install && sudo ldconfig'
        self.svr2_bms.execute_cmd(cmd, timeout=300)

        #check if service is running
        cmd = 'sudo service libvirt-bin start'
        self.svr2_bms.execute_cmd(cmd, timeout=10)
        cmd = 'sudo service libvirt-bin status'
        res = self.svr2_bms.execute_cmd(cmd, timeout=10)
        if re.search(r'running', res, re.M | re.I):
            print "libvirt-bin service is running after upgrading to 1.2.19 version"
        else:
            print "\n"
            print "Command: %s" % (cmd)
            print "Response: %s" % (res)
            print "===================================================="
            print "libvirt-bin service is not running! please check it"
            print "===================================================="
            sys.exit(1)

        cmd = 'sudo libvirtd --version'
        res = self.svr2_bms.execute_cmd(cmd, timeout=10)
        if re.search(r'19', res, re.M | re.I):
            print "Upgraded libvirt-bin version 1.2.19 is running"
        else:
            print "\n"
            print "Command: %s" % (cmd)
            print "Response: %s" % (res)
            print "==================================================================="
            print "Upgraded libvirt-bin version 1.2.19 is not running! please check it"
            print "==================================================================="
            sys.exit(1)

    def create_vmx_conf(self, re_img, fpc_img, hdd_img):
        print 'Creating vMX confguration file'
        vmx_conf_template = jinja_env.get_template('vmx.conf')
        user_data = vmx_conf_template.render(
            vmx_hostname=CONF['vmx']['hostname'],
            vmx_ext_intf=CONF['vmx']['management_interface'],
            vmx_re_img=re_img,
            vmx_re_hdd=hdd_img,
            vmx_pfe_img=fpc_img,
            vmx_ctl_ip_addr=get_ip(CONF['DEFAULTS']['vmx_vcp_address']),
            vmx_pfe_ip_addr=get_ip(CONF['DEFAULTS']['vmx_vfp_address']))
        fobj = open(sk_img_path + '/vmx.conf', 'w')
        if hasattr(user_data, '__iter__'):
            for data in user_data:
                fobj.write(data)
            fobj.close()
        else:
            fobj.write(user_data)
            fobj.close()

    def create_vmx_junosdev_conf(self):
        print 'Creating vmx-junosdev file'
        dev_conf_template = jinja_env.get_template('vmx-junosdev.conf')
        user_data = dev_conf_template.render(
            vmx_hostname=CONF['vmx']['hostname'],
            ctrldata_intf=CONF['vmx']['ctrldata_interface'])
        fobj = open(sk_img_path + '/vmx-junosdev.conf', 'w')
        if hasattr(user_data, '__iter__'):
            for data in user_data:
                fobj.write(data)
            fobj.close()
        else:
            fobj.write(user_data)
            fobj.close()

    def create_vmx_default_config(self):
        print 'Creating vMX init configuration expect script file'
        init_conf_template = jinja_env.get_template('vmx-init-cfg.exp')
        user_data = init_conf_template.render(
            vmx_hostname=CONF['vmx']['hostname'],
            vmx_root_passwd=CONF['DEFAULTS']['root_password'],
            vmx_dns_server=CONF['vmx']['dns_servers'],
            vmx_ntp_server=CONF['DEFAULTS']['ntp_servers'],
            vmx_ctrldata_ip=CONF['vmx']['ctrldata_address'],
            vmx_re_ipaddr=CONF['DEFAULTS']['vmx_vcp_address'],
            vmx_default_gateway=CONF['DEFAULTS']['vmx_ext_gateway'],
            vmx_loopback_ip=CONF['DEFAULTS']['vmx_loopback_ip'])
        fobj = open(sk_img_path + '/vmx-init-cfg.exp', 'w')
        if hasattr(user_data, '__iter__'):
            for data in user_data:
                fobj.write(data)
            fobj.close()
        else:
            fobj.write(user_data)
            fobj.close()

    def pre_up_vmx(self):
        print 'Preparing the infra-structure for installing vMX'
        global vmx_folder

        vmximage = CONF['DEFAULTS']['vmximage']
        vmx_base_folder = '{0}/images/'.format(self.svr2_bms_home)
        self.svr2_bms.chdir(vmx_base_folder)
        cmd = 'tar -xzf %s' % (vmximage)
        self.svr2_bms.execute_cmd(cmd, timeout=60)

        fname, fext = os.path.splitext(vmximage)
        vmx_folder_name = 'vmx-' + fname.split('-')[2]
        vmx_img_folder = vmx_base_folder + vmx_folder_name + '/images/'
        vmx_folder = vmx_base_folder + vmx_folder_name + '/'
        self.svr2_bms.chdir(vmx_img_folder)

        cmd = 'ls | grep -E \'qcow2|vFPC\''
        op = self.svr2_bms.execute_cmd(cmd, timeout=60)
        vmx_img_list = op.splitlines()
        re_img = vmx_img_folder + vmx_img_list[0]
        fpc_img = vmx_img_folder + vmx_img_list[1]
        hdd_img = vmx_img_folder + 'vmxhdd.img'

        self.create_vmx_conf(re_img, fpc_img, hdd_img)
        self.create_vmx_junosdev_conf()
        self.create_vmx_default_config()

        self.svr2_bms.chdir(vmx_folder + 'config/')
        jumphost_img_url = 'http://{}/pockit_images/'.format(self.server1_ip)
        cmd = 'wget -O vmx.conf {0}vmx.conf'.format(jumphost_img_url)
        if self.svr2_bms.status_command(cmd) != 0:
            print "\n"
            print "Command: %s" % (cmd)
            print "===================================================="
            print "Error in downloading vmx.conf file onto server %s" % (
                self.svr2_bms_name)
            print "===================================================="
            sys.exit(1)
        else:
            print "Successfully downloaded vmx.conf file onto server %s" % (
                self.svr2_bms_name)

        cmd = 'wget -O vmx-junosdev.conf {0}vmx-junosdev.conf'.format(
            jumphost_img_url)
        if self.svr2_bms.status_command(cmd) != 0:
            print "\n"
            print "Command: %s" % (cmd)
            print "============================================================"
            print "Error in downloading vmx-junosdev.conf file onto server %s" % (
                self.svr2_bms_name)
            print "============================================================"
            sys.exit(1)
        else:
            print "Successfully downloaded vmx-junosdev.conf file onto server %s" % (
                self.svr2_bms_name)

        self.svr2_bms.chdir(vmx_folder)
        cmd = 'wget -O vmx-init-cfg.exp {0}vmx-init-cfg.exp'.format(
            jumphost_img_url)
        if self.svr2_bms.status_command(cmd) != 0:
            print "\n"
            print "Command: %s" % (cmd)
            print "============================================================"
            print "Error in downloading vmx-init-cfg.exp file onto server %s" % (
                self.svr2_bms_name)
            print "============================================================"
            sys.exit(1)
        else:
            print "Successfully downloaded vmx-init-cfg.exp file onto server %s" % (
                self.svr2_bms_name)

    def deploy_vmx(self):
        print "Deploying vMX. Begins.."
        global vmx_folder
        self.svr2_bms.chdir(vmx_folder)

        #delete virbr0 bridge
        cmd = '/sbin/ifconfig virbr0 down'
        self.svr2_bms.execute_cmd(cmd, timeout=10)

        cmd = 'brctl delbr virbr0'
        self.svr2_bms.execute_cmd(cmd, timeout=10)

        cmd = 'sudo ./vmx.sh -lv --install'
        res = self.svr2_bms.execute_cmd(cmd, timeout=360)
        if 'error(s)' in res:
            print "\n"
            print "Command: %s" % (cmd)
            print "Response: %s" % (res)
            print "======================================"
            print "Error in deploying vMX onto server %s" % (
                self.svr2_bms_name)
            print "======================================"
            sys.exit(1)
        else:
            print "Successfully deployed vMX onto server %s" % (
                self.svr2_bms_name)

        cmd = 'sudo ./vmx.sh --bind-dev'
        op = self.svr2_bms.execute_cmd(cmd, timeout=40)

        cmd = 'sudo ./vmx.sh --bind-check'
        op = self.svr2_bms.execute_cmd(cmd, timeout=40)
        print "Deploying vMX. Ends.."

    def configure_and_verify_vmx(self):
        print "Configuring and verifying vMX management IP reachability"
        print "Waiting for vMX to boot up.."

        waiting(200)
        global vmx_folder
        self.svr2_bms.chdir(vmx_folder)

        cmd = 'expect vmx-init-cfg.exp'
        op = self.svr2_bms.execute_cmd(cmd, timeout=120)
        waiting(10)

        vmx_re_ipaddr = get_ip(CONF['DEFAULTS']['vmx_vcp_address'])
        cmd = 'ping -c 5 {}'.format(vmx_re_ipaddr)
        res = self.svr2_bms.execute_cmd(cmd, timeout=60)
        if re.search(r'5 received', res, re.M | re.I):
            print "vMX mgmt IP %s is reachable, its UP and RUNNING" % (
                vmx_re_ipaddr)
        else:
            print "\n"
            print "Command: %s" % (cmd)
            print "Response: %s" % (res)
            print "============================================================"
            print "vMX mgmt IP %s is not reachable, please check configuration!" % (
                vmx_re_ipaddr)
            print "============================================================"
            sys.exit(1)

        svr2 = self.svr2_bms_name
        print "Configuring ctrldata interface with IP on server(%s)" % (
            self.svr2_bms_name)
        cmd = '/sbin/ifconfig {0} 0.0.0.0 up'.format(
            CONF[svr2]['ctrldata_interface'])
        res = self.svr2_bms.execute_cmd(cmd, timeout=20)

        ctl_ip = get_ip(CONF[svr2]['ctrldata_address'])
        ctl_netmask = get_netmask(CONF[svr2]['ctrldata_address'])
        cmd = '/sbin/ifconfig vmx_link1 {0} netmask {1} up'.format(
            ctl_ip, ctl_netmask)
        res = self.svr2_bms.execute_cmd(cmd, timeout=20)
        waiting(5)

        vmx_loopback_ip = CONF['DEFAULTS']['vmx_loopback_ip']
        vmx_ctl_ip = get_ip(CONF['vmx']['ctrldata_address'])
        cmd = 'route add -host {0}/32 gw {1}'.format(vmx_loopback_ip,
                                                     vmx_ctl_ip)
        res = self.svr2_bms.execute_cmd(cmd, timeout=40)
        waiting(10)

        #on jumphost add route to vmx loopback ip
        command = 'route add -host {0}/32 gw {1}'.format(
            vmx_loopback_ip, ctl_ip)
        res = execute(command, ignore_errors=False)
        waiting(5)

        #ping vMX loopback ip
        print "Checking vmx_loopback IP(%s) is reachable" % (vmx_loopback_ip)
        cmd = 'ping -c 5 {}'.format(vmx_loopback_ip)
        res = self.svr2_bms.execute_cmd(cmd, timeout=60)
        if re.search(r'5 received', res, re.M | re.I):
            print "vMX loopback IP %s is reachable from server(%s)" % (
                vmx_loopback_ip, svr2)
        else:
            print "\n"
            print "Command: %s" % (cmd)
            print "Response: %s" % (res)
            print "=========================================================================="
            print "vMX loopback IP %s is not reachable from server(%s). Check configuration." % (
                vmx_loopback_ip, svr2)
            print "=========================================================================="
            sys.exit(1)
    def deploy_sm_on_vm(self):

        if sm_creation:
            print "Servermanager software deployment is started on VM..."

            vm = 'servermanager'
            sm_m_ip = get_ip(CONF[vm]['management_address'])
            sm_local_user = CONF[vm]['local_user']
            sm_local_passwd = CONF[vm]['local_password']

            #create dhcp.template file
            self.create_dhcp_template(vm)

            sm_img = CONF['DEFAULTS']['smimage']
            svr1 = CONF['DEFAULTS']['bms'][0]
            server1_mgmt_ip = get_ip(CONF[svr1]['management_address'])

            sm_vm = RemoteConnection()
            sm_vm.connect(sm_m_ip,
                          username=sm_local_user,
                          password=sm_local_passwd)
            cmd = 'eval echo ~$USER'
            sm_home_dir = sm_vm.execute_cmd(cmd, timeout=10)

            #check internet reachability from SM
            cmd = 'ping -c 4 %s' % internet_ip
            res = sm_vm.execute_cmd(cmd, timeout=10)
            if re.search(r'4 received', res, re.M | re.I):
                print "Internet IP (%s) is reachable from servermanager VM." % internet_ip
            else:
                print "\n"
                print "Command:%s" % (cmd)
                print "Response:%s" % (res)
                print "========================================================================="
                print "Internet IP (%s) is not reachable from servermanager...please check configuration" % internet_ip
                print "========================================================================="
                sys.exit(1)

            cmd = 'ping -c 4 %s' % internet_www
            res = sm_vm.execute_cmd(cmd, timeout=10)
            if re.search(r'4 received', res, re.M | re.I):
                print "Internet URL (%s) is reachable from servermanager VM." % internet_www
            else:
                print "\n"
                print "Command:%s" % (cmd)
                print "Response:%s" % (res)
                print "========================================================================="
                print "Internet URL (%s) is not reachable from servermanager...please check configuration" % internet_www
                print "========================================================================="
                sys.exit(1)

            #create dir images
            cmd = 'rm -rf images/; mkdir -p images'
            sm_vm.execute_cmd(cmd, timeout=10)
            cmd = 'rm -rf json-files/; mkdir -p json-files'
            sm_vm.execute_cmd(cmd, timeout=10)

            sm_vm.chdir(sm_home_dir + '/images/')
            jumphost_img_url = 'http://{}/pockit_images/'.format(
                server1_mgmt_ip)
            cmd = 'wget -q {0}{1} -O {1}'.format(jumphost_img_url, sm_img)
            if sm_vm.status_command(cmd) != 0:
                print "\n"
                print "Command:%s" % (cmd)
                print "=============================================================="
                print "Error in downloading servermanager build onto servermanager VM"
                print "=============================================================="
                sys.exit(1)
            else:
                print "Successfully downloaded servermanager build onto servermanager VM"

            cmd = 'sudo dpkg -i {0}/images/{1}'.format(sm_home_dir, sm_img)
            if sm_vm.status_command(cmd) != 0:
                print "\n"
                print "Command:%s" % (cmd)
                print "====================================================="
                print "Error in depackaging the servermanager debian package"
                print "====================================================="
                sys.exit(1)
            else:
                print "Successfully Depackaged the servermanager debian package"

            print "Starting setup.sh --all command, this will take a while"
            global kill
            global stop
            sm_vm.chdir('/opt/contrail/contrail_server_manager/')
            queue = Queue.Queue()
            t = threading.Thread(target=self.progress_bar_print, args=())
            t.start()
            try:
                #cmd = 'sudo ./setup.sh --sm={0}'.format(sm_img)
                cmd = 'sudo ./setup.sh --all'
                if sm_vm.status_command(cmd) != 0:
                    kill = True
                    stop = True
                    print "\n"
                    print "Command:%s" % (cmd)
                    print "======================================="
                    print "Error in running setup.sh --all command"
                    print "======================================="
                    sys.exit(1)
                else:
                    stop = True
                    print "\nSuccessfully completed running setup.sh --all command"
            except KeyboardInterrupt or EOFError:
                print "entering keyboard error"
                kill = True
                stop = True
            sm_vm.chdir(sm_home_dir + '/json-files/')
            cmd = 'sudo wget -q {0}dhcp.template -O /etc/cobbler/dhcp.template'.format(
                jumphost_img_url)
            if sm_vm.status_command(cmd, timeout=90) != 0:
                print "\n"
                print "Command:%s" % (cmd)
                print "=========================================================="
                print "Error in copying dhcp.template file on to servermanager VM"
                print "=========================================================="
                sys.exit(1)
            else:
                print "Successfully copied dhcp.template file on to servermanager VM"

            cmd = 'sudo sed -i \'s/= authn_testing/= authn_configfile/g\' /etc/cobbler/modules.conf'
            sm_vm.execute_cmd(cmd, timeout=10)
            cmd = 'sudo sed -i \'/^cobbler_username/c\cobbler_username  \t= cobbler\' /opt/contrail/server_manager/sm-config.ini'
            sm_vm.execute_cmd(cmd, timeout=10)
            cmd = 'sudo sed -i \'/^cobbler_password/c\cobbler_password  \t= cobbler\' /opt/contrail/server_manager/sm-config.ini'
            sm_vm.execute_cmd(cmd, timeout=10)
            cmd = 'sudo cp /etc/contrail_smgr/cobbler/named.conf.options.u.sample /etc/bind/named.conf.options'
            sm_vm.execute_cmd(cmd, timeout=10)
            cmd = 'sudo cp /etc/contrail_smgr/cobbler/named.template.u.sample /etc/cobbler/named.template'
            sm_vm.execute_cmd(cmd, timeout=10)

            #comment ubuntu ntp servers in /etc/ntp.conf and add ntp_server
            sm_ntp_server = CONF['DEFAULTS']['ntp_servers']
            cmd = 'sudo sed -i \'/ubunt/s/^/#/\' /etc/ntp.conf'
            sm_vm.execute_cmd(cmd, timeout=10)

            cmd = 'echo server {} | sudo tee --append /etc/ntp.conf > /dev/null'.format(
                sm_ntp_server)
            sm_vm.execute_cmd(cmd, timeout=10)

            #restart ntp service
            cmd = 'sudo service ntp stop; sudo ntpdate {};sudo service ntp start'.format(
                sm_ntp_server)
            res = sm_vm.execute_cmd(cmd, timeout=30)
            waiting(10)

            cmd = 'sudo service contrail-server-manager status'
            res = sm_vm.execute_cmd(cmd, timeout=10)
            print "Servermanager software deployment ended."
            print "\n"
            print "Starting servermanager service on the VM"
            cmd = 'sudo service contrail-server-manager start'
            res = sm_vm.execute_cmd(cmd, timeout=10)
            waiting(20)
    def create_sm_vm(self):
        vm = 'servermanager'
        vm_name = CONF[vm]['hostname']
        sm_m_ip = get_ip(CONF[vm]['management_address'])
        global sm_creation
        sm_creation = False

        #Check if the SM VM is created and service is UP
        command = 'virsh list | grep %s | awk \'{print $3}\'' % (vm_name)
        res = execute(command, ignore_errors=True)
        vm_running = False
        if res == 'running':
            print "SM VM is created and UP and RUNNING"
            vm_running = True

        #check if VM is reachable if so, check service is running
        #ping SM
        vm_pingable = False
        res = os.system('ping -c 5 ' + sm_m_ip)
        if res == 0:
            print "Servermanger VM is UP and reachable"
            vm_pingable = True

            sm_local_user = CONF[vm]['local_user']
            sm_local_passwd = CONF[vm]['local_password']

            sm_vm = RemoteConnection()
            sm_vm.connect(sm_m_ip,
                          username=sm_local_user,
                          password=sm_local_passwd)
            cmd = 'sudo service contrail-server-manager status'
            res = sm_vm.execute_cmd(cmd, timeout=10)
            if re.search(r'not|unrecognized', res, re.M | re.I):
                sm_vm_service = False
            else:
                print "contrail-server-manager service is running."
                sm_vm_service = True
        else:
            print "Servermanager VM is not reachable or does not exists"

        if vm_running and vm_pingable and sm_vm_service:
            print "The existing servermanager VM is UP and RUNNING, hence not deleting and creating a new one"
            #adding route to ipmi subnet
            print "Adding route to IPMI network to reach from servermanager"
            ipmi_network = CONF['DEFAULTS']['ipmi_network']
            sm_gw = CONF[vm]['gateway']
            cmd = 'route add -net {0} gw {1}'.format(ipmi_network, sm_gw)
            res = sm_vm.execute_cmd(cmd, timeout=10)
        else:
            sm_creation = True
            print "Creating servermanager VM with name  " + vm_name

            print "Deleting existing servermanager VM"
            command = 'virsh destroy ' + vm_name
            execute(command, ignore_errors=True)
            waiting(3)
            command = 'virsh undefine ' + vm_name
            execute(command, ignore_errors=True)
            waiting(3)

            vncpasswd = CONF[vm]['local_password']
            OS = '--os-variant=ubuntutrusty --os-type=linux'
            ARCH = '--arch=x86_64'
            NET1 = '--network bridge=mgmtbr'
            NET2 = '--network bridge=extbr'
            Disk = '--boot hd'
            Gr = '--graphics vnc,password={} --noautoconsole'.format(vncpasswd)
            Cpu = '--vcpus=%s' % CONF[vm]['vcpus']
            Ram = '--ram=%s' % CONF[vm]['memory']
            Src = '--disk path=/var/lib/libvirt/images/%s.qcow2,format=qcow2,bus=virtio --import --autostart --disk path=/var/lib/libvirt/images/%s_init.iso,bus=virtio,device=cdrom' % (
                vm_name, vm_name)

            #print OS, ARCH, NET1, NET2, Disk, Gr, Cpu, Ram, Src
            command = 'cp -f /var/lib/libvirt/images/%s /var/lib/libvirt/images/%s.qcow2' % (
                cloudimg, vm_name)
            execute(command, ignore_errors=False)
            command = 'qemu-img resize /var/lib/libvirt/images/%s.qcow2 +%sG' % (
                vm_name, CONF[vm]['harddisk'])
            execute(command, ignore_errors=False)

            # create user-data and meta-data file
            print "Creating user-data and meta-data files to boot ubuntu cloud image. Begins.."
            vm_disk_path = '/root/sm_vm/{0}'.format(vm_name)
            command = 'rm -rf {0}'.format(vm_disk_path)
            execute(command, ignore_errors=False)
            waiting(3)
            command = 'mkdir -p {0}'.format(vm_disk_path)
            execute(command, ignore_errors=False)

            sm_root_password = CONF['DEFAULTS']['root_password']
            sm_dns_search = CONF[vm]['dns_search']
            sm_dns_server = CONF[vm]['dns_servers']
            sm_local_user = CONF[vm]['local_user']
            sm_local_passwd = CONF[vm]['local_password']
            sm_m_ip = get_ip(CONF[vm]['management_address'])
            sm_m_netmask = get_netmask(CONF[vm]['management_address'])
            sm_ext_ip = get_ip(CONF['DEFAULTS']['sm_ext_address'])
            sm_ext_netmask = get_netmask(CONF['DEFAULTS']['sm_ext_address'])
            sm_ext_gateway = CONF['DEFAULTS']['sm_ext_gateway']
            command = 'cat /home/{0}/.ssh/id_rsa.pub'.format(sm_local_user)
            sm_rsa_user = execute(command, ignore_errors=False)
            command = 'cat /root/.ssh/id_rsa.pub'.format(sm_local_user)
            sm_rsa_root = execute(command, ignore_errors=False)

            print "Creating user-data file required for cloud init image.."
            userdata_tmpl = jinja_env.get_template('user-data')
            userdata = userdata_tmpl.render(password=sm_root_password,
                                            local_user=sm_local_user,
                                            local_password=sm_local_passwd,
                                            dns_server=sm_dns_server,
                                            ssh_rsa_user=sm_rsa_user,
                                            ssh_rsa_root=sm_rsa_root,
                                            dns_search=sm_dns_search)
            fobj = open(vm_disk_path + '/user-data', 'w')
            if hasattr(userdata, '__iter__'):
                for data in userdata:
                    fobj.write(data)
                fobj.close()
            else:
                fobj.write(userdata)
                fobj.close()

            print "Creating meta-data file required for cloud init image.."
            metadata_tmpl = jinja_env.get_template('meta-data')
            metadata = metadata_tmpl.render(hostname=vm_name,
                                            mgmt_ip=sm_m_ip,
                                            mgmt_netmask=sm_m_netmask,
                                            ext_ip=sm_ext_ip,
                                            ext_netmask=sm_ext_netmask,
                                            ext_gateway=sm_ext_gateway)
            fobj = open(vm_disk_path + '/meta-data', 'w')
            if hasattr(metadata, '__iter__'):
                for data in metadata:
                    fobj.write(data)
                fobj.close()
            else:
                fobj.write(metadata)
                fobj.close()
            print "Creating user-data and meta-data files to boot ubuntu cloud image. Ends.."

            # Launch VM
            print "Launching servermanager VM..."
            cwddir = os.getcwd()
            command = 'cd {}'.format(vm_disk_path + '/')
            os.chdir(vm_disk_path)
            command = 'cloud-localds  -v  /var/lib/libvirt/images/{}_init.iso user-data meta-data'.format(
                vm_name)
            os.system(command)
            os.chdir(cwddir)
            command = 'virt-install {} {} {} {} {} {} {} {} {} --name={}'.format(
                OS, ARCH, NET1, NET2, Disk, Src, Gr, Cpu, Ram, vm_name)
            execute(command, ignore_errors=False)
            print "Launched servermanager VM, Waiting for the VM to come up...."
            self.verify_sm_vm(vm)
class reImageAndDeploy(object):
    def __init__(self):
        global jinja_env
        jinja_env = Environment(
            loader=FileSystemLoader(from_project_root('contrail')))
        self.sm_vm = None
        self.sm_api = None
        self.sm_home = None

    def check_sm_status(self):
        print "Checking servermanager reachablility."
        sm_m_ip = get_ip(CONF['servermanager']['management_address'])
        sm_local_user = CONF['servermanager']['local_user']
        sm_local_passwd = CONF['servermanager']['local_password']

        count = 5
        for _ in range(count):
            #ping SM
            res = os.system('ping -c 3 ' + sm_m_ip)
            if res == 0:
                print "servermanager is reachable"
                self.sm_vm = RemoteConnection()
                self.sm_vm.connect(sm_m_ip,
                                   username=sm_local_user,
                                   password=sm_local_passwd)
                self.sm_api = serverManager(sm_m_ip)
                cmd = 'eval echo ~$USER'
                self.sm_home = self.sm_vm.execute_cmd(cmd, timeout=10)
                return
            else:
                print "Waiting for servermanager to respond."
                print "Retry after 30sec"
                waiting(30)
        print "\n"
        print "Response: %s" % (res)
        print "=================================================================================="
        print "servermanager is not reachable, please check or run ./smgr_provision.sh if not done"
        print "=================================================================================="
        sys.exit(1)

    def copy_iso_contrail_images(self):
        print "Copying ubuntu OS ISO and contrail installer package onto servermanager VM"

        svr1 = CONF['DEFAULTS']['bms'][0]
        server1_mgmt_ip = get_ip(CONF[svr1]['management_address'])
        sm_m_ip = get_ip(CONF['servermanager']['management_address'])
        ubuntuiso = CONF['DEFAULTS']['ubuntuimage']
        contrailimg = CONF['DEFAULTS']['contrailimage']

        jumphost_img_url = 'http://{}/pockit_images/'.format(server1_mgmt_ip)

        self.sm_vm.chdir('{}/images/'.format(self.sm_home))
        cmd = 'wget -q {0}{1} -O {1}'.format(jumphost_img_url, ubuntuiso)
        if self.sm_vm.status_command(cmd) != 0:
            print "\n"
            print "Command: %s" % (cmd)
            print "=========================================================="
            print "Error in downloading ubuntu ISO image onto server-manager"
            print "=========================================================="
            sys.exit(1)
        else:
            print "Downloaded ubuntu ISO image successfully onto server-manager"

        cmd = 'wget -q {0}{1} -O {1}'.format(jumphost_img_url, contrailimg)
        if self.sm_vm.status_command(cmd) != 0:
            print "\n"
            print "Command: %s" % (cmd)
            print "==================================================================="
            print "Error in downloading contrail installer package onto server-manager"
            print "==================================================================="
            sys.exit(1)
        else:
            print "Downloaded contrail installer package successfully onto server-manager"

    def create_cluster_json_file(self):
        print "Creating contrail cluster json file"
        cluster_tmpl = jinja_env.get_template('cluster.json')
        userdata = cluster_tmpl.render(
            dns_search=CONF['DEFAULTS']['dns_search'],
            cluster_gateway=CONF['DEFAULTS']['cluster_gateway'],
            contrail_asn=CONF['DEFAULTS']['contrail_asn'],
            openstack_password=CONF['DEFAULTS']['contrail_os_webui_passwd'],
            mysql_password=CONF['DEFAULTS']['contrail_mysql_passwd'],
            cassandra_db_size=CONF['DEFAULTS']['cassandra_db_size'],
            cluster_netmask=CONF['DEFAULTS']['cluster_netmask'])
        with open(sk_img_path + '/cluster.json', 'w+') as fobj:
            fobj.write(userdata)

    def create_image_json_file(self):
        print "Creating cluster host os image and contrail installer package file"
        img_tmpl = jinja_env.get_template('image.json')
        userdata = img_tmpl.render(
            kickseed_file=self.sm_home + '/json-files/contrail-ubuntu-sk.seed',
            kickstart_file=self.sm_home + '/json-files/contrail-ubuntu-sk.ks',
            ubuntu_iso_file=self.sm_home + '/images/' +
            CONF['DEFAULTS']['ubuntuimage'],
            contrail_pkg_file=self.sm_home + '/images/' +
            CONF['DEFAULTS']['contrailimage'])
        with open(sk_img_path + '/image.json', 'w+') as fobj:
            fobj.write(userdata)

    def create_vm_server_json_files(self):
        vm_list = CONF['DEFAULTS']['reimagevms']
        print "Creating VM %s json files" % (vm_list)
        filenum = 1
        for vm in vm_list:
            vm_tmpl = jinja_env.get_template('server.json')
            userdata = vm_tmpl.render(
                ctrldata_intf=CONF[vm]['ctrldata_interface'],
                dns_search=CONF[vm]['dns_search'],
                server_name=CONF[vm]['hostname'],
                ipmi_address="",
                ipmi_password="",
                ipmi_username="",
                mgmt_address=CONF[vm]['management_address'],
                mgmt_mac=CONF[vm]['management_mac'],
                mgmt_intf=CONF[vm]['management_interface'],
                mgmt_gateway=CONF[vm]['gateway'],
                ctrldata_address=CONF[vm]['ctrldata_address'],
                ctrldata_mac=CONF[vm]['ctrldata_mac'],
                disk_partition=CONF[vm]['partition'],
                root_password=CONF['DEFAULTS']['root_password'],
                roles=CONF[vm]['roles'])
            with open(sk_img_path + '/server{}.json'.format(filenum),
                      'w+') as fobj:
                fobj.write(userdata)
            filenum += 1

    def create_bms_server_json_files(self):
        vm_list = CONF['DEFAULTS']['reimagevms']
        bms_list = CONF['DEFAULTS']['bms']
        print "Creating BMS %s json files" % (bms_list)
        filenum = len(vm_list) + 1
        for vm in bms_list[1:]:
            vm_tmpl = jinja_env.get_template('server.json')
            userdata = vm_tmpl.render(
                ctrldata_intf=CONF[vm]['ctrldata_interface'],
                dns_search=CONF[vm]['dns_search'],
                server_name=CONF[vm]['hostname'],
                ipmi_address=CONF[vm]['ipmi_address'],
                ipmi_password=CONF[vm]['ipmi_password'],
                ipmi_username=CONF[vm]['ipmi_username'],
                mgmt_address=CONF[vm]['management_address'],
                mgmt_mac=CONF[vm]['management_mac'],
                mgmt_intf=CONF[vm]['management_interface'],
                mgmt_gateway=CONF[vm]['gateway'],
                ctrldata_address=CONF[vm]['ctrldata_address'],
                ctrldata_mac=CONF[vm]['ctrldata_mac'],
                disk_partition=CONF[vm]['partition'],
                root_password=CONF['DEFAULTS']['root_password'],
                roles=CONF[vm]['roles'])
            with open(sk_img_path + '/server{}.json'.format(filenum),
                      'w+') as fobj:
                fobj.write(userdata)
            filenum += 1

    def copy_seed_ks_to_websvr(self):
        print "Copying ubuntu seed and kickstart files on to jumphost webserver"
        cmd = 'cp {0}/contrail-ubuntu-sk.seed {1}/'.format(
            from_project_root('contrail'), sk_img_path)
        execute(cmd, ignore_errors=True)
        cmd = 'cp {0}/contrail-ubuntu-sk.ks {1}/'.format(
            from_project_root('contrail'), sk_img_path)
        execute(cmd, ignore_errors=True)

    def copy_json_files_to_sm(self):
        print "Copying cluster, image, VM and BMS json files on to server-manager VM"
        svr1 = CONF['DEFAULTS']['bms'][0]
        server1_mgmt_ip = get_ip(CONF[svr1]['management_address'])
        jumphost_img_url = 'http://{}/pockit_images/'.format(server1_mgmt_ip)

        cmd = 'cd {0}/json-files/ && wget -O cluster.json {1}/cluster.json'.format(
            self.sm_home, jumphost_img_url)
        if self.sm_vm.status_command(cmd) != 0:
            print "\n"
            print "Command: %s" % (cmd)
            print "============================================================"
            print "Error in downloading cluster.json file onto servermanager VM"
            print "============================================================"
            sys.exit(1)
        else:
            print "Downloaded cluster.json file successfully onto servermanager VM"

        cmd = 'cd {0}/json-files/ && wget -O image.json {1}/image.json'.format(
            self.sm_home, jumphost_img_url)
        if self.sm_vm.status_command(cmd) != 0:
            print "\n"
            print "Command: %s" % (cmd)
            print "============================================================"
            print "Error in downloading image.json file onto servermanager VM"
            print "============================================================"
            sys.exit(1)
        else:
            print "Downloaded image.json file successfully onto servermanager VM"

        cluster_list = CONF['DEFAULTS']['reimagevms'] + CONF['DEFAULTS'][
            'bms'][1:]
        for i in range(1, len(cluster_list) + 1):
            cmd = 'cd {0}/json-files/ && wget -O server{1}.json {2}/server{1}.json'.format(
                self.sm_home, i, jumphost_img_url)
            if self.sm_vm.status_command(cmd) != 0:
                print "\n"
                print "Command: %s" % (cmd)
                print "============================================================="
                print "Error in downloading server{}.json file onto servermanager VM".format(
                    i)
                print "============================================================"
                sys.exit(1)
            else:
                print "Downloaded server{}.json file successfully onto servermanager VM".format(
                    i)

        self.copy_seed_ks_to_websvr()
        cmd = 'cd {0}/json-files/ && wget -O contrail-ubuntu-sk.ks {1}/contrail-ubuntu-sk.ks'.format(
            self.sm_home, jumphost_img_url)
        if self.sm_vm.status_command(cmd) != 0:
            print "\n"
            print "Command: %s" % (cmd)
            print "============================================================"
            print "Error in downloading kickstart file onto servermanager VM"
            print "============================================================"
            sys.exit(1)
        else:
            print "Downloaded kickstart file successfully onto servermanager VM"

        cmd = 'cd {0}/json-files/ && wget -O contrail-ubuntu-sk.seed {1}/contrail-ubuntu-sk.seed'.format(
            self.sm_home, jumphost_img_url)
        if self.sm_vm.status_command(cmd) != 0:
            print "\n"
            print "Command: %s" % (cmd)
            print "============================================================"
            print "Error in downloading kickseed file onto servermanager VM"
            print "============================================================"
            sys.exit(1)
        else:
            print "Downloaded kickseed file successfully onto servermanager VM"

    def reimage_vms(self):
        vm_list = CONF['DEFAULTS']['reimagevms']
        print "Re-Imaging %s VM's. Begins.." % (vm_list)

        #add cluster, image and server1,2
        print "Adding cluster.json to servermanager"
        cmd = 'server-manager delete cluster --cluster_id {}'.format(
            sk_cluster_id)
        self.sm_vm.execute_cmd(cmd, timeout=30)
        cmd = 'server-manager add cluster -f {}/json-files/cluster.json'.format(
            self.sm_home)
        self.sm_vm.execute_cmd(cmd, timeout=30)
        res = self.sm_api.cluster_get(sk_cluster_id)
        if res != 200:
            print "\n"
            print "Command: %s" % (cmd)
            print "Response: %s" % (res)
            print "======================================"
            print "Error in adding cluster.json to servermanager"
            print "======================================"
            sys.exit(1)
        else:
            print "Successfully added cluster.json to servermanager"

        print "Adding image.json to servermanager"
        cmd = 'server-manager delete image --image_id {}'.format(ubuntu_img_id)
        self.sm_vm.execute_cmd(cmd, timeout=120)
        cmd = 'server-manager add image -f {}/json-files/image.json'.format(
            self.sm_home)
        self.sm_vm.execute_cmd(cmd, timeout=120)
        res = self.sm_api.image_get(ubuntu_img_id)
        if res != 200:
            print "\n"
            print "Command: %s" % (cmd)
            print "Response: %s" % (res)
            print "======================================"
            print "Error in adding image.json to servermanager"
            print "======================================"
            sys.exit(1)
        else:
            print "Successfully added image.json to servermanager"

        vm_list = CONF['DEFAULTS']['reimagevms']
        i = 1
        for vm in vm_list:
            print "Adding server{}.json to servermanager".format(i)
            cmd = 'server-manager delete server --server_id {}'.format(
                CONF[vm]['hostname'])
            self.sm_vm.execute_cmd(cmd, timeout=30)
            cmd = 'server-manager add server -f {0}/json-files/server{1}.json'.format(
                self.sm_home, i)
            self.sm_vm.execute_cmd(cmd, timeout=30)
            res = self.sm_api.server_get(CONF[vm]['hostname'])
            if res != 200:
                print "\n"
                print "Command: %s" % (cmd)
                print "Response: %s" % (res)
                print "============================================="
                print "Error in adding server{}.json to servermanager".format(
                    i)
                print "============================================="
                sys.exit(1)
            else:
                print "Successfully added server{}.json to servermanager".format(
                    i)
                print "Issuing reimage server{}".format(i)
                cmd = 'yes y | server-manager reimage --server_id {}'.format(
                    CONF[vm]['hostname'])
                self.sm_vm.execute_cmd(cmd, timeout=10)
                i += 1

        for vm in vm_list:
            print "Re-imaging  %s  VM" % CONF[vm]['hostname']
            cmd = "virsh destroy %s" % CONF[vm]['hostname']
            execute(cmd, ignore_errors=True)
            waiting(5)
            cmd = "virsh undefine %s" % CONF[vm]['hostname']
            execute(cmd, ignore_errors=True)
            waiting(5)
            cmd = "virt-install --os-variant=ubuntutrusty --os-type=linux --arch=x86_64 --network mac=%s,bridge=mgmtbr --network mac=%s,bridge=ctrldatabr  --file=/var/lib/libvirt/images/%s.qcow2 --graphics vnc,password=%s --noautoconsole --vcpus=%s --ram=%s --pxe --name %s --autostart --file-size=%s" % (
                CONF[vm]['management_mac'], CONF[vm]['ctrldata_mac'],
                CONF[vm]['hostname'], CONF[vm]['local_password'],
                CONF[vm]['vcpus'], CONF[vm]['memory'], CONF[vm]['hostname'],
                CONF[vm]['harddisk'])
            execute(cmd, ignore_errors=False)
            waiting(10)
            print "Waiting for %s VM to reimage" % vm
        self.check_reimage_status(vm_list)
        print "Re-Imaging %s VM's. Ends.." % vm_list

    def check_reimage_status(self, svr_list):
        queue = Queue.Queue()
        threads = []
        for vm in svr_list:
            svr = CONF[vm]['hostname']
            t = threading.Thread(target=self.check_reimage_vm_status,
                                 args=(svr, ),
                                 name=svr)
            threads.append(t)
            t.start()
        for t in threads:
            t.join()
        print "Completed VM re-image status verification"

    def check_reimage_vm_status(self, vmname):
        count = 40
        expected_res = "reimage_completed"
        for _ in range(count):
            #Get reimage status
            res = self.sm_api.server_get(server_id=vmname, status=True)
            print "Waiting for %s VM to get re-imaged.." % vmname
            print "Expected status:   %s      ::      Current status:  %s  " % (
                expected_res, res)
            if res == expected_res:
                waiting(10)
                cmd = "virsh list --all"
                res = execute(cmd, ignore_errors=True)
                if re.search(r'shut off', res, re.M | re.I):
                    print "Re-imaged VM is shut-off, starting now..."
                    cmd = "virsh start " + vmname
                    res = execute(cmd, ignore_errors=True)
                    waiting(5)
                else:
                    print "None of the re-imaged VM's are shut-off"
                return
            else:
                print "Retry after 30sec"
                waiting(30)
        print "\n"
        print "==========================================================="
        print "VM %s did not complete Re-imaging, hence exiting..." % vmname
        print "==========================================================="
        sys.exit(1)

    def check_ipmi_connectivity(self):
        print "Checking IPMI reachability before re-imaging BMS"
        waiting(10)
        bms_list = CONF['DEFAULTS']['bms']
        for bms in bms_list:
            ipmi_ip = CONF[bms]['ipmi_address']
            cmd = 'ping -c 4 %s' % (ipmi_ip)
            res = self.sm_vm.execute_cmd(cmd, timeout=10)
            if re.search(r'4 received', res, re.M | re.I):
                print "IPMI IP (%s) of BMS (%s) is reachable from servermanager." % (
                    ipmi_ip, bms)
            else:
                print "\n"
                print "Command: %s" % (cmd)
                print "Response: %s" % (res)
                print "=============================================================================="
                print "IPMI IP (%s) of BMS (%s) is not reachable from servermanager.check IPMI config" % (
                    ipmi_ip, bms)
                print "=============================================================================="
                sys.exit(1)

    def reimage_bms(self):
        print "Re-Imaging BMS's. Begins.."

        #check ipmi
        self.check_ipmi_connectivity()

        bms_list = CONF['DEFAULTS']['bms']
        vms_list = CONF['DEFAULTS']['reimagevms']
        i = len(vms_list) + 1
        for bms in bms_list[1:]:
            print "Adding server{}.json to servermanager".format(i)
            cmd = 'server-manager delete server --server_id {}'.format(
                CONF[bms]['hostname'])
            self.sm_vm.execute_cmd(cmd, timeout=30)
            cmd = 'server-manager add server -f {0}/json-files/server{1}.json'.format(
                self.sm_home, i)
            self.sm_vm.execute_cmd(cmd, timeout=30)
            res = self.sm_api.server_get(CONF[bms]['hostname'])
            if res != 200:
                print "\n"
                print "Command: %s" % (cmd)
                print "Response: %s" % (res)
                print "============================================="
                print "Error in adding server{}.json to servermanager".format(
                    i)
                print "============================================="
                sys.exit(1)
            else:
                print "Successfully added server{}.json to servermanager".format(
                    i)
                print "Issuing reimage server{}".format(i)
                cmd = 'yes y | server-manager reimage --server_id {}'.format(
                    CONF[bms]['hostname'])
                self.sm_vm.execute_cmd(cmd, timeout=10)
                i += 1
        print "Waiting for BMS's to get reimaged"
        self.check_bms_reimage_status(bms_list[1:])
        print "Re-Imaging %s BMS's. Ends.." % bms_list[1:]

    def check_bms_reimage_status(self, svr_list):
        queue = Queue.Queue()
        threads = []
        for bms in svr_list:
            svr = CONF[bms]['hostname']
            t = threading.Thread(target=self.check_reimage_bms_status,
                                 args=(svr, ),
                                 name=svr)
            threads.append(t)
            t.start()
        for t in threads:
            t.join()
        print "Completed reimage status verification for BMS"

    def check_reimage_bms_status(self, bmsname):
        count = 80
        expected_res = "reimage_completed"
        for _ in range(count):
            #Get reimage status
            res = self.sm_api.server_get(server_id=bmsname, status=True)
            print "Waiting for BMS %s to re-image.." % bmsname
            print "Expected status:   %s   ::  Current status:  %s  " % (
                expected_res, res)
            if res == expected_res:
                print "BMS %s is re-imaged" % bmsname
                return
            else:
                print "Retry after 30sec"
                waiting(30)
                if count == 30 and res == "restart_issued":
                    print "Looks like some issue with BMS (%s) powering on, please check" % (
                        bmsname)
                    continue
        print "\n"
        print "==============================================================="
        print "BMS server %s did not complete Re-imaging, hence exiting..." % bmsname
        print "==============================================================="
        sys.exit(1)

    def wait_bms_reboot(self):
        bms_list = CONF['DEFAULTS']['bms']
        for bms in bms_list[1:]:
            count = 40
            for _ in range(count):
                bms_ip = get_ip(CONF[bms]['management_address'])
                cmd = 'ping -c 4 %s' % (bms_ip)
                res = self.sm_vm.execute_cmd(cmd, timeout=30)
                if re.search(r'4 received', res, re.M | re.I):
                    print "MGMT IP(%s)of (%s) is reachable from servermanager." % (
                        bms_ip, bms)
                    continue
                else:
                    print "Waiting for the BMS %s to reboot and come up" % bms
                    print "Retry after 30sec"
                    waiting(30)

    def check_topology_connectivity(self):
        print "Checking the management IP and ctrldata IP reachability from servermanager VM"
        waiting(5)
        all_list = CONF['DEFAULTS']['bms'] + CONF['DEFAULTS']['reimagevms']
        for inventory in all_list:
            mgmt_ip = get_ip(CONF[inventory]['management_address'])
            cmd = 'ping -c 4 %s' % (mgmt_ip)
            res = self.sm_vm.execute_cmd(cmd, timeout=30)
            if re.search(r'4 received', res, re.M | re.I):
                print "MGMT IP(%s)of (%s) is reachable from servermanager." % (
                    mgmt_ip, inventory)
            else:
                print "\n"
                print "Command: %s" % (cmd)
                print "Response: %s" % (res)
                print "=============================================================================="
                print "MGMT IP(%s)of (%s) is not reachable from servermanager.check configuration" % (
                    mgmt_ip, inventory)
                print "=============================================================================="
                sys.exit(1)
        #TO DO check connectivity from cc-host VM, os-host, BMS

    def remove_server2_from_cluster(self):

        bms2 = CONF['DEFAULTS']['bms'][1]
        print "\nRemoving %s from contrail cluster, as it will be used to deploy vMX on it" % (
            bms2)

        svr2_ip = get_ip(CONF[bms2]['management_address'])
        svr2_netmask = get_netmask(CONF[bms2]['management_address'])
        svr2_usr = CONF['DEFAULTS']['root_username']
        svr2_passwd = CONF['DEFAULTS']['root_password']
        svr2_connect = RemoteConnection()
        svr2_connect.connect(svr2_ip, username=svr2_usr, password=svr2_passwd)

        #reboot
        bms2_name = CONF[bms2]['hostname']
        print "Rebooting %s" % (bms2_name)
        cmd = 'reboot'
        svr2_connect.execute_cmd(cmd, timeout=10)
        waiting(10)

        print "Deleting %s from the cluster" % (bms2_name)
        cmd = 'server-manager delete server --server_id %s' % (bms2_name)
        res = self.sm_vm.execute_cmd(cmd, timeout=10)

        #wait till the bms to comes up
        count = 40
        for _ in range(count):
            #ping server2
            res = os.system('ping -c 3 ' + svr2_ip)
            if res == 0:
                print "%s is reachable and is UP" % (bms2_name)
                waiting(60)
                return
            else:
                print "Waiting for %s to come UP.." % (bms2_name)
                print "Retry after 30sec"
                waiting(30)
        print "\n"
        print "========================================================================="
        print "{0} is not reachable, did not boot up with correct mgmt_address {1}".format(
            bms2_anme, svr2_ip)
        print "========================================================================="
        sys.exit(1)

    def provision_contrail(self):
        print "Started provisioning contrail on to the cluster"
        waiting(5)
        provision_list = CONF['DEFAULTS']['reimagevms'] + CONF['DEFAULTS'][
            'bms'][2:]

        self.check_topology_connectivity()
        self.remove_server2_from_cluster()

        cmd = 'yes y | server-manager provision --cluster_id {0} {1}'.format(
            sk_cluster_id, contrail_pkg_id)
        res = self.sm_vm.execute_cmd(cmd, timeout=30)
        self.check_provision_status(provision_list)
        sys.exit(0)

    def check_provision_status(self, provision_list):
        queue = Queue.Queue()
        threads = []
        for svr in provision_list:
            svr = CONF[svr]['hostname']
            t = threading.Thread(target=self.provision_status,
                                 args=(svr, ),
                                 name=svr)
            threads.append(t)
            t.start()
        for t in threads:
            t.join()
        print "Completed contrail provision status verification"

    def provision_status(self, bmsname):
        count = 300
        expected_res = "provision_completed"
        for _ in range(count):
            #Get provision status
            res = self.sm_api.server_get(server_id=bmsname, status=True)
            print "Waiting for BMS %s to get provisioned on %s." % (
                bmsname, sk_cluster_id)
            print "Expected status:   %s        ::      Current status:  %s  " % (
                expected_res, res)
            if res == expected_res:
                print "BMS %s is provisioned" % bmsname
                return
            else:
                print "Retry after 30sec"
                waiting(30)
        print "\n"
        print "================================================="
        print "BMS %s did not get provisioned, hence exiting..." % bmsname
        print "================================================="
        sys.exit(1)