def setup_keepalived():
"""Task to provision VIP for openstack nodes with keepalived"""
mgmt_ip = hstr_to_ip(env.host_string)
self_host = get_control_host_string(env.host_string)
self_ip = hstr_to_ip(self_host)
openstack_host_password = env.passwords[env.host_string]
if (getattr(env, 'openstack_admin_password', None)):
openstack_admin_password = env.openstack_admin_password
else:
openstack_admin_password = '******'
internal_vip = get_from_testbed_dict('ha', 'internal_vip', None)
external_vip = get_from_testbed_dict('ha', 'external_vip', None)
openstack_host_list = [get_control_host_string(openstack_host)\
for openstack_host in env.roledefs['openstack']]
with cd(INSTALLER_DIR):
cmd = "PASSWORD=%s ADMIN_TOKEN=%s python setup-vnc-keepalived.py\
--self_ip %s --internal_vip %s --mgmt_self_ip %s\
--openstack_index %d" % (
openstack_host_password, openstack_admin_password, self_ip,
internal_vip, mgmt_ip, (openstack_host_list.index(self_host) + 1))
if external_vip:
cmd += ' --external_vip %s' % external_vip
run(cmd)
def setup_keepalived():
"""Task to provision VIP for openstack nodes with keepalived"""
mgmt_ip = hstr_to_ip(env.host_string)
self_host = get_control_host_string(env.host_string)
self_ip = hstr_to_ip(self_host)
openstack_host_password = env.passwords[env.host_string]
if (getattr(env, 'openstack_admin_password', None)):
openstack_admin_password = env.openstack_admin_password
else:
openstack_admin_password = '******'
internal_vip = get_from_testbed_dict('ha', 'internal_vip', None)
external_vip = get_from_testbed_dict('ha', 'external_vip', None)
openstack_host_list = [get_control_host_string(openstack_host)\
for openstack_host in env.roledefs['openstack']]
myindex = openstack_host_list.index(self_host)
if myindex >= 1:
# Wait for VIP to be assiciated to MASTER
with settings(host_string=env.roledefs['openstack'][0], warn_only=True):
while run("ip addr | grep %s" % internal_vip).failed:
sleep(2)
print "Waiting for VIP to be associated to MASTER VRRP."
continue
with cd(INSTALLER_DIR):
cmd = "PASSWORD=%s ADMIN_TOKEN=%s python setup-vnc-keepalived.py\
--self_ip %s --internal_vip %s --mgmt_self_ip %s\
--openstack_index %d --num_nodes %d" % (openstack_host_password,
openstack_admin_password, self_ip, internal_vip, mgmt_ip,
(openstack_host_list.index(self_host) + 1), len(env.roledefs['openstack']))
if external_vip:
cmd += ' --external_vip %s' % external_vip
run(cmd)
def setup_keepalived():
"""Task to provision VIP for openstack nodes with keepalived"""
mgmt_ip = hstr_to_ip(env.host_string)
self_host = get_control_host_string(env.host_string)
self_ip = hstr_to_ip(self_host)
openstack_host_password = env.passwords[env.host_string]
if (getattr(env, 'openstack_admin_password', None)):
openstack_admin_password = env.openstack_admin_password
else:
openstack_admin_password = '******'
internal_vip = get_from_testbed_dict('ha', 'internal_vip', None)
external_vip = get_from_testbed_dict('ha', 'external_vip', None)
openstack_host_list = [get_control_host_string(openstack_host)\
for openstack_host in env.roledefs['openstack']]
with cd(INSTALLER_DIR):
cmd = "PASSWORD=%s ADMIN_TOKEN=%s python setup-vnc-keepalived.py\
--self_ip %s --internal_vip %s --mgmt_self_ip %s\
--openstack_index %d" % (openstack_host_password,
openstack_admin_password, self_ip, internal_vip, mgmt_ip,
(openstack_host_list.index(self_host) + 1))
if external_vip:
cmd += ' --external_vip %s' % external_vip
run(cmd)
def setup_keepalived():
"""Task to provision VIP for openstack nodes with keepalived"""
mgmt_ip = hstr_to_ip(env.host_string)
self_host = get_control_host_string(env.host_string)
self_ip = hstr_to_ip(self_host)
openstack_host_password = env.passwords[env.host_string]
if (getattr(env, 'openstack_admin_password', None)):
openstack_admin_password = env.openstack_admin_password
else:
openstack_admin_password = '******'
internal_vip = get_from_testbed_dict('ha', 'internal_vip', None)
external_vip = get_from_testbed_dict('ha', 'external_vip', None)
openstack_host_list = [get_control_host_string(openstack_host)\
for openstack_host in env.roledefs['openstack']]
myindex = openstack_host_list.index(self_host)
if myindex >= 1:
# Wait for VIP to be assiciated to MASTER
with settings(host_string=env.roledefs['openstack'][0], warn_only=True):
while run("ip addr | grep %s" % internal_vip).failed:
sleep(2)
print "Waiting for VIP to be associated to MASTER VRRP."
continue
with cd(INSTALLER_DIR):
cmd = "PASSWORD=%s ADMIN_TOKEN=%s python setup-vnc-keepalived.py\
--self_ip %s --internal_vip %s --mgmt_self_ip %s\
--openstack_index %d --num_nodes %d" % (openstack_host_password,
openstack_admin_password, self_ip, internal_vip, mgmt_ip,
(openstack_host_list.index(self_host) + 1), len(env.roledefs['openstack']))
if external_vip:
cmd += ' --external_vip %s' % external_vip
run(cmd)
def mount_glance_images():
nfs_server = get_from_testbed_dict('ha', 'nfs_server', hstr_to_ip(get_nfs_server()))
nfs_glance_path = get_from_testbed_dict('ha', 'nfs_glance_path', '/var/tmp/glance-images/')
with settings(warn_only=True):
out = sudo('sudo mount %s:%s /var/lib/glance/images' % (nfs_server, nfs_glance_path))
if out.failed and 'already mounted' not in out:
raise RuntimeError(out)
if sudo('grep "%s:%s /var/lib/glance/images nfs" /etc/fstab' % (nfs_server, nfs_glance_path)).failed:
sudo('echo "%s:%s /var/lib/glance/images nfs nfsvers=3,hard,intr,auto 0 0" >> /etc/fstab' % (nfs_server, nfs_glance_path))
def mount_glance_images():
nfs_server = get_from_testbed_dict('ha', 'nfs_server', hstr_to_ip(env.roledefs['compute'][0]))
nfs_glance_path = get_from_testbed_dict('ha', 'nfs_glance_path', '/var/tmp/glance-images/')
with settings(warn_only=True):
out = run('sudo mount %s:%s /var/lib/glance/images' % (nfs_server, nfs_glance_path))
if out.failed and 'already mounted' not in out:
raise RuntimeError(out)
if run('grep "%s:%s /var/lib/glance/images nfs" /etc/fstab' % (nfs_server, nfs_glance_path)).failed:
run('echo "%s:%s /var/lib/glance/images nfs nfsvers=3,hard,intr,auto 0 0" >> /etc/fstab' % (nfs_server, nfs_glance_path))
def setup_rabbitmq_cluster(force=False):
"""Task to cluster the rabbit servers."""
amqp_roles = []
rabbit_servers = get_from_testbed_dict('cfgm', 'amqp_hosts', None)
if rabbit_servers:
print "Using external rabbitmq servers %s" % rabbit_servers
else:
# Provision rabbitmq cluster in cfgm role nodes.
print "Provisioning rabbitq in cfgm nodes"
amqp_roles = ['cfgm']
# Provision rabbitmq cluster in openstack on request
if get_from_testbed_dict('openstack', 'manage_amqp', 'no') == 'yes':
# Provision rabbitmq cluster in openstack role nodes aswell.
amqp_roles.append('openstack')
for role in amqp_roles:
env.roledefs['rabbit'] = env.roledefs[role]
if not force:
with settings(warn_only=True):
result = execute("verify_cluster_status", retry='no')
if result and False not in result.values():
print "RabbitMQ cluster is up and running in role[%s]; No need to cluster again." % role
continue
rabbitmq_cluster_uuid = getattr(testbed, 'rabbitmq_cluster_uuid', None)
if not rabbitmq_cluster_uuid:
rabbitmq_cluster_uuid = uuid.uuid4()
if not is_xenial_or_above():
execute(listen_at_supervisor_support_port)
execute(remove_mnesia_database)
execute(verify_rabbit_node_hostname)
execute(allow_rabbitmq_port)
execute(rabbitmq_env)
execute(config_rabbitmq)
execute("stop_rabbitmq_and_set_cookie", rabbitmq_cluster_uuid)
execute(start_rabbitmq)
# adding sleep to workaround rabbitmq bug 26370 prevent
# "rabbitmqctl cluster_status" from breaking the database,
# this is seen in ci
time.sleep(60)
#execute(rabbitmqctl_stop_app)
#execute(rabbitmqctl_reset)
#execute("rabbitmqctl_start_app_node", env.roledefs['rabbit'][0])
#execute(add_node_to_rabbitmq_cluster)
#execute(rabbitmqctl_start_app)
if (role is 'openstack' and get_openstack_internal_vip() or
role is 'cfgm' and get_contrail_internal_vip()):
execute('set_ha_policy_in_rabbitmq')
execute('set_tcp_keepalive')
execute('set_tcp_keepalive_on_compute')
result = execute(verify_cluster_status)
if False in result.values():
print "Unable to setup RabbitMQ cluster in role[%s]...." % role
exit(1)
def setup_rabbitmq_cluster(force=False):
"""Task to cluster the rabbit servers."""
amqp_roles = []
rabbit_servers = get_from_testbed_dict('cfgm', 'amqp_hosts', None)
if rabbit_servers:
print "Using external rabbitmq servers %s" % rabbit_servers
else:
# Provision rabbitmq cluster in cfgm role nodes.
print "Provisioning rabbitq in cfgm nodes"
amqp_roles = ['cfgm']
# Provision rabbitmq cluster in openstack on request
if get_from_testbed_dict('openstack', 'manage_amqp', 'no') == 'yes':
# Provision rabbitmq cluster in openstack role nodes aswell.
amqp_roles.append('openstack')
for role in amqp_roles:
env.roledefs['rabbit'] = env.roledefs[role]
if not force:
with settings(warn_only=True):
result = execute("verify_cluster_status", retry='no')
if result and False not in result.values():
print "RabbitMQ cluster is up and running in role[%s]; No need to cluster again." % role
continue
rabbitmq_cluster_uuid = getattr(testbed, 'rabbitmq_cluster_uuid', None)
if not rabbitmq_cluster_uuid:
rabbitmq_cluster_uuid = uuid.uuid4()
execute(listen_at_supervisor_support_port)
execute(remove_mnesia_database)
execute(verify_rabbit_node_hostname)
execute(allow_rabbitmq_port)
execute(rabbitmq_env)
execute(config_rabbitmq)
execute("stop_rabbitmq_and_set_cookie", rabbitmq_cluster_uuid)
execute(start_rabbitmq)
# adding sleep to workaround rabbitmq bug 26370 prevent
# "rabbitmqctl cluster_status" from breaking the database,
# this is seen in ci
time.sleep(60)
#execute(rabbitmqctl_stop_app)
#execute(rabbitmqctl_reset)
#execute("rabbitmqctl_start_app_node", env.roledefs['rabbit'][0])
#execute(add_node_to_rabbitmq_cluster)
#execute(rabbitmqctl_start_app)
if (role is 'openstack' and get_openstack_internal_vip() or
role is 'cfgm' and get_contrail_internal_vip()):
execute('set_ha_policy_in_rabbitmq')
execute('set_tcp_keepalive')
execute('set_tcp_keepalive_on_compute')
result = execute(verify_cluster_status)
if False in result.values():
print "Unable to setup RabbitMQ cluster in role[%s]...." % role
exit(1)
def setup_glance_images_loc():
nfs_server = get_from_testbed_dict('ha', 'nfs_server', None)
nfs_glance_path = get_from_testbed_dict('ha', 'nfs_glance_path', '/var/tmp/glance-images/')
if not nfs_server:
with settings(host_string=env.roledefs['compute'][0]):
run('mkdir -p /var/tmp/glance-images/')
run('chmod 777 /var/tmp/glance-images/')
run('echo "/var/tmp/glance-images *(rw,sync,no_subtree_check)" >> /etc/exports')
run('sudo /etc/init.d/nfs-kernel-server restart')
execute('mount_glance_images')
def setup_glance_images_loc():
nfs_server = get_from_testbed_dict('ha', 'nfs_server', None)
nfs_glance_path = get_from_testbed_dict('ha', 'nfs_glance_path', '/var/tmp/glance-images/')
if not nfs_server:
with settings(host_string=env.roledefs['compute'][0]):
run('mkdir -p /var/tmp/glance-images/')
run('chmod 777 /var/tmp/glance-images/')
run('echo "/var/tmp/glance-images *(rw,sync,no_subtree_check)" >> /etc/exports')
run('sudo /etc/init.d/nfs-kernel-server restart')
execute('mount_glance_images')
def setup_glance_images_loc():
nfs_server = get_from_testbed_dict("ha", "nfs_server", None)
nfs_glance_path = get_from_testbed_dict("ha", "nfs_glance_path", "/var/tmp/glance-images/")
if not nfs_server:
nfs_server = get_nfs_server()
with settings(host_string=nfs_server):
sudo("mkdir -p /var/tmp/glance-images/")
sudo("chmod 777 /var/tmp/glance-images/")
sudo('echo "/var/tmp/glance-images *(rw,sync,no_subtree_check)" >> /etc/exports')
sudo("sudo /etc/init.d/nfs-kernel-server restart")
execute("mount_glance_images")
def setup_galera_cluster():
"""Task to cluster the openstack nodes with galera"""
if len(env.roledefs['openstack']) <= 1:
print "Single Openstack cluster, skipping galera cluster setup."
return
if env.roledefs['openstack'].index(env.host_string) == 0:
execute('setup_passwordless_ssh', *env.roledefs['openstack'])
self_host = get_control_host_string(env.host_string)
self_ip = hstr_to_ip(self_host)
openstack_host_password = env.passwords[env.host_string]
if (getattr(env, 'openstack_admin_password', None)):
openstack_admin_password = env.openstack_admin_password
else:
openstack_admin_password = '******'
openstack_host_list = [get_control_host_string(openstack_host)\
for openstack_host in env.roledefs['openstack']]
galera_ip_list = [hstr_to_ip(galera_host)\
for galera_host in openstack_host_list]
keystone_ip = get_keystone_ip()
internal_vip = get_from_testbed_dict('ha', 'internal_vip', None)
with cd(INSTALLER_DIR):
run("PASSWORD=%s ADMIN_TOKEN=%s python setup-vnc-galera.py\
--self_ip %s --keystone_ip %s --galera_ip_list %s\
--internal_vip %s --openstack_index %d" %
(openstack_host_password, openstack_admin_password, self_ip,
keystone_ip, ' '.join(galera_ip_list), internal_vip,
(openstack_host_list.index(self_host) + 1)))
def setup_galera_cluster():
"""Task to cluster the openstack nodes with galera"""
if len(env.roledefs['openstack']) <= 1:
print "Single Openstack cluster, skipping galera cluster setup."
return
if env.roledefs['openstack'].index(env.host_string) == 0:
execute('setup_passwordless_ssh', *env.roledefs['openstack'])
self_host = get_control_host_string(env.host_string)
self_ip = hstr_to_ip(self_host)
openstack_host_password = env.passwords[env.host_string]
if (getattr(env, 'openstack_admin_password', None)):
openstack_admin_password = env.openstack_admin_password
else:
openstack_admin_password = '******'
openstack_host_list = [get_control_host_string(openstack_host)\
for openstack_host in env.roledefs['openstack']]
galera_ip_list = [hstr_to_ip(galera_host)\
for galera_host in openstack_host_list]
keystone_ip = get_keystone_ip()
internal_vip = get_from_testbed_dict('ha', 'internal_vip', None)
with cd(INSTALLER_DIR):
run("PASSWORD=%s ADMIN_TOKEN=%s python setup-vnc-galera.py\
--self_ip %s --keystone_ip %s --galera_ip_list %s\
--internal_vip %s --openstack_index %d" % (openstack_host_password,
openstack_admin_password, self_ip, keystone_ip,
' '.join(galera_ip_list), internal_vip,
(openstack_host_list.index(self_host) + 1)))
def setup_rabbitmq_cluster(force=False):
"""Task to cluster the rabbit servers."""
if len(env.roledefs['cfgm']) <= 1:
print "Single cfgm cluster, skipping rabbitmq cluster setup."
return
if not force:
with settings(warn_only=True):
result = execute(verify_cluster_status)
if result and False not in result.values():
print "RabbitMQ cluster is up and running; No need to cluster again."
return
rabbitmq_cluster_uuid = getattr(testbed, 'rabbitmq_cluster_uuid', None)
if not rabbitmq_cluster_uuid:
rabbitmq_cluster_uuid = uuid.uuid4()
execute(listen_at_supervisor_config_port)
execute(remove_mnesia_database)
execute(verify_cfgm_hostname)
execute(allow_rabbitmq_port)
execute(config_rabbitmq)
execute("stop_rabbitmq_and_set_cookie", rabbitmq_cluster_uuid)
execute(start_rabbitmq)
#execute(rabbitmqctl_stop_app)
#execute(rabbitmqctl_reset)
#execute("rabbitmqctl_start_app_node", env.roledefs['cfgm'][0])
#execute(add_cfgm_to_rabbitmq_cluster)
#execute(rabbitmqctl_start_app)
if get_from_testbed_dict('ha', 'internal_vip', None):
execute('set_ha_policy_in_rabbitmq')
result = execute(verify_cluster_status)
if False in result.values():
print "Unable to setup RabbitMQ cluster...."
exit(1)
def setup_ha():
execute('pre_check')
if get_from_testbed_dict('ha', 'internal_vip', None):
print "Multi Openstack setup, provisioning openstack HA."
execute('setup_keepalived')
execute('setup_galera_cluster')
execute('fix_wsrep_cluster_address')
execute('fixup_restart_haproxy_in_openstack')
execute('setup_glance_images_loc')
execute('fix_memcache_conf')
execute('tune_tcp')
def install_openstack_node(*args):
"""Installs openstack pkgs in one or list of nodes. USAGE:fab install_openstack_node:[email protected],[email protected]"""
for host_string in args:
with settings(host_string=host_string):
pkg = ['contrail-openstack']
if len(env.roledefs['openstack']) > 1 and get_from_testbed_dict('ha', 'internal_vip', None):
pkg.append('contrail-openstack-ha')
if detect_ostype() == 'Ubuntu':
apt_install(pkg)
else:
yum_install(pkg)
def install_openstack_node(*args):
"""Installs openstack pkgs in one or list of nodes. USAGE:fab install_openstack_node:[email protected],[email protected]"""
for host_string in args:
with settings(host_string=host_string):
pkg = ['contrail-openstack']
if len(env.roledefs['openstack']) > 1 and get_from_testbed_dict(
'ha', 'internal_vip', None):
pkg.append('contrail-openstack-ha')
if detect_ostype() == 'Ubuntu':
apt_install(pkg)
else:
yum_install(pkg)
def setup_ha():
execute('pre_check')
if get_from_testbed_dict('ha', 'internal_vip', None):
print "Multi Openstack setup, provisioning openstack HA."
execute('setup_keepalived')
execute('setup_galera_cluster')
execute('fix_wsrep_cluster_address')
execute('fix_restart_xinetd_conf')
execute('fixup_restart_haproxy_in_openstack')
execute('fixup_restart_haproxy_in_collector')
execute('setup_glance_images_loc')
execute('fix_memcache_conf')
execute('tune_tcp')
execute('fix_cmon_param_and_add_keys_to_compute')
execute('create_and_copy_service_token')
def setup_ha():
execute('pre_check')
if get_from_testbed_dict('ha', 'internal_vip', None):
print "Multi Openstack setup, provisioning openstack HA."
execute('setup_keepalived')
execute('setup_galera_cluster')
execute('fix_wsrep_cluster_address')
execute('fix_restart_xinetd_conf')
execute('fixup_restart_haproxy_in_openstack')
execute('fixup_restart_haproxy_in_collector')
execute('setup_glance_images_loc')
execute('fix_memcache_conf')
execute('tune_tcp')
execute('fix_cmon_param_and_add_keys_to_compute')
execute('create_and_copy_service_token')
def fix_cmon_param_and_add_keys_to_compute():
cmon_param = '/etc/contrail/ha/cmon_param'
compute_host_list = []
for host_string in env.roledefs['compute']:
with settings(host_string=host_string,
password=env.passwords[host_string]):
host_name = sudo('hostname')
compute_host_list.append(host_name)
# Get AMQP host list
amqp_in_role = 'cfgm'
if get_from_testbed_dict('openstack', 'manage_amqp', 'no') == 'yes':
amqp_in_role = 'openstack'
amqp_host_list = []
for host_string in env.roledefs[amqp_in_role]:
with settings(host_string=host_string,
password=env.passwords[host_string]):
host_name = sudo('hostname')
amqp_host_list.append(host_name)
computes = 'COMPUTES=("' + '" "'.join(compute_host_list) + '")'
sudo("echo '%s' >> %s" % (computes, cmon_param))
sudo("echo 'COMPUTES_SIZE=${#COMPUTES[@]}' >> %s" % cmon_param)
sudo("echo 'COMPUTES_USER=root' >> %s" % cmon_param)
sudo("echo 'PERIODIC_RMQ_CHK_INTER=60' >> %s" % cmon_param)
sudo("echo 'RABBITMQ_RESET=True' >> %s" % cmon_param)
amqps = 'DIPHOSTS=("' + '" "'.join(amqp_host_list) + '")'
sudo("echo '%s' >> %s" % (amqps, cmon_param))
sudo("echo 'DIPS_HOST_SIZE=${#DIPHOSTS[@]}' >> %s" % cmon_param)
id_rsa_pubs = {}
if files.exists('~/.ssh', use_sudo=True):
sudo('chmod 700 ~/.ssh')
if (not files.exists('~/.ssh/id_rsa', use_sudo=True)
and not files.exists('~/.ssh/id_rsa.pub', use_sudo=True)):
sudo('ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N ""')
elif (not files.exists('~/.ssh/id_rsa', use_sudo=True)
or not files.exists('~/.ssh/id_rsa.pub', use_sudo=True)):
sudo('rm -rf ~/.ssh/id_rsa*')
sudo('ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N ""')
id_rsa_pubs.update({env.host_string: sudo('cat ~/.ssh/id_rsa.pub')})
for host_string in env.roledefs['compute']:
with settings(host_string=host_string):
sudo("mkdir -p ~/.ssh/")
for host, id_rsa_pub in id_rsa_pubs.items():
files.append('~/.ssh/authorized_keys',
id_rsa_pub,
use_sudo=True)
sudo('chmod 640 ~/.ssh/authorized_keys')
def fix_cmon_param_and_add_keys_to_compute():
cmon_param = '/etc/contrail/ha/cmon_param'
compute_host_list = []
for host_string in env.roledefs['compute']:
with settings(host_string=host_string, password=get_env_passwords(host_string)):
host_name = sudo('hostname')
compute_host_list.append(host_name)
# Get AMQP host list
amqp_in_role = 'cfgm'
if get_from_testbed_dict('openstack', 'manage_amqp', 'no') == 'yes':
amqp_in_role = 'openstack'
amqp_host_list = []
for host_string in env.roledefs[amqp_in_role]:
with settings(host_string=host_string, password=get_env_passwords(host_string)):
host_name = sudo('hostname -s')
amqp_host_list.append(host_name)
computes = 'COMPUTES=("' + '" "'.join(compute_host_list) + '")'
sudo("grep -q 'COMPUTES' %s || echo '%s' >> %s" % (cmon_param, computes, cmon_param))
sudo("grep -q 'COMPUTES_SIZE' %s || echo 'COMPUTES_SIZE=${#COMPUTES[@]}' >> %s" % (cmon_param, cmon_param))
sudo("grep -q 'COMPUTES_USER' %s || echo 'COMPUTES_USER=root' >> %s" % (cmon_param, cmon_param))
sudo("grep -q 'PERIODIC_RMQ_CHK_INTER' %s || echo 'PERIODIC_RMQ_CHK_INTER=60' >> %s" % (cmon_param, cmon_param))
sudo("grep -q 'RABBITMQ_RESET' %s || echo 'RABBITMQ_RESET=True' >> %s" % (cmon_param, cmon_param))
amqps = 'DIPHOSTS=("' + '" "'.join(amqp_host_list) + '")'
sudo("grep -q 'DIPHOSTS' %s || echo '%s' >> %s" % (cmon_param, amqps, cmon_param))
sudo("grep -q 'DIPS_HOST_SIZE' %s || echo 'DIPS_HOST_SIZE=${#DIPHOSTS[@]}' >> %s" % (cmon_param, cmon_param))
sudo("sort %s | uniq > /tmp/cmon_param" % cmon_param)
sudo("mv /tmp/cmon_param %s" % cmon_param)
id_rsa_pubs = {}
if files.exists('~/.ssh', use_sudo=True):
sudo('chmod 700 ~/.ssh')
if (not files.exists('~/.ssh/id_rsa', use_sudo=True) and
not files.exists('~/.ssh/id_rsa.pub', use_sudo=True)):
sudo('ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N ""')
elif (not files.exists('~/.ssh/id_rsa', use_sudo=True) or
not files.exists('~/.ssh/id_rsa.pub', use_sudo=True)):
sudo('rm -rf ~/.ssh/id_rsa*')
sudo('ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N ""')
id_rsa_pubs.update({env.host_string : sudo('cat ~/.ssh/id_rsa.pub')})
for host_string in env.roledefs['compute']:
with settings(host_string=host_string):
sudo("mkdir -p ~/.ssh/")
for host, id_rsa_pub in id_rsa_pubs.items():
files.append('~/.ssh/authorized_keys',
id_rsa_pub, use_sudo=True)
sudo('chmod 640 ~/.ssh/authorized_keys')
def setup_rabbitmq_cluster(force=False):
"""Task to cluster the rabbit servers."""
# Provision rabbitmq cluster in cfgm role nodes.
amqp_roles = ['cfgm']
if get_from_testbed_dict('openstack', 'manage_amqp', 'no') == 'yes':
# Provision rabbitmq cluster in openstack role nodes aswell.
amqp_roles.append('openstack')
for role in amqp_roles:
env.roledefs['rabbit'] = env.roledefs[role]
if not force:
with settings(warn_only=True):
result = execute(verify_cluster_status)
if result and False not in result.values():
print "RabbitMQ cluster is up and running in role[%s]; No need to cluster again." % role
continue
rabbitmq_cluster_uuid = getattr(testbed, 'rabbitmq_cluster_uuid', None)
if not rabbitmq_cluster_uuid:
rabbitmq_cluster_uuid = uuid.uuid4()
execute(listen_at_supervisor_config_port)
execute(remove_mnesia_database)
execute(verify_rabbit_node_hostname)
execute(allow_rabbitmq_port)
execute(config_rabbitmq)
execute("stop_rabbitmq_and_set_cookie", rabbitmq_cluster_uuid)
execute(start_rabbitmq)
if len(env.roledefs['rabbit']) <= 1:
print "Single cfgm cluster, Starting rabbitmq."
return
#execute(rabbitmqctl_stop_app)
#execute(rabbitmqctl_reset)
#execute("rabbitmqctl_start_app_node", env.roledefs['rabbit'][0])
#execute(add_node_to_rabbitmq_cluster)
#execute(rabbitmqctl_start_app)
if (role is 'openstack' and get_openstack_internal_vip() or
role is 'cfgm' and get_contrail_internal_vip()):
execute('set_ha_policy_in_rabbitmq')
execute('set_tcp_keepalive')
execute('set_tcp_keepalive_on_compute')
result = execute(verify_cluster_status)
if False in result.values():
print "Unable to setup RabbitMQ cluster in role[%s]...." % role
exit(1)
def join_rabbitmq_cluster(new_ctrl_host):
""" Task to join a new rabbit server into an existing cluster """
# Provision rabbitmq cluster in cfgm role nodes.
amqp_roles = ['cfgm']
if get_from_testbed_dict('openstack', 'manage_amqp', 'no') == 'yes':
#Provision rabbitmq cluster in openstack role nodes aswell.
amqp_roles.append('openstack')
for role in amqp_roles:
env.roledefs['rabbit'] = env.roledefs[role]
# copy the erlang cookie from one of the other nodes.
rabbitmq_cluster_uuid = None
for host_string in env.roledefs['rabbit']:
with settings(host_string=host_string, warn_only=True):
if host_string != new_ctrl_host and\
sudo('ls /var/lib/rabbitmq/.erlang.cookie').succeeded:
rabbitmq_cluster_uuid = \
sudo('cat /var/lib/rabbitmq/.erlang.cookie')
break;
if rabbitmq_cluster_uuid is None:
raise RuntimeError("Not able to get the Erlang cookie from the cluster nodes")
if not is_xenial_or_above():
execute(listen_at_supervisor_support_port_node, new_ctrl_host)
execute(remove_mnesia_database_node, new_ctrl_host)
execute(verify_rabbit_node_hostname)
execute(allow_rabbitmq_port_node, new_ctrl_host)
execute(rabbitmq_env)
execute(config_rabbitmq)
execute('stop_rabbitmq_and_set_cookie_node', rabbitmq_cluster_uuid, new_ctrl_host)
execute('start_rabbitmq_node', new_ctrl_host)
# adding sleep to workaround rabbitmq bug 26370 prevent
# "rabbitmqctl cluster_status" from breaking the database,
# this is seen in ci
time.sleep(30)
if (role is 'openstack' and get_openstack_internal_vip() or
role is 'cfgm' and get_contrail_internal_vip()):
execute('set_ha_policy_in_rabbitmq')
execute('set_tcp_keepalive')
result = execute(verify_cluster_status)
if False in result.values():
print "Unable to setup RabbitMQ cluster in role[%s]...." % role
exit(1)
def purge_node_from_rabbitmq_cluster(del_rabbitmq_node, role):
if get_from_testbed_dict('openstack', 'manage_amqp', 'no') == 'no' and\
role == 'openstack':
# We are not managing the RabbitMQ server. No-op.
return
if get_contrail_internal_vip() != get_openstack_internal_vip() and\
role == 'cfgm':
# Openstack and Contrail are in two different nodes. Cfgm
# rabbitmq will point to the Openstack node. No-op.
return
env.roledefs['rabbit'] = env.roledefs[role]
del_rabbitmq_ip = hstr_to_ip(del_rabbitmq_node)
del_rabbitmq_ctrl_ip = hstr_to_ip(get_control_host_string(del_rabbitmq_node))
if ping_test(del_rabbitmq_node):
with settings(host_string = del_rabbitmq_node, warn_only = True):
sudo("rabbitmqctl stop_app")
sudo("rabbitmqctl reset")
sudo("service supervisor-support-service stop")
sudo("mv /var/lib/rabbitmq/.erlang.cookie /var/lib/rabbitmq/.erlang.cookie.removed")
sudo("mv /etc/rabbitmq/rabbitmq.config /etc/rabbitmq/rabbitmq.config.removed")
else:
# If the node is not reachable, then delete the node remotely from one
# of the nodes in the cluster.
with settings(host_string = env.roledefs['rabbit'][0], warn_only = True):
hostname = local('getent hosts %s | awk \'{print $3\'}' % del_rabbitmq_ctrl_ip, capture = True)
sudo("rabbitmqctl forget_cluster_node rabbit@%s" % hostname)
# Giving some time for the other nodes to re-adjust the cluster,
time.sleep(30)
execute(config_rabbitmq)
for host_string in env.roledefs[role]:
with settings(host_string = host_string):
sudo("service rabbitmq-server restart")
# Give time for RabbitMQ to recluster
time.sleep(30)
result = execute(verify_cluster_status)
if False in result.values():
print "Unable to recluster RabbitMQ cluster after removing the node %s" % del_rabbitmq_node
exit(1)
def setup_rabbitmq_cluster(force=False):
"""Task to cluster the rabbit servers."""
# Provision rabbitmq cluster in cfgm role nodes.
amqp_roles = ["cfgm"]
if get_from_testbed_dict("openstack", "manage_amqp", "no") == "yes":
# Provision rabbitmq cluster in openstack role nodes aswell.
amqp_roles.append("openstack")
for role in amqp_roles:
env.roledefs["rabbit"] = env.roledefs[role]
if not force:
with settings(warn_only=True):
result = execute("verify_cluster_status", retry="no")
if result and False not in result.values():
print "RabbitMQ cluster is up and running in role[%s]; No need to cluster again." % role
continue
rabbitmq_cluster_uuid = getattr(testbed, "rabbitmq_cluster_uuid", None)
if not rabbitmq_cluster_uuid:
rabbitmq_cluster_uuid = uuid.uuid4()
execute(listen_at_supervisor_support_port)
execute(remove_mnesia_database)
execute(verify_rabbit_node_hostname)
execute(allow_rabbitmq_port)
execute(rabbitmq_env)
execute(config_rabbitmq)
execute("stop_rabbitmq_and_set_cookie", rabbitmq_cluster_uuid)
execute(start_rabbitmq)
# adding sleep to workaround rabbitmq bug 26370 prevent "rabbitmqctl cluster_status" from breaking the database, this is seen in ci
time.sleep(60)
# execute(rabbitmqctl_stop_app)
# execute(rabbitmqctl_reset)
# execute("rabbitmqctl_start_app_node", env.roledefs['rabbit'][0])
# execute(add_node_to_rabbitmq_cluster)
# execute(rabbitmqctl_start_app)
if role is "openstack" and get_openstack_internal_vip() or role is "cfgm" and get_contrail_internal_vip():
execute("set_ha_policy_in_rabbitmq")
execute("set_tcp_keepalive")
execute("set_tcp_keepalive_on_compute")
result = execute(verify_cluster_status)
if False in result.values():
print "Unable to setup RabbitMQ cluster in role[%s]...." % role
exit(1)
def fix_cmon_param_and_add_keys_to_compute():
cmon_param = "/etc/contrail/ha/cmon_param"
compute_host_list = []
for host_string in env.roledefs["compute"]:
with settings(host_string=host_string, password=get_env_passwords(host_string)):
host_name = sudo("hostname")
compute_host_list.append(host_name)
# Get AMQP host list
amqp_in_role = "cfgm"
if get_from_testbed_dict("openstack", "manage_amqp", "no") == "yes":
amqp_in_role = "openstack"
amqp_host_list = []
for host_string in env.roledefs[amqp_in_role]:
with settings(host_string=host_string, password=get_env_passwords(host_string)):
host_name = sudo("hostname")
amqp_host_list.append(host_name)
computes = 'COMPUTES=("' + '" "'.join(compute_host_list) + '")'
sudo("echo '%s' >> %s" % (computes, cmon_param))
sudo("echo 'COMPUTES_SIZE=${#COMPUTES[@]}' >> %s" % cmon_param)
sudo("echo 'COMPUTES_USER=root' >> %s" % cmon_param)
sudo("echo 'PERIODIC_RMQ_CHK_INTER=60' >> %s" % cmon_param)
sudo("echo 'RABBITMQ_RESET=True' >> %s" % cmon_param)
amqps = 'DIPHOSTS=("' + '" "'.join(amqp_host_list) + '")'
sudo("echo '%s' >> %s" % (amqps, cmon_param))
sudo("echo 'DIPS_HOST_SIZE=${#DIPHOSTS[@]}' >> %s" % cmon_param)
sudo("sort %s | uniq > /tmp/cmon_param" % cmon_param)
sudo("mv /tmp/cmon_param %s" % cmon_param)
id_rsa_pubs = {}
if files.exists("~/.ssh", use_sudo=True):
sudo("chmod 700 ~/.ssh")
if not files.exists("~/.ssh/id_rsa", use_sudo=True) and not files.exists("~/.ssh/id_rsa.pub", use_sudo=True):
sudo('ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N ""')
elif not files.exists("~/.ssh/id_rsa", use_sudo=True) or not files.exists("~/.ssh/id_rsa.pub", use_sudo=True):
sudo("rm -rf ~/.ssh/id_rsa*")
sudo('ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N ""')
id_rsa_pubs.update({env.host_string: sudo("cat ~/.ssh/id_rsa.pub")})
for host_string in env.roledefs["compute"]:
with settings(host_string=host_string):
sudo("mkdir -p ~/.ssh/")
for host, id_rsa_pub in id_rsa_pubs.items():
files.append("~/.ssh/authorized_keys", id_rsa_pub, use_sudo=True)
sudo("chmod 640 ~/.ssh/authorized_keys")
def join_rabbitmq_cluster(new_ctrl_host):
""" Task to join a new rabbit server into an existing cluster """
# Provision rabbitmq cluster in cfgm role nodes.
amqp_roles = ['cfgm']
if get_from_testbed_dict('openstack', 'manage_amqp', 'no') == 'yes':
#Provision rabbitmq cluster in openstack role nodes aswell.
amqp_roles.append('openstack')
for role in amqp_roles:
env.roledefs['rabbit'] = env.roledefs[role]
# copy the erlang cookie from one of the other nodes.
rabbitmq_cluster_uuid = None
for host_string in env.roledefs['rabbit']:
with settings(host_string=host_string, warn_only=True):
if host_string != new_ctrl_host and\
sudo('ls /var/lib/rabbitmq/.erlang.cookie').succeeded:
rabbitmq_cluster_uuid = \
sudo('cat /var/lib/rabbitmq/.erlang.cookie')
break;
if rabbitmq_cluster_uuid is None:
raise RuntimeError("Not able to get the Erlang cookie from the cluster nodes")
execute(listen_at_supervisor_support_port_node, new_ctrl_host)
execute(remove_mnesia_database_node, new_ctrl_host)
execute(verify_rabbit_node_hostname)
execute(allow_rabbitmq_port_node, new_ctrl_host)
execute(rabbitmq_env)
execute(config_rabbitmq)
execute('stop_rabbitmq_and_set_cookie_node', rabbitmq_cluster_uuid, new_ctrl_host)
execute('start_rabbitmq_node', new_ctrl_host)
# adding sleep to workaround rabbitmq bug 26370 prevent
# "rabbitmqctl cluster_status" from breaking the database,
# this is seen in ci
time.sleep(30)
if (role is 'openstack' and get_openstack_internal_vip() or
role is 'cfgm' and get_contrail_internal_vip()):
execute('set_ha_policy_in_rabbitmq')
execute('set_tcp_keepalive')
result = execute(verify_cluster_status)
if False in result.values():
print "Unable to setup RabbitMQ cluster in role[%s]...." % role
exit(1)
def fix_cmon_param_and_add_keys_to_compute():
cmon_param = '/etc/contrail/ha/cmon_param'
compute_host_list = []
for host_string in env.roledefs['compute']:
with settings(host_string=host_string, password=env.passwords[host_string]):
host_name = run('hostname')
compute_host_list.append(host_name)
# Get AMQP host list
amqp_in_role = 'cfgm'
if get_from_testbed_dict('openstack', 'manage_amqp', 'no') == 'yes':
amqp_in_role = 'openstack'
amqp_host_list = []
for host_string in env.roledefs[amqp_in_role]:
with settings(host_string=host_string, password=env.passwords[host_string]):
host_name = run('hostname')
amqp_host_list.append(host_name)
computes = 'COMPUTES=("' + '" "'.join(compute_host_list) + '")'
run("echo '%s' >> %s" % (computes, cmon_param))
run("echo 'COMPUTES_SIZE=${#COMPUTES[@]}' >> %s" % cmon_param)
run("echo 'COMPUTES_USER=root' >> %s" % cmon_param)
run("echo 'PERIODIC_RMQ_CHK_INTER=60' >> %s" % cmon_param)
amqps = 'DIPHOSTS=("' + '" "'.join(amqp_host_list) + '")'
run("echo '%s' >> %s" % (amqps, cmon_param))
run("echo 'DIPS_HOST_SIZE=${#DIPHOSTS[@]}' >> %s" % cmon_param)
id_rsa_pubs = {}
if files.exists('/root/.ssh'):
run('chmod 700 /root/.ssh')
if not files.exists('/root/.ssh/id_rsa') and not files.exists('/root/.ssh/id_rsa.pub'):
run('ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N ""')
elif not files.exists('/root/.ssh/id_rsa') or not files.exists('/root/.ssh/id_rsa.pub'):
run('rm -rf /root/.ssh/id_rsa*')
run('ssh-keygen -b 2048 -t rsa -f /root/.ssh/id_rsa -q -N ""')
id_rsa_pubs.update({env.host_string : run('cat /root/.ssh/id_rsa.pub')})
for host_string in env.roledefs['compute']:
with settings(host_string=host_string):
run("mkdir -p /root/.ssh/")
for host, id_rsa_pub in id_rsa_pubs.items():
files.append('/root/.ssh/authorized_keys', id_rsa_pub)
run('chmod 640 /root/.ssh/authorized_keys')
def purge_node_from_rabbitmq_cluster(del_rabbitmq_node, role):
if get_from_testbed_dict('openstack', 'manage_amqp', 'no') == 'no' and\
role == 'openstack':
# We are not managing the RabbitMQ server. No-op.
return
env.roledefs['rabbit'] = env.roledefs[role]
del_rabbitmq_ip = hstr_to_ip(del_rabbitmq_node)
del_rabbitmq_ctrl_ip = hstr_to_ip(get_control_host_string(del_rabbitmq_node))
if ping_test(del_rabbitmq_node):
with settings(host_string = del_rabbitmq_node, warn_only = True):
sudo("rabbitmqctl stop_app")
sudo("rabbitmqctl reset")
sudo("service supervisor-support-service stop")
sudo("mv /var/lib/rabbitmq/.erlang.cookie /var/lib/rabbitmq/.erlang.cookie.removed")
sudo("mv /etc/rabbitmq/rabbitmq.config /etc/rabbitmq/rabbitmq.config.removed")
else:
# If the node is not reachable, then delete the node remotely from one
# of the nodes in the cluster.
with settings(host_string = env.roledefs['rabbit'][0], warn_only = True):
hostname = local('getent hosts %s | awk \'{print $3\'}' % del_rabbitmq_ctrl_ip, capture = True)
sudo("rabbitmqctl forget_cluster_node rabbit@%s" % hostname)
# Giving some time for the other nodes to re-adjust the cluster,
time.sleep(30)
execute(config_rabbitmq)
for host_string in env.roledefs[role]:
with settings(host_string = host_string):
sudo("service rabbitmq-server restart")
# Give time for RabbitMQ to recluster
time.sleep(30)
result = execute(verify_cluster_status)
if False in result.values():
print "Unable to recluster RabbitMQ cluster after removing the node %s" % del_rabbitmq_node
exit(1)
def purge_node_from_rabbitmq_cluster(del_rabbitmq_node, role):
if get_from_testbed_dict("openstack", "manage_amqp", "no") == "no" and role == "openstack":
# We are not managing the RabbitMQ server. No-op.
return
env.roledefs["rabbit"] = env.roledefs[role]
del_rabbitmq_ip = hstr_to_ip(del_rabbitmq_node)
del_rabbitmq_ctrl_ip = hstr_to_ip(get_control_host_string(del_rabbitmq_node))
if ping_test(del_rabbitmq_node):
with settings(host_string=del_rabbitmq_node, warn_only=True):
sudo("rabbitmqctl stop_app")
sudo("rabbitmqctl reset")
sudo("service supervisor-support-service stop")
sudo("mv /var/lib/rabbitmq/.erlang.cookie /var/lib/rabbitmq/.erlang.cookie.removed")
sudo("mv /etc/rabbitmq/rabbitmq.config /etc/rabbitmq/rabbitmq.config.removed")
else:
# If the node is not reachable, then delete the node remotely from one
# of the nodes in the cluster.
with settings(host_string=env.roledefs["rabbit"][0], warn_only=True):
hostname = local("getent hosts %s | awk '{print $3'}" % del_rabbitmq_ctrl_ip, capture=True)
sudo("rabbitmqctl forget_cluster_node rabbit@%s" % hostname)
# Giving some time for the other nodes to re-adjust the cluster,
time.sleep(30)
execute(config_rabbitmq)
for host_string in env.roledefs[role]:
with settings(host_string=host_string):
sudo("service rabbitmq-server restart")
# Give time for RabbitMQ to recluster
time.sleep(30)
result = execute(verify_cluster_status)
if False in result.values():
print "Unable to recluster RabbitMQ cluster after removing the node %s" % del_rabbitmq_node
exit(1)
def setup_rabbitmq_cluster(force=False):
"""Task to cluster the rabbit servers."""
if len(env.roledefs['cfgm']) <= 1:
print "Single cfgm cluster, skipping rabbitmq cluster setup."
return
if not force:
with settings(warn_only=True):
result = execute(verify_cluster_status)
if result and False not in result.values():
print "RabbitMQ cluster is up and running; No need to cluster again."
return
rabbitmq_cluster_uuid = getattr(testbed, 'rabbitmq_cluster_uuid', None)
if not rabbitmq_cluster_uuid:
rabbitmq_cluster_uuid = uuid.uuid4()
execute(listen_at_supervisor_config_port)
execute(remove_mnesia_database)
execute(verify_cfgm_hostname)
execute(allow_rabbitmq_port)
execute(config_rabbitmq)
execute("stop_rabbitmq_and_set_cookie", rabbitmq_cluster_uuid)
execute(start_rabbitmq)
#execute(rabbitmqctl_stop_app)
#execute(rabbitmqctl_reset)
#execute("rabbitmqctl_start_app_node", env.roledefs['cfgm'][0])
#execute(add_cfgm_to_rabbitmq_cluster)
#execute(rabbitmqctl_start_app)
if get_from_testbed_dict('ha', 'internal_vip', None):
execute('set_ha_policy_in_rabbitmq')
execute('set_tcp_keepalive')
result = execute(verify_cluster_status)
if False in result.values():
print "Unable to setup RabbitMQ cluster...."
exit(1)
def get_vcenter_port():
return get_from_testbed_dict('vcenter', 'port', '443')
def get_vcenter_ip():
return get_from_testbed_dict('vcenter', 'server', None)
def get_vcenter_admin_password():
return get_from_testbed_dict('vcenter', 'password', 'Contrail123!')
def get_vcenter_admin_user():
return get_from_testbed_dict('vcenter', 'username', '*****@*****.**')
