def install_storage_compute_node(*args):
"""Installs storage pkgs in one or list of nodes. USAGE:fab install_compute_storage_node:[email protected],[email protected]"""
for host_string in args:
with settings(host_string=host_string):
pkg = ["contrail-storage"]
if detect_ostype() == "ubuntu":
apt_install_overwrite(pkg)
else:
yum_install(pkg)
def install_storage_webui_node(*args):
"""Installs storage pkgs in one or list of nodes. USAGE:fab install_storage_webui:[email protected],[email protected]"""
for host_string in args:
with settings(host_string=host_string):
pkg = ['contrail-web-storage']
if detect_ostype() == 'Ubuntu':
apt_install(pkg)
else:
yum_install(pkg)
def install_storage_compute_node(*args):
"""Installs storage pkgs in one or list of nodes. USAGE:fab install_compute_storage_node:[email protected],[email protected]"""
for host_string in args:
with settings(host_string=host_string):
pkg = ['contrail-storage']
if detect_ostype() == 'ubuntu':
apt_install_overwrite(pkg)
else:
yum_install(pkg)
def install_storage_master_node(*args):
"""Installs storage pkgs in one or list of nodes. USAGE:fab install_openstack_storage_node:[email protected],[email protected]"""
for host_string in args:
with settings(host_string=host_string):
pkg = ['contrail-storage']
if detect_ostype() == 'ubuntu':
apt_install_overwrite(pkg)
else:
yum_install(pkg)
def install_storage_master_node(*args):
"""Installs storage pkgs in one or list of nodes. USAGE:fab install_openstack_storage_node:[email protected],[email protected]"""
for host_string in args:
with settings(host_string=host_string):
pkg = ['contrail-storage', 'contrail-web-storage']
if detect_ostype() == 'Ubuntu':
apt_install(pkg)
else:
yum_install(pkg)
def upgrade_zookeeper_node(*args):
for host_string in args:
with settings(host_string=host_string):
if detect_ostype() == 'Ubuntu':
print "No need to upgrade specifically zookeeper in ubuntu."
return
with settings(warn_only=True):
run('rpm -e --nodeps zookeeper zookeeper-lib zkpython')
yum_install(['zookeeper'])
run('yum -y --nogpgcheck --disablerepo=* --enablerepo=contrail_install_repo reinstall contrail-config')
run('chkconfig zookeeper on')
def zookeeper_rolling_restart():
zoo_cfg = "/etc/zookeeper/conf/zoo.cfg"
cfgm_nodes = copy.deepcopy(env.roledefs['cfgm'])
database_nodes = copy.deepcopy(env.roledefs['database'])
if (len(database_nodes) % 2) != 1:
print "Recommended to run odd number of zookeeper(database) nodes."
print "Add a new node to the existing clusters testbed,py and install contrail-install-packages in it.\n\
Installing/Provisioning will be done as part of Upgrade"
exit(0)
if cfgm_nodes == database_nodes:
print "No need for rolling restart."
old_nodes = list(set(cfgm_nodes).difference(set(database_nodes)))
new_nodes = list(set(database_nodes).difference(set(cfgm_nodes)))
for new_node in new_nodes:
zk_index = (database_nodes.index(new_node) + len(cfgm_nodes) + 1)
with settings(host_string=new_node, password=env.passwords[new_node]):
pdist = detect_ostype()
# Install zookeeper in the new node.
execute('create_install_repo_node', new_node)
if pdist in ['centos']:
yum_install(['contrail-openstack-database'])
elif pdist in ['Ubuntu']:
apt_install(['contrail-openstack-database'])
run("ln -sf /bin/true /sbin/chkconfig")
run("chkconfig zookeeper on")
# Fix zookeeper configs
run("sudo sed 's/^#log4j.appender.ROLLINGFILE.MaxBackupIndex=/log4j.appender.ROLLINGFILE.MaxBackupIndex=/g' /etc/zookeeper/conf/log4j.properties > log4j.properties.new")
run("sudo mv log4j.properties.new /etc/zookeeper/conf/log4j.properties")
if pdist in ['centos']:
run('echo export ZOO_LOG4J_PROP="INFO,CONSOLE,ROLLINGFILE" >> /usr/lib/zookeeper/bin/zkEnv.sh')
if pdist in ['Ubuntu']:
run('echo ZOO_LOG4J_PROP="INFO,CONSOLE,ROLLINGFILE" >> /etc/zookeeper/conf/environment')
#put cluster-unique zookeeper's instance id in myid
run('sudo echo "%s" > /var/lib/zookeeper/myid' % (zk_index))
# Add new nodes to existing zookeeper quorum
with settings(host_string=cfgm_nodes[0], password=env.passwords[cfgm_nodes[0]]):
for new_node in new_nodes:
zk_index = (database_nodes.index(new_node) + len(cfgm_nodes) + 1)
run('echo "server.%d=%s:2888:3888" >> %s' % (zk_index, hstr_to_ip(new_node), zoo_cfg))
tmp_dir= tempfile.mkdtemp()
get(zoo_cfg, tmp_dir)
# Restart zookeeper in all nodes to make new nodes join zookeeper quorum
for zookeeper_node in cfgm_nodes + new_nodes:
with settings(host_string=zookeeper_node, password=env.passwords[zookeeper_node]):
put(tmp_dir+'/zoo.cfg', zoo_cfg)
# Start Zookeeper in new database node
run("service zookeeper restart")
print "Waiting 5 seconds for the new nodes in the zookeeper quorum to be synced."
sleep(5)
# Shutdown onld nodes one by one and also make sure leader/follower election is complete.
# after each shut downs
zoo_nodes = cfgm_nodes + database_nodes
for old_node in old_nodes:
zoo_nodes.remove(old_node)
with settings(host_string=old_node, password=env.passwords[old_node]):
# Stop Zookeeper in old cfgm node
run("service zookeeper stop")
retries = 3
while retries:
zookeeper_status = verfiy_zookeeper(*zoo_nodes)
if 'leader' in zookeeper_status.values() and 'standalone' not in zookeeper_status.values():
print zookeeper_status
break
else:
retries -= 1
if retries:
continue
print "Zookeepr leader/follower election has problems. Fix it and retry upgrade"
print zookeeper_status
exit(1)
# Correct the server id in zoo.cfg for the new nodes in the zookeeper quorum
with settings(host_string=database_nodes[0], password=env.passwords[database_nodes[0]]):
run("sed -i '/^server.*3888/d' %s" % zoo_cfg)
for zookeeper_node in database_nodes:
zk_index = (database_nodes.index(zookeeper_node) + 1)
run('echo "server.%d=%s:2888:3888" >> %s' % (zk_index, hstr_to_ip(zookeeper_node), zoo_cfg))
tmp_dir= tempfile.mkdtemp()
get(zoo_cfg, tmp_dir)
# Correct the myid in myid file for the new nodes in the zookeeper quorum
for zookeeper_node in database_nodes:
zk_index = (database_nodes.index(zookeeper_node) + 1)
with settings(host_string=zookeeper_node, password=env.passwords[zookeeper_node]):
#put cluster-unique zookeeper's instance id in myid
run('sudo echo "%s" > /var/lib/zookeeper/myid' % (zk_index))
run("service zookeeper stop")
# Restart all the zookeeper nodes in the new quorum
for zookeeper_node in database_nodes:
with settings(host_string=zookeeper_node, password=env.passwords[zookeeper_node]):
put(tmp_dir+'/zoo.cfg', zoo_cfg)
run("service zookeeper restart")
# Mkae sure leader/folower election is complete
with settings(host_string=zookeeper_node, password=env.passwords[zookeeper_node]):
retries = 3
while retries:
zookeeper_status = verfiy_zookeeper(*database_nodes)
if 'leader' in zookeeper_status.values() and 'standalone' not in zookeeper_status.values():
print zookeeper_status
break
else:
retries -= 1
if retries:
continue
print "Zookeepr leader/follower election has problems. Fix it and retry upgrade"
print zookeeper_status
exit(1)
def zookeeper_rolling_restart():
zoo_cfg = "/etc/zookeeper/conf/zoo.cfg"
cfgm_nodes = copy.deepcopy(env.roledefs['cfgm'])
database_nodes = copy.deepcopy(env.roledefs['database'])
if (len(database_nodes) % 2) != 1:
print "Recommended to run odd number of zookeeper(database) nodes."
print "Add a new node to the existing clusters testbed,py and install contrail-install-packages in it.\n\
Installing/Provisioning will be done as part of Upgrade"
exit(0)
if cfgm_nodes == database_nodes:
print "No need for rolling restart."
old_nodes = list(set(cfgm_nodes).difference(set(database_nodes)))
new_nodes = list(set(database_nodes).difference(set(cfgm_nodes)))
for new_node in new_nodes:
zk_index = (database_nodes.index(new_node) + len(cfgm_nodes) + 1)
with settings(host_string=new_node, password=env.passwords[new_node]):
pdist = detect_ostype()
# Install zookeeper in the new node.
execute('create_install_repo_node', new_node)
if pdist in ['centos']:
yum_install(['contrail-openstack-database'])
elif pdist in ['Ubuntu']:
apt_install(['contrail-openstack-database'])
run("ln -sf /bin/true /sbin/chkconfig")
run("chkconfig zookeeper on")
# Fix zookeeper configs
run("sudo sed 's/^#log4j.appender.ROLLINGFILE.MaxBackupIndex=/log4j.appender.ROLLINGFILE.MaxBackupIndex=/g' /etc/zookeeper/conf/log4j.properties > log4j.properties.new"
)
run("sudo mv log4j.properties.new /etc/zookeeper/conf/log4j.properties"
)
if pdist in ['centos']:
run('echo export ZOO_LOG4J_PROP="INFO,CONSOLE,ROLLINGFILE" >> /usr/lib/zookeeper/bin/zkEnv.sh'
)
if pdist in ['Ubuntu']:
run('echo ZOO_LOG4J_PROP="INFO,CONSOLE,ROLLINGFILE" >> /etc/zookeeper/conf/environment'
)
#put cluster-unique zookeeper's instance id in myid
run('sudo echo "%s" > /var/lib/zookeeper/myid' % (zk_index))
# Add new nodes to existing zookeeper quorum
with settings(host_string=cfgm_nodes[0],
password=env.passwords[cfgm_nodes[0]]):
for new_node in new_nodes:
zk_index = (database_nodes.index(new_node) + len(cfgm_nodes) + 1)
run('echo "server.%d=%s:2888:3888" >> %s' %
(zk_index, hstr_to_ip(new_node), zoo_cfg))
tmp_dir = tempfile.mkdtemp()
get(zoo_cfg, tmp_dir)
# Restart zookeeper in all nodes to make new nodes join zookeeper quorum
for zookeeper_node in cfgm_nodes + new_nodes:
with settings(host_string=zookeeper_node,
password=env.passwords[zookeeper_node]):
put(tmp_dir + '/zoo.cfg', zoo_cfg)
# Start Zookeeper in new database node
run("service zookeeper restart")
print "Waiting 5 seconds for the new nodes in the zookeeper quorum to be synced."
sleep(5)
# Shutdown onld nodes one by one and also make sure leader/follower election is complete.
# after each shut downs
zoo_nodes = cfgm_nodes + database_nodes
for old_node in old_nodes:
zoo_nodes.remove(old_node)
with settings(host_string=old_node, password=env.passwords[old_node]):
# Stop Zookeeper in old cfgm node
run("service zookeeper stop")
retries = 3
while retries:
zookeeper_status = verfiy_zookeeper(*zoo_nodes)
if 'leader' in zookeeper_status.values(
) and 'standalone' not in zookeeper_status.values():
print zookeeper_status
break
else:
retries -= 1
if retries:
continue
print "Zookeepr leader/follower election has problems. Fix it and retry upgrade"
print zookeeper_status
exit(1)
# Correct the server id in zoo.cfg for the new nodes in the zookeeper quorum
with settings(host_string=database_nodes[0],
password=env.passwords[database_nodes[0]]):
run("sed -i '/^server.*3888/d' %s" % zoo_cfg)
for zookeeper_node in database_nodes:
zk_index = (database_nodes.index(zookeeper_node) + 1)
run('echo "server.%d=%s:2888:3888" >> %s' %
(zk_index, hstr_to_ip(zookeeper_node), zoo_cfg))
tmp_dir = tempfile.mkdtemp()
get(zoo_cfg, tmp_dir)
# Correct the myid in myid file for the new nodes in the zookeeper quorum
for zookeeper_node in database_nodes:
zk_index = (database_nodes.index(zookeeper_node) + 1)
with settings(host_string=zookeeper_node,
password=env.passwords[zookeeper_node]):
#put cluster-unique zookeeper's instance id in myid
run('sudo echo "%s" > /var/lib/zookeeper/myid' % (zk_index))
run("service zookeeper stop")
# Restart all the zookeeper nodes in the new quorum
for zookeeper_node in database_nodes:
with settings(host_string=zookeeper_node,
password=env.passwords[zookeeper_node]):
put(tmp_dir + '/zoo.cfg', zoo_cfg)
run("service zookeeper restart")
# Mkae sure leader/folower election is complete
with settings(host_string=zookeeper_node,
password=env.passwords[zookeeper_node]):
retries = 3
while retries:
zookeeper_status = verfiy_zookeeper(*database_nodes)
if 'leader' in zookeeper_status.values(
) and 'standalone' not in zookeeper_status.values():
print zookeeper_status
break
else:
retries -= 1
if retries:
continue
print "Zookeepr leader/follower election has problems. Fix it and retry upgrade"
print zookeeper_status
exit(1)
