def get_roles_and_hosts_list(req, cluster_id):
roles_id_list = set()
hosts_id_list = set()
hosts_list = []
cluster_networks = daisy_cmn.get_cluster_networks_detail(req, cluster_id)
roles = daisy_cmn.get_cluster_roles_detail(req,cluster_id)
for role in roles:
if role['deployment_backend'] != daisy_cmn.zenic_backend_name:
continue
role_hosts = daisy_cmn.get_hosts_of_role(req, role['id'])
if role_hosts:
for role_host in role_hosts:
if role_host['host_id'] not in hosts_id_list:
host = daisy_cmn.get_host_detail(req, role_host['host_id'])
host_ip = get_host_network_ip(req, host, cluster_networks, 'MANAGEMENT')
hosts_id_list.add(host['id'])
host_cfg = {}
host_cfg['mgtip'] = host_ip
host_cfg['rootpwd'] = host['root_pwd']
hosts_list.append(host_cfg)
roles_id_list.add(role['id'])
return (roles_id_list, hosts_list)
def _get_roles_and_hosts_ip_list(self, req, cluster_id):
host_ha_list = set()
host_ip_list = set()
role_id_list = set()
hosts_id_list = []
hosts_list = []
roles = daisy_cmn.get_cluster_roles_detail(req,cluster_id)
cluster_networks = daisy_cmn.get_cluster_networks_detail(req, cluster_id)
for role in roles:
if role['deployment_backend'] != daisy_cmn.tecs_backend_name:
continue
role_hosts = daisy_cmn.get_hosts_of_role(req, role['id'])
if role_hosts:
for role_host in role_hosts:
host = daisy_cmn.get_host_detail(req, role_host['host_id'])
host_ip = tecs_cmn.get_host_network_ip(req, host, cluster_networks, 'MANAGEMENT')
if role['name'] == "CONTROLLER_HA":
host_ha_list.add(host_ip)
host_ip_list.add(host_ip)
hosts_id_list.append({host['id']:host_ip})
role_id_list.add(role['id'])
for host in hosts_id_list:
if host not in hosts_list:
hosts_list.append(host)
return (role_id_list, host_ip_list, host_ha_list, hosts_list)
def get_deployment_backends(req, cluster_id, backends_order):
cluster_roles = daisy_cmn.get_cluster_roles_detail(req,cluster_id)
cluster_backends = set([role['deployment_backend'] for role in cluster_roles if daisy_cmn.get_hosts_of_role(req, role['id'])])
ordered_backends = [backend for backend in backends_order if backend in cluster_backends]
other_backends = [backend for backend in cluster_backends if backend not in backends_order]
deployment_backends =ordered_backends + other_backends
return deployment_backends
def get_ha_and_compute_ips(req, cluster_id):
controller_ha_nodes = {}
computer_ips = []
roles = daisy_cmn.get_cluster_roles_detail(req,cluster_id)
cluster_networks = daisy_cmn.get_cluster_networks_detail(req, cluster_id)
for role in roles:
if role['deployment_backend'] != daisy_cmn.tecs_backend_name:
continue
role_hosts = daisy_cmn.get_hosts_of_role(req, role['id'])
for role_host in role_hosts:
#host has installed tecs are exclusive
if (role_host['status'] == tecs_state['ACTIVE'] or
role_host['status'] == tecs_state['UPDATING'] or
role_host['status'] == tecs_state['UPDATE_FAILED']):
continue
host_detail = daisy_cmn.get_host_detail(req,
role_host['host_id'])
host_ip = tecs_cmn.get_host_network_ip(req,
host_detail,
cluster_networks,
'MANAGEMENT')
if role['name'] == "CONTROLLER_HA":
pxe_mac = [interface['mac'] for interface in host_detail['interfaces']
if interface['is_deployment'] == True]
if pxe_mac and pxe_mac[0]:
controller_ha_nodes[host_ip] = pxe_mac[0]
else:
min_mac = get_host_min_mac(host_detail['interfaces'])
controller_ha_nodes[host_ip] = min_mac
if role['name'] == "COMPUTER":
computer_ips.append(host_ip)
return (controller_ha_nodes, computer_ips)
def update_host_progress_to_db(req, role_id_list, host,
status, message, progress=0.0):
for role_id in role_id_list:
role_hosts = daisy_cmn.get_hosts_of_role(req, role_id)
for role_host in role_hosts:
if role_host['host_id'] == host['id']:
role_host['status'] = status
role_host['progress'] = progress
role_host['messages'] = message
daisy_cmn.update_role_host(req, role_host['id'], role_host)
def update_progress_to_db(req, role_id_list, status, hosts_list,host_ip=None):
"""
Write uninstall progress and status to db, we use global lock object 'uninstall_mutex'
to make sure this function is thread safety.
:param req: http req.
:param role_id_list: Column neeb be update in role table.
:param status: Uninstall status.
:return:
"""
for role_id in role_id_list:
role_hosts = daisy_cmn.get_hosts_of_role(req, role_id)
for host_id_ip in hosts_list:
host_ip_tmp=host_id_ip.values()[0]
host_id_tmp=host_id_ip.keys()[0]
if host_ip:
for role_host in role_hosts:
if (host_ip_tmp == host_ip and
role_host['host_id']== host_id_tmp):
role_host_meta = {}
if 0 == cmp(status, tecs_state['UNINSTALLING']):
role_host_meta['progress'] = 10
role_host_meta['messages'] = 'TECS uninstalling'
if 0 == cmp(status, tecs_state['UNINSTALL_FAILED']):
role_host_meta['messages'] = 'TECS uninstalled failed'
elif 0 == cmp(status, tecs_state['ACTIVE']):
role_host_meta['progress'] = 100
role_host_meta['messages'] = 'TECS uninstalled successfully'
if role_host_meta:
role_host_meta['status'] = status
daisy_cmn.update_role_host(req,
role_host['id'],
role_host_meta)
else:
role = {}
if 0 == cmp(status, tecs_state['UNINSTALLING']):
for role_host in role_hosts:
role_host_meta = {}
role_host_meta['status'] = status
role_host_meta['progress'] = 0
daisy_cmn.update_role_host(req,
role_host['id'],
role_host_meta)
role['progress']=0
role['messages'] = 'TECS uninstalling'
if 0 == cmp(status, tecs_state['UNINSTALL_FAILED']):
role['messages'] = 'TECS uninstalled failed'
elif 0 == cmp(status, tecs_state['INIT']):
role['progress'] = 100
role['messages'] = 'TECS uninstalled successfully'
if role:
role['status'] = status
daisy_cmn.update_role(req, role_id, role)
if 0 == cmp(status, tecs_state['INIT']):
daisy_cmn.delete_role_hosts(req, role_id)
def update_progress_to_db(req,role_id_list,status,hosts_list,host_ip=None):
"""
Write update progress and status to db,
to make sure this function is thread safety.
:param req: http req.
:param role_id_list: Column neeb be update in role table.
:param status: Update status.
:return:
"""
for role_id in role_id_list:
role_hosts = daisy_cmn.get_hosts_of_role(req, role_id)
for host_id_ip in hosts_list:
host_ip_tmp=host_id_ip.values()[0]
host_id_tmp=host_id_ip.keys()[0]
if host_ip:
for role_host in role_hosts:
if (host_ip_tmp == host_ip and
role_host['host_id']== host_id_tmp):
role_host_meta = {}
if 0 == cmp(status, tecs_state['UPDATING']):
role_host_meta['progress'] = 10
role_host_meta['messages'] = 'TECS upgrading'
if 0 == cmp(status, tecs_state['UPDATE_FAILED']):
role_host_meta['messages'] = 'TECS upgraded failed'
elif 0 == cmp(status, tecs_state['ACTIVE']):
role_host_meta['progress'] = 100
role_host_meta['messages'] = 'TECS upgraded successfully'
if role_host_meta:
role_host_meta['status'] = status
daisy_cmn.update_role_host(req,
role_host['id'],
role_host_meta)
else:
role = {}
if 0 == cmp(status, tecs_state['UPDATING']):
for role_host in role_hosts:
role_host_meta = {}
role_host_meta['status'] = status
role_host_meta['progress'] = 0
role_host_meta['messages'] = 'TECS upgrading'
daisy_cmn.update_role_host(req,
role_host['id'],
role_host_meta)
role['progress']=0
role['messages'] = 'TECS upgrading'
if 0 == cmp(status, tecs_state['UPDATE_FAILED']):
role['messages'] = 'TECS upgraded failed'
elif 0 == cmp(status, tecs_state['ACTIVE']):
role['progress'] = 100
role['messages'] = 'TECS upgraded successfully'
if role:
role['status'] = status
daisy_cmn.update_role(req, role_id, role)
def update_host_progress_to_db(req,
role_id_list,
host,
status,
message,
progress=0.0):
for role_id in role_id_list:
role_hosts = daisy_cmn.get_hosts_of_role(req, role_id)
for role_host in role_hosts:
if role_host['host_id'] == host['id']:
role_host['status'] = status
role_host['progress'] = progress
role_host['messages'] = message
daisy_cmn.update_role_host(req, role_host['id'], role_host)
def get_deployment_backends(req, cluster_id, backends_order):
cluster_roles = daisy_cmn.get_cluster_roles_detail(req, cluster_id)
cluster_backends = set([
role['deployment_backend'] for role in cluster_roles
if daisy_cmn.get_hosts_of_role(req, role['id'])
])
ordered_backends = [
backend for backend in backends_order if backend in cluster_backends
]
other_backends = [
backend for backend in cluster_backends
if backend not in backends_order
]
deployment_backends = ordered_backends + other_backends
return deployment_backends
def uninstall(self, req, cluster_id):
"""
Uninstall TECS to a cluster.
:param req: The WSGI/Webob Request object
:raises HTTPBadRequest if x-install-cluster is missing
"""
(role_id_list, host_ip_list,host_ha_list, hosts_list) = self._get_roles_and_hosts_ip_list(req, cluster_id)
if role_id_list:
if not host_ip_list:
msg = _("there is no host in cluster %s") % cluster_id
raise exception.ThreadBinException(msg)
unstl.update_progress_to_db(req, role_id_list, tecs_state['UNINSTALLING'], hosts_list)
threads = []
for host_ip in host_ip_list:
t = threading.Thread(target=unstl.thread_bin,args=(req,host_ip,role_id_list,hosts_list))
t.setDaemon(True)
t.start()
threads.append(t)
LOG.info(_("Uninstall threads have started, please waiting...."))
try:
for t in threads:
t.join()
except:
LOG.warn(_("Join uninstall thread %s failed!" % t))
else:
uninstall_failed_flag = False
for role_id in role_id_list:
role_hosts=daisy_cmn.get_hosts_of_role(req,role_id)
for role_host in role_hosts:
if role_host['status'] == tecs_state['UNINSTALL_FAILED']:
unstl.update_progress_to_db(req, role_id_list, tecs_state['UNINSTALL_FAILED'],hosts_list)
uninstall_failed_flag = True
break
if not uninstall_failed_flag:
LOG.info(_("All uninstall threads have done, set all roles status to 'init'!"))
unstl.update_progress_to_db(req, role_id_list, tecs_state['INIT'], hosts_list)
try:
(status, output) = commands.getstatusoutput('rpm -e --nodeps openstack-packstack\
openstack-packstack-puppet openstack-puppet-modules puppet')
except exception.Invalid as e:
raise HTTPBadRequest(explanation=e.msg, request=req)
def valid_used_networks(self, req, cluster_id):
cluster_roles = daisy_cmn.get_cluster_roles_detail(req, cluster_id)
cluster_backends = set([role['deployment_backend']
for role in cluster_roles if
daisy_cmn.get_hosts_of_role(req, role['id'])])
for backend in cluster_backends:
try:
backend_common = importutils.import_module(
'daisy.api.backends.%s.common' % backend)
except Exception:
pass
else:
if hasattr(backend_common, 'get_used_networks'):
networks = backend_common.get_used_networks(req,
cluster_id)
if networks:
common.valid_cluster_networks(networks)
common.check_gateway_uniqueness(networks)
def _update_install_progress_to_db(self):
"""
Update progress of intallation to db.
:return:
"""
roles = daisy_cmn.get_cluster_roles_detail(self.req, self.cluster_id)
for role in roles:
if role['deployment_backend'] != daisy_cmn.proton_backend_name:
continue
role_hosts = daisy_cmn.get_hosts_of_role(self.req, role['id'])
for role_host in role_hosts:
if role_host['status'] != proton_state['ACTIVE']:
self.need_install = True
role_host['status'] = self.state
daisy_cmn.update_role_host(self.req, role_host['id'],
role_host)
role['status'] = self.state
role['messages'] = self.message
daisy_cmn.update_role(self.req, role['id'], role)
def valid_used_networks(self, req, cluster_id):
cluster_roles = daisy_cmn.get_cluster_roles_detail(req, cluster_id)
cluster_backends = set([
role['deployment_backend'] for role in cluster_roles
if daisy_cmn.get_hosts_of_role(req, role['id'])
])
for backend in cluster_backends:
try:
backend_common = importutils.import_module(
'daisy.api.backends.%s.common' % backend)
except Exception:
pass
else:
if hasattr(backend_common, 'get_used_networks'):
networks = backend_common.get_used_networks(
req, cluster_id)
if networks:
common.valid_cluster_networks(networks)
common.check_gateway_uniqueness(networks)
def update_progress_to_db(req, role_id, status, progress_percentage_step=0.0):
"""
Write uninstall progress and status to db, we use global lock object
'uninstall_mutex' to make sure this function is thread safety.
:param req: http req.
:param role_id_list: Column neeb be update in role table.
:param status: Uninstall status.
:return:
"""
global uninstall_mutex
global uninstall_proton_progress
uninstall_mutex.acquire(True)
uninstall_proton_progress -= progress_percentage_step
role = {}
role_hosts = daisy_cmn.get_hosts_of_role(req, role_id)
if status == proton_state['UNINSTALLING']:
role['status'] = status
role['progress'] = uninstall_proton_progress
role['messages'] = 'Proton uninstalling'
for role_host in role_hosts:
role_host_meta = dict()
role_host_meta['status'] = status
role_host_meta['progress'] = uninstall_proton_progress
daisy_cmn.update_role_host(req, role_host['id'], role_host_meta)
if status == proton_state['UNINSTALL_FAILED']:
role['status'] = status
role['messages'] = 'Uninstall-failed'
for role_host in role_hosts:
role_host_meta = dict()
role_host_meta['status'] = status
daisy_cmn.update_role_host(req, role_host['id'], role_host_meta)
elif status == proton_state['INIT']:
role['status'] = status
role['progress'] = 0
role['messages'] = 'Proton uninstall successfully'
daisy_cmn.delete_role_hosts(req, role_id)
daisy_cmn.update_role(req, role_id, role)
uninstall_mutex.release()
def get_roles_and_hosts_list(req, cluster_id):
roles_id_list = set()
hosts_id_list = set()
hosts_list = []
cluster_networks = daisy_cmn.get_cluster_networks_detail(req, cluster_id)
roles = daisy_cmn.get_cluster_roles_detail(req, cluster_id)
for role in roles:
if role['deployment_backend'] != daisy_cmn.kolla_backend_name:
continue
role_hosts = daisy_cmn.get_hosts_of_role(req, role['id'])
if role_hosts:
for role_host in role_hosts:
if role_host['host_id'] not in hosts_id_list:
host = daisy_cmn.get_host_detail(req, role_host['host_id'])
host_ip = get_host_network_ip(req, host, cluster_networks,
'MANAGEMENT')
hosts_id_list.add(host['id'])
host_cfg = {}
host_cfg['id'] = host['id']
host_cfg['mgtip'] = host_ip
host_cfg['rootpwd'] = host['root_pwd']
hosts_list.append(host_cfg)
roles_id_list.add(role['id'])
return (roles_id_list, hosts_id_list, hosts_list)
def upgrade(self, req, cluster_id):
"""
update TECS to a cluster.
:param req: The WSGI/Webob Request object
:raises HTTPBadRequest if x-install-cluster is missing
"""
(role_id_list,host_ip_list,host_ha_list,hosts_list) = self._get_roles_and_hosts_ip_list(req, cluster_id)
if role_id_list:
if not host_ip_list:
msg = _("there is no host in cluster %s") % cluster_id
raise exception.ThreadBinException(msg)
unreached_hosts = daisy_cmn.check_ping_hosts(host_ip_list, 1)
if unreached_hosts:
self.message = "hosts %s ping failed" % unreached_hosts
raise exception.NotFound(message=self.message)
daisy_cmn.subprocess_call('rm -rf /root/.ssh/known_hosts')
if os_handle.check_tfg_exist():
os_handle.upgrade_os(req, hosts_list)
unreached_hosts = daisy_cmn.check_ping_hosts(host_ip_list, 30)
if unreached_hosts:
self.message = "hosts %s ping failed after tfg upgrade" % unreached_hosts
raise exception.NotFound(message=self.message)
# check and get TECS version
tecs_version_pkg_file = tecs_cmn.check_and_get_tecs_version(tecs_cmn.daisy_tecs_path)
if not tecs_version_pkg_file:
self.state = tecs_state['INSTALL_FAILED']
self.message = "TECS version file not found in %s" % tecs_cmn.daisy_tecs_path
raise exception.NotFound(message=self.message)
threads = []
LOG.info(_("Begin to update TECS controller nodes, please waiting...."))
upgrd.update_progress_to_db(req, role_id_list, tecs_state['UPDATING'], hosts_list)
for host_ip in host_ha_list:
LOG.info(_("Update TECS controller node %s..." % host_ip))
rc = upgrd.thread_bin(req,role_id_list,host_ip,hosts_list)
if rc == 0:
LOG.info(_("Update TECS for %s successfully" % host_ip))
else:
LOG.info(_("Update TECS failed for %s, return %s" % (host_ip,rc)))
return
LOG.info(_("Begin to update TECS other nodes, please waiting...."))
max_parallel_upgrade_number = int(CONF.max_parallel_os_upgrade_number)
compute_ip_list = host_ip_list - host_ha_list
while compute_ip_list:
threads = []
if len(compute_ip_list) > max_parallel_upgrade_number:
upgrade_hosts = compute_ip_list[:max_parallel_upgrade_number]
compute_ip_list = compute_ip_list[max_parallel_upgrade_number:]
else:
upgrade_hosts = compute_ip_list
compute_ip_list = []
for host_ip in upgrade_hosts:
t = threading.Thread(target=upgrd.thread_bin,args=(req,role_id_list,host_ip,hosts_list))
t.setDaemon(True)
t.start()
threads.append(t)
try:
for t in threads:
t.join()
except:
LOG.warn(_("Join update thread %s failed!" % t))
for role_id in role_id_list:
role_hosts=daisy_cmn.get_hosts_of_role(req,role_id)
for role_host in role_hosts:
if (role_host['status'] == tecs_state['UPDATE_FAILED'] or
role_host['status'] == tecs_state['UPDATING']):
role_id = [role_host['role_id']]
upgrd.update_progress_to_db(req,
role_id,
tecs_state['UPDATE_FAILED'],
hosts_list)
break
elif role_host['status'] == tecs_state['ACTIVE']:
role_id = [role_host['role_id']]
upgrd.update_progress_to_db(req,
role_id,
tecs_state['ACTIVE'],
hosts_list)