def check_rabbitmq():
# node role check
if not NODE_ROLE.is_fuel():
if not NODE_ROLE.is_controller():
LOG.warn('This command can only run on fuel or controller node !')
return
if NODE_ROLE.is_fuel():
check_all_nodes('rabbitmq')
return
LOG.info('%s%s Checking rabbitmq cluster status' % ('=' * 5, '>'))
# get all controller node hostname
controllers = get_controllers_hostname()
if controllers is None:
LOG.error('Can not get the controllers node list !')
return
# get masters & slaves node list
running_nodes = get_rabbitmq_nodes()
if running_nodes is None:
LOG.error('Can not get the running node list for rabbitmq cluster !')
return
# check all controller nodes in masters + slaves node list
error_nodes = []
for node in controllers:
if node.split('.')[0] not in running_nodes:
error_nodes.append(node)
if error_nodes:
LOG.error('Node %s not in rabbitmq cluster !' % error_nodes)
LOG.error('Rabbitmq cluster check faild !')
else:
LOG.info('Rabbitmq cluster check successfully !')
def check_all_nodes(check_obj):
if check_obj is "all":
if LOG.enable_debug:
check_cmd = "sudo eayunstack --debug doctor cls --all"
else:
check_cmd = "sudo eayunstack doctor cls --all"
else:
if LOG.enable_debug:
check_cmd = "sudo eayunstack --debug doctor cls -n %s" % check_obj
else:
check_cmd = "sudo eayunstack doctor cls -n %s" % check_obj
# get controller node list
node_list = get_node_list("controller")
# ssh to all controller node to check obj
if len(node_list) == 0:
LOG.warn("Node list is null !")
return
else:
if check_obj == "ceph":
# only need to check one node for ceph cluster
ceph_node = node_list[0]
LOG.info("%s Role: %-10s Node: %-13s %s" % ("*" * 15, "controller", ceph_node, "*" * 15))
ssh_connect2(ceph_node, check_cmd)
else:
for node in node_list:
LOG.info("%s Role: %-10s Node: %-13s %s" % ("*" * 15, "controller", node, "*" * 15))
ssh_connect2(node, check_cmd)
def base_delete(self, resource_name, resource_ids, delete_func):
no_log_resources = []
while resource_ids:
for resource_id in resource_ids:
# avoid LOG delete info many times
if resource_id not in no_log_resources:
with log_disabled():
LOG.info('Delete %s [%s]' %
(resource_name, resource_id))
no_log_resources.append(resource_id)
try:
delete_func(resource_id)
# delete successfully, break
resource_ids.remove(resource_id)
break
except Conflict:
# retry: deal with conflict.
continue
except NotFound:
# when call destroy_volume(),
# will delete volumes and snapshots,
# if snapshots NotFound, do nothing.
resource_ids.remove(resource_id)
break
except Exception as e:
LOG.warn('Can not delete %s [%s]' %
(resource_name, resource_id))
LOG.error(e)
# something else wrong, break, won't retry
resource_ids.remove(resource_id)
break
def _run(self):
self.base_delete('floating ip', self.floatingips,
neutronclient.delete_floatingip)
for port_id in self.ports:
try:
with log_disabled():
LOG.info('Delete port [%s]' % port_id)
neutronclient.delete_port(port_id)
except Conflict as e:
with log_disabled():
LOG.info(' Solving conflict: remove interface...')
router_id = neutronclient.show_port(
port_id)['port']['device_id']
neutronclient.remove_interface_router(router_id,
{'port_id': port_id})
except Exception as e:
LOG.warn('Can not delete port [%s]' % port_id)
LOG.error(e)
# if firewall create with target router,
# CAN NOT delete router before firewall is deleted.
# NOTE: already add retry
self.base_delete('router', self.routers, neutronclient.delete_router)
self.base_delete('subnet', self.subnets, neutronclient.delete_subnet)
self.base_delete('network', self.networks,
neutronclient.delete_network)
def check_mysql():
# node role check
if not NODE_ROLE.is_fuel():
if not NODE_ROLE.is_controller():
LOG.warn('This command can only run on fuel or controller node !')
return
if NODE_ROLE.is_fuel():
check_all_nodes('mysql')
return
LOG.info('%s%s Checking mysql cluster status' % ('=' * 5, '>'))
# get running node list for mysql cluster
running_nodes = get_mysql_nodes()
if running_nodes is None:
LOG.error('Can not get the running node list for mysql cluster !')
return
# get all controller node hostname
controllers = get_controllers_hostname()
if controllers is None:
LOG.error('Can not get the controllers node list !')
return
# check all controller node in mysql cluster
error_nodes = []
for node in controllers:
if node not in running_nodes:
error_nodes.append(node)
if error_nodes:
LOG.error('Node %s is not running in mysql cluster !' % error_nodes)
LOG.error('Mysql cluster check faild !')
else:
LOG.info('Mysql cluster check successfully !')
def push_conf_file_to_influxdbnode():
LOG.info('Push conf file to influxdb node.')
push_repo_file_to_node(INFLUXDB_HOST, 'influxdb_grafana',
INFLUXDB_REPO_CONF_FILEPATH, backup=True)
push_repo_file_to_node(INFLUXDB_HOST, 'nailgun',
NAILGUN_REPO_CONF_FILEPATH)
push_yaml_to_node(INFLUXDB_HOST, ASTUTE_CONF_FILEPATH, 'astute.yaml')
def update():
'''update eayunstack-tools on all nodes'''
node_list = get_node_list('all')
update_cmd = 'yum clean all && yum -y -d 0 update eayunstack-tools'
results = run_cmd_on_nodes(node_list, update_cmd)
get_current_version = \
'rpm --queryformat "%{VERSION} %{RELEASE}" -q eayunstack-tools'
current_version = run_cmd_on_nodes(node_list, get_current_version)
correct_version = run_command(
'rpm --queryformat "%{VERSION} %{RELEASE}" -q eayunstack-tools')
for node in node_list:
out = results[node][0]
err = results[node][1]
current_ver = current_version[node][0].split(' ')[0] + \
'-' + current_version[node][0].split(' ')[1].split('.')[0]
correct_ver = correct_version[0].split(' ')[0] + \
'-' + correct_version[0].split(' ')[1].split('.')[0]
if err or current_ver != correct_ver:
LOG.error('Update on %s failed !' % node)
LOG.error('Current version: %s' % current_ver)
for l in err.split('\n'):
LOG.error(l)
print
else:
LOG.info('Update on %s successfully.' % node)
LOG.info('Current version: %s' % current_ver)
print
def delete_backend_snapshots_rbd(snapshots_id, volume_id):
success = True
rbd_pool = get_backend_pool(volume_id)
LOG.info(' Deleting backend(rbd) snapshots ...')
for snapshot_id in snapshots_id:
LOG.info(' [%s]Deleting backend snapshot ...' % snapshot_id)
(s, o) = commands.getstatusoutput(
'rbd -p %s snap unprotect --image volume-%s --snap snapshot-%s'
% (rbd_pool, volume_id, snapshot_id))
if s == 0:
(ss, oo) = commands.getstatusoutput(
'rbd -p %s snap rm --image volume-%s --snap snapshot-%s'
% (rbd_pool, volume_id, snapshot_id))
if ss != 0:
LOG.error('Can not delete backend snapshot "snapshot-%s" !' % snapshot_id)
success = False
elif o.find('No such file or directory') > 0:
LOG.error(' This snapshot does not exist !')
success = False
elif o.find('Device or resource busy') > 0:
LOG.error(' Unprotecting snapshot failed. Device or resource busy !')
success = False
else:
success = False
return success
def check_all_nodes(check_obj):
if check_obj is 'all':
if LOG.enable_debug:
check_cmd = 'sudo eayunstack --debug doctor cls --all'
else:
check_cmd = 'sudo eayunstack doctor cls --all'
else:
if LOG.enable_debug:
check_cmd = 'sudo eayunstack --debug doctor cls -n %s' % check_obj
else:
check_cmd = 'sudo eayunstack doctor cls -n %s' % check_obj
# get controller node list
node_list = get_node_list('controller')
# ssh to all controller node to check obj
if len(node_list) == 0:
LOG.warn('Node list is null !')
return
else:
if check_obj == 'ceph':
# only need to check one node for ceph cluster
ceph_node = node_list[0]
run_doctor_cmd_on_node('controller', ceph_node, check_cmd)
else:
(proc_list, pipe) = run_doctor_on_nodes('controller', node_list, check_cmd)
for proc in proc_list:
proc.join()
LOG.info(pipe.recv(), remote=True)
def delete_backend_volume_eqlx(volume_id):
# get provider_location
sql_get_provider_location = 'SELECT provider_location FROM volumes WHERE id =\'%s\';' % volume_id
provider_location = str(db_connect(sql_get_provider_location)).split()[1]
# ssh to controller and compute node to delete the iscsi connection
LOG.info(' Deleting iscsi connection ...')
controller_list = get_node_list('controller')
compute_list = get_node_list('compute')
node_list = controller_list + compute_list
for node in node_list:
cmd_show = 'iscsiadm -m session -o show'
(out, err) = ssh_connect(node, cmd_show)
if out.find(provider_location) > 0:
cmd_delete = 'iscsiadm -m node -u -T %s' % provider_location
(o, e) = ssh_connect(node, cmd_delete)
if o.find('successful') < 0:
LOG.error(
' Can not delete the iscsi connection "%s" at host %s.' %
(provider_location, node))
# ssh to eqlx to delete the volume
LOG.info(' Deleting backend(eqlx) volume ...')
## set eqlx volume status to offline
cmd_set_eqlx_volume_offline = 'volume select volume-%s offline' % volume_id
out = eqlx_ssh_execute(cmd_set_eqlx_volume_offline)
if len(out) == 3:
LOG.error(' ' + out[1])
return False
## delete the eqlx volume
cmd_delete_eqlx_volume = 'volume delete volume-%s' % volume_id
result = eqlx_ssh_execute(cmd_delete_eqlx_volume)
if not result or result[1] != 'Volume deletion succeeded.':
LOG.error(' Delete backend volume faild !')
return False
else:
return True
def delete_snapshots(snapshots_id):
LOG.info('Deleting snapshot %s ...' % snapshots_id)
if delete_backend_snapshots(snapshots_id):
update_snapshots_db(snapshots_id)
return True
else:
return False
def check_mysql():
# node role check
if not NODE_ROLE.is_fuel():
if not NODE_ROLE.is_controller():
LOG.warn('This command can only run on fuel or controller node !')
return
if NODE_ROLE.is_fuel():
check_all_nodes('mysql')
return
LOG.info('%s%s Checking mysql cluster status' %('='*5, '>'))
# get running node list for mysql cluster
running_nodes = get_mysql_nodes()
if running_nodes is None:
LOG.error('Can not get the running node list for mysql cluster !')
return
# get all controller node hostname
controllers = get_controllers_hostname()
if controllers is None:
LOG.error('Can not get the controllers node list !')
return
# check all controller node in mysql cluster
error_nodes = []
for node in controllers:
if node not in running_nodes:
error_nodes.append(node)
if error_nodes:
LOG.error('Node %s is not running in mysql cluster !' % error_nodes)
LOG.error('Mysql cluster check faild !')
else:
LOG.info('Mysql cluster check successfully !')
def delete_volume(volume_id, volume_status):
LOG.info('Deleting volume %s ...' % volume_id)
if volume_status == 'creating':
update_db(volume_id)
else:
if delete_backend_volume(volume_id):
update_db(volume_id)
def check_all_nodes(check_obj):
if check_obj is 'all':
if LOG.enable_debug:
check_cmd = 'sudo eayunstack --debug doctor cls --all'
else:
check_cmd = 'sudo eayunstack doctor cls --all'
else:
if LOG.enable_debug:
check_cmd = 'sudo eayunstack --debug doctor cls -n %s' % check_obj
else:
check_cmd = 'sudo eayunstack doctor cls -n %s' % check_obj
# get controller node list
node_list = get_node_list('controller')
# ssh to all controller node to check obj
if len(node_list) == 0:
LOG.warn('Node list is null !')
return
else:
if check_obj == 'ceph':
# only need to check one node for ceph cluster
ceph_node = node_list[0]
LOG.info('%s Role: %-10s Node: %-13s %s'
% ('*'*15, 'controller', ceph_node, '*'*15))
ssh_connect2(ceph_node, check_cmd)
else:
for node in node_list:
LOG.info('%s Role: %-10s Node: %-13s %s'
% ('*'*15, 'controller', node, '*'*15))
ssh_connect2(node, check_cmd)
def check_all_nodes(check_obj):
if check_obj is 'all':
if LOG.enable_debug:
check_cmd = 'sudo eayunstack --debug doctor cls --all'
else:
check_cmd = 'sudo eayunstack doctor cls --all'
else:
if LOG.enable_debug:
check_cmd = 'sudo eayunstack --debug doctor cls -n %s' % check_obj
else:
check_cmd = 'sudo eayunstack doctor cls -n %s' % check_obj
# get controller node list
node_list = get_node_list('controller')
# ssh to all controller node to check obj
if len(node_list) == 0:
LOG.warn('Node list is null !')
return
else:
if check_obj == 'ceph':
# only need to check one node for ceph cluster
ceph_node = node_list[0]
res = run_doctor_cmd_on_node('controller', ceph_node, check_cmd)
LOG.info(res, remote=True)
else:
nodes = []
for node in node_list:
node_info = {}
node_info['role'] = 'controller'
node_info['name'] = node
nodes.append(node_info)
result = run_doctor_on_nodes(nodes, check_cmd)
for res in result:
LOG.info(res, remote=True)
def check_all_openstack_node_online(nodes_info):
LOG.info('Checking all openstack node is online ...')
for node in nodes_info:
host = str(node['fuelweb_admin'].split('/')[0])
if not check_node_online(host):
return False
return True
def _check_managed_status(resource):
if resource['@managed'] == 'true':
LOG.info('Resource %s was managed on node %s' \
% (resource['@id'], resource['node']['@name']))
else:
LOG.error('Resource %s was unmanaged on node %s' \
% (resource['@id'], resource['node']['@name']))
def delete_volume(volume_id, volume_status):
LOG.info('Deleting volume %s ...' % volume_id)
if volume_status == 'creating':
update_db(volume_id)
else:
if delete_backend_volume(volume_id):
update_db(volume_id)
def delete_backend_volume_eqlx(volume_id):
# get provider_location
sql_get_provider_location = 'SELECT provider_location FROM volumes WHERE id =\'%s\';' % volume_id
provider_location = str(db_connect(sql_get_provider_location)).split()[1]
# ssh to controller and compute node to delete the iscsi connection
LOG.info(' Deleting iscsi connection ...')
controller_list = get_node_list('controller')
compute_list = get_node_list('compute')
node_list = controller_list + compute_list
for node in node_list:
cmd_show = 'iscsiadm -m session -o show'
(out, err) = ssh_connect(node, cmd_show)
if out.find(provider_location) > 0:
cmd_delete = 'iscsiadm -m node -u -T %s' % provider_location
(o, e) = ssh_connect(node, cmd_delete)
if o.find('successful') < 0:
LOG.error(' Can not delete the iscsi connection "%s" at host %s.' % (provider_location, node))
# ssh to eqlx to delete the volume
LOG.info(' Deleting backend(eqlx) volume ...')
## set eqlx volume status to offline
cmd_set_eqlx_volume_offline = 'volume select volume-%s offline' % volume_id
out = eqlx_ssh_execute(cmd_set_eqlx_volume_offline)
if len(out) == 3:
LOG.error(' ' + out[1])
return False
## delete the eqlx volume
cmd_delete_eqlx_volume = 'volume delete volume-%s' % volume_id
result = eqlx_ssh_execute(cmd_delete_eqlx_volume)
if not result or result[1] != 'Volume deletion succeeded.':
LOG.error(' Delete backend volume faild !')
return False
else:
return True
def update_volume_quota(volume_id):
LOG.info(' [%s]Updating volume quota ...' % volume_id)
# get volume size & project id
sql_get_size_project_id = 'SELECT size,project_id FROM volumes WHERE id=\'%s\';' % volume_id
get_size_project_id = db_connect(sql_get_size_project_id)
size = get_size_project_id[0]
project_id = get_size_project_id[1]
# get backend type
backend_type = get_backend_type(volume_id)
sql_update_gigabytes = 'UPDATE quota_usages SET in_use=in_use-%s where project_id=\'%s\' and resource=\'gigabytes\';' % (
size, project_id)
sql_update_volumes = 'UPDATE quota_usages SET in_use=in_use-1 where project_id=\'%s\' and resource=\'volumes\';' % project_id
db_connect(sql_update_gigabytes)
db_connect(sql_update_volumes)
if backend_type == 'eqlx':
sql_update_gigabytes_eqlx = 'UPDATE quota_usages SET in_use=in_use-%s where project_id=\'%s\' and resource=\'gigabytes_eqlx\';' % (
size, project_id)
sql_update_volumes_eqlx = 'UPDATE quota_usages SET in_use=in_use-1 where project_id=\'%s\' and resource=\'volumes_eqlx\';' % project_id
db_connect(sql_update_gigabytes_eqlx)
db_connect(sql_update_volumes_eqlx)
elif backend_type == 'rbd':
sql_update_gigabytes_rbd = 'UPDATE quota_usages SET in_use=in_use-%s where project_id=\'%s\' and resource=\'gigabytes_rbd\';' % (
size, project_id)
sql_update_volumes_rbd = 'UPDATE quota_usages SET in_use=in_use-1 where project_id=\'%s\' and resource=\'volumes_rbd\';' % project_id
db_connect(sql_update_gigabytes_rbd)
db_connect(sql_update_volumes_rbd)
def update():
'''update eayunstack-tools on all nodes'''
node_list = get_node_list('all')
update_cmd = 'yum clean all && yum -y -d 0 update eayunstack-tools'
results = run_cmd_on_nodes(node_list, update_cmd)
get_current_version = \
'rpm --queryformat "%{VERSION} %{RELEASE}" -q eayunstack-tools'
current_version = run_cmd_on_nodes(node_list, get_current_version)
correct_version = run_command(
'rpm --queryformat "%{VERSION} %{RELEASE}" -q eayunstack-tools')
for node in node_list:
out = results[node][0]
err = results[node][1]
current_ver = current_version[node][0].split(' ')[0] + \
'-' + current_version[node][0].split(' ')[1].split('.')[0]
correct_ver = correct_version[0].split(' ')[0] + \
'-' + correct_version[0].split(' ')[1].split('.')[0]
if err or current_ver != correct_ver:
LOG.error('Update on %s failed !' % node)
LOG.error('Current version: %s' % current_ver)
for l in err.split('\n'):
LOG.error(l)
print
else:
LOG.info('Update on %s successfully.' % node)
LOG.info('Current version: %s' % current_ver)
print
def install_packages_on_influxdbnode():
LOG.info('Install rpm packages "%s" on node %s .'
% (INSTALL_PACKAGES, INFLUXDB_HOST))
(out, err) = ssh_connect(INFLUXDB_HOST, 'yum -d 0 -e 0 -y install %s'
% INSTALL_PACKAGES)
if out != '':
log_split_output(out, 'warn')
def install_packages_on_influxdbnode():
LOG.info('Install rpm packages "%s" on node %s .' %
(INSTALL_PACKAGES, INFLUXDB_HOST))
(out, err) = ssh_connect(INFLUXDB_HOST,
'yum -d 0 -e 0 -y install %s' % INSTALL_PACKAGES)
if out != '':
log_split_output(out, 'warn')
def delete_backend_snapshots_rbd(snapshots_id, volume_id):
success = True
rbd_pool = get_backend_pool(volume_id)
LOG.info(' Deleting backend(rbd) snapshots ...')
for snapshot_id in snapshots_id:
LOG.info(' [%s]Deleting backend snapshot ...' % snapshot_id)
(s, o) = commands.getstatusoutput(
'rbd -p %s snap unprotect --image volume-%s --snap snapshot-%s' %
(rbd_pool, volume_id, snapshot_id))
if s == 0:
(ss, oo) = commands.getstatusoutput(
'rbd -p %s snap rm --image volume-%s --snap snapshot-%s' %
(rbd_pool, volume_id, snapshot_id))
if ss != 0:
LOG.error('Can not delete backend snapshot "snapshot-%s" !' %
snapshot_id)
success = False
elif o.find('No such file or directory') > 0:
LOG.error(' This snapshot does not exist !')
success = False
elif o.find('Device or resource busy') > 0:
LOG.error(
' Unprotecting snapshot failed. Device or resource busy !')
success = False
else:
success = False
return success
def detach_volume(attached_servers, volume_id):
LOG.info('Detaching volume "%s" .' % volume_id)
volume_bootable = get_volume_info(volume_id).bootable
if volume_bootable == 'false':
# detach volume from instance by python sdk first
logging.disable(logging.INFO)
for server_id in attached_servers:
pc.nova_delete_server_volume(server_id, volume_id)
logging.disable(logging.NOTSET)
t = 0
while t <= 14:
volume_status = get_volume_info(volume_id).status
if volume_status == 'available':
break
time.sleep(3)
t+=3
# if timeout, detach-disk by virsh on compute node & update database
if get_volume_info(volume_id).status != 'available':
if detach_disk_on_compute_node(attached_servers, volume_id):
# update database
LOG.info(' Updating database.')
# NOTE use UTC time
detach_at = time.strftime('%Y-%m-%d %X', time.gmtime())
sql_update_cinder_db = 'UPDATE volumes SET status="available",attach_status="detached" WHERE id="%s";' % volume_id
cinder_db.connect(sql_update_cinder_db)
for server_id in attached_servers:
sql_update_nova_db = 'UPDATE block_device_mapping SET deleted_at="%s",deleted=id WHERE instance_uuid="%s" and volume_id="%s" and deleted=0;' % (detach_at, server_id, volume_id)
nova_db.connect(sql_update_nova_db)
if get_volume_info(volume_id).status == 'available':
return True
else:
LOG.warn('Can not detach root device. Please delete instance "%s" first.' % attached_servers)
return False
def check_all_nodes(check_obj):
if check_obj is 'all':
if LOG.enable_debug:
check_cmd = 'sudo eayunstack --debug doctor cls --all'
else:
check_cmd = 'sudo eayunstack doctor cls --all'
else:
if LOG.enable_debug:
check_cmd = 'sudo eayunstack --debug doctor cls -n %s' % check_obj
else:
check_cmd = 'sudo eayunstack doctor cls -n %s' % check_obj
# get controller node list
node_list = get_node_list('controller')
# ssh to all controller node to check obj
if len(node_list) == 0:
LOG.warn('Node list is null !')
return
else:
if check_obj == 'ceph':
# only need to check one node for ceph cluster
ceph_node = node_list[0]
run_doctor_cmd_on_node('controller', ceph_node, check_cmd)
else:
nodes = []
for node in node_list:
node_info = {}
node_info['role'] = 'controller'
node_info['name'] = node
nodes.append(node_info)
result = run_doctor_on_nodes(nodes, check_cmd)
for res in result:
LOG.info(res, remote=True)
def check_all_openstack_node_online(nodes_info):
LOG.info('Checking all openstack node is online ...')
for node in nodes_info:
host = str(node['fuelweb_admin'].split('/')[0])
if not check_node_online(host):
return False
return True
def check_all_nodes(check_obj):
if check_obj is 'all':
if LOG.enable_debug:
check_cmd = 'sudo eayunstack --debug doctor cls --all'
else:
check_cmd = 'sudo eayunstack doctor cls --all'
else:
if LOG.enable_debug:
check_cmd = 'sudo eayunstack --debug doctor cls -n %s' % check_obj
else:
check_cmd = 'sudo eayunstack doctor cls -n %s' % check_obj
# get controller node list
node_list = get_node_list('controller')
# ssh to all controller node to check obj
if len(node_list) == 0:
LOG.warn('Node list is null !')
return
else:
if check_obj == 'ceph':
# only need to check one node for ceph cluster
ceph_node = node_list[0]
run_doctor_cmd_on_node('controller', ceph_node, check_cmd)
else:
(proc_list, pipe) = run_doctor_on_nodes('controller', node_list,
check_cmd)
for proc in proc_list:
proc.join()
LOG.info(pipe.recv(), remote=True)
def delete_instance(instance_id, delete_disk=False):
if not pc.nova_server_exist(instance_id):
LOG.error('Instance "%s" is not exist !' % instance_id)
return
instance_status = get_instance_status(instance_id)
if determine_delete_instance(instance_id, instance_status):
LOG.info('Delete instance "%s".' % instance_id)
instance_power_state = get_instance_power_state(instance_id)
if instance_power_state == 'running':
LOG.info('Instance "%s" is running, try to destroy it.' % instance_id)
if destroy_instance(instance_id):
delete_vnic_vbr(instance_id)
delete_instance_dir(instance_id)
undefine_instance(instance_id)
delete_ports(instance_id)
update_disk_state(instance_id)
if delete_disk:
delete_disks(instance_id)
update_nova_db(instance_id)
else:
delete_vnic_vbr(instance_id)
delete_instance_dir(instance_id)
undefine_instance(instance_id)
delete_ports(instance_id)
update_disk_state(instance_id)
if delete_disk:
delete_disks(instance_id)
update_nova_db(instance_id)
def update_nova_db(instance_id):
LOG.info('Update nova database.')
tenant_id = get_tenant_id(instance_id)
flavor_id = get_flavor(instance_id)['id']
ram_usage = get_flavor_resource(flavor_id, 'ram')
vcpu_usage = get_flavor_resource(flavor_id, 'vcpus')
# update instances table vm_state power_state
nova_db.connect('UPDATE instances SET vm_state="deleted",power_state=0,deleted=id WHERE uuid="%s";' % instance_id)
# update quota_usages table instances
nova_db.connect('UPDATE quota_usages SET in_use=in_use-1 WHERE project_id="%s" and resource="instances";' % tenant_id)
# update quota_usages table
nova_db.connect('UPDATE quota_usages SET in_use=in_use-%s WHERE project_id="%s" and resource="ram";' % (ram_usage, tenant_id))
# update quota_usages table
nova_db.connect('UPDATE quota_usages SET in_use=in_use-%s WHERE project_id="%s" and resource="cores";' % (vcpu_usage, tenant_id))
# update instance_faults table
nova_db.connect('UPDATE instance_faults SET deleted=id WHERE instance_uuid="%s";' % instance_id)
# update instance_info_caches table
nova_db.connect('UPDATE instance_info_caches SET deleted=id WHERE instance_uuid="%s";' % instance_id)
# update security_group_instance_association table
nova_db.connect('UPDATE security_group_instance_association SET deleted=id WHERE instance_uuid="%s";' % instance_id)
# update block_device_mapping table
nova_db.connect('UPDATE block_device_mapping SET deleted=id WHERE instance_uuid="%s";' % instance_id)
# update fixed_ips table
nova_db.connect('UPDATE fixed_ips SET deleted=id WHERE instance_uuid="%s";' % instance_id)
# update virtual_interfaces table
nova_db.connect('UPDATE virtual_interfaces SET deleted=id WHERE instance_uuid="%s";' % instance_id)
def check_rabbitmq():
# node role check
if not NODE_ROLE.is_fuel():
if not NODE_ROLE.is_controller():
LOG.warn('This command can only run on fuel or controller node !')
return
if NODE_ROLE.is_fuel():
check_all_nodes('rabbitmq')
return
LOG.info('%s%s Checking rabbitmq cluster status' %('='*5, '>'))
# get all controller node hostname
controllers = get_controllers_hostname()
if controllers is None:
LOG.error('Can not get the controllers node list !')
return
# get masters & slaves node list
running_nodes = get_rabbitmq_nodes()
if running_nodes is None:
LOG.error('Can not get the running node list for rabbitmq cluster !')
return
# check all controller nodes in masters + slaves node list
error_nodes = []
for node in controllers:
if node.split('.')[0] not in running_nodes:
error_nodes.append(node)
if error_nodes:
LOG.error('Node %s not in rabbitmq cluster !' % error_nodes)
LOG.error('Rabbitmq cluster check faild !')
else:
LOG.info('Rabbitmq cluster check successfully !')
def protect_image(uuid):
'''Protect kernel image and initrd image'''
LOG.info('Image protecting...')
(stat, out) = commands.getstatusoutput('source %s && glance image-update --is-protected True %s' % (env_path, uuid))
if stat != 0:
LOG.error('%s' % out)
else:
LOG.info('Protected successfully.\n')
def update_volume_table(volume_id):
LOG.info(' [%s]Updating volumes table ...' % volume_id)
sql_update = 'UPDATE volumes SET deleted=1,status=\'deleted\' WHERE id=\'%s\';' % volume_id
db_connect(sql_update)
sql_select = 'SELECT deleted,status FROM volumes WHERE id =\'%s\';' % volume_id
rest = db_connect(sql_select)
if rest[0] != 1 or rest[1] != 'deleted':
LOG.error(' Database update faild !')
def check_node_profiles(role):
component_list = eval('get_%s_component' % role)()
for c in component_list:
LOG.info('Checking "%s" Component' % c.capitalize())
profile_list = eval('get_%s_profiles' % c)()
for p in profile_list:
LOG.debug('Profile: ' + p)
check_profile(p, role)
def ssh_connect2(hostname, commands):
"""exec ssh command and print the result """
out, err = ssh_connect(hostname, commands)
if out:
LOG.info(out, remote=True)
elif err:
LOG.info(err, remote=True)
return out, err
def push_conf_file_to_openstack_node(nodes_info):
LOG.info('Push conf file to openstack node.')
for node in nodes_info:
host = str(node['fuelweb_admin'].split('/')[0])
push_repo_file_to_node(host, 'lma_collector', LMA_REPO_CONF_FILEPATH)
src_path = CONF_TMP_DIR + 'globals.yaml' + '-' + host
dst_file_name = 'globals.yaml'
push_yaml_to_node(host, src_path, dst_file_name)
def update_volume_table(volume_id):
LOG.info(' [%s]Updating volumes table ...' % volume_id)
sql_update = 'UPDATE volumes SET deleted=1,status=\'deleted\' WHERE id=\'%s\';' % volume_id
db_connect(sql_update)
sql_select = 'SELECT deleted,status FROM volumes WHERE id =\'%s\';' % volume_id
rest = db_connect(sql_select)
if rest[0] != 1 or rest[1] != 'deleted':
LOG.error(' Database update faild !')
def check_node_profiles(role):
component_list = eval('get_%s_component' % role)()
for c in component_list:
LOG.info('Checking "%s" Component' % c.capitalize())
profile_list = eval('get_%s_profiles' % c)()
for p in profile_list:
LOG.debug('Profile: ' + p)
check_profile(p, role)
def push_conf_file_to_influxdbnode():
LOG.info('Push conf file to influxdb node.')
push_repo_file_to_node(INFLUXDB_HOST,
'influxdb_grafana',
INFLUXDB_REPO_CONF_FILEPATH,
backup=True)
push_repo_file_to_node(INFLUXDB_HOST, 'nailgun',
NAILGUN_REPO_CONF_FILEPATH)
push_yaml_to_node(INFLUXDB_HOST, ASTUTE_CONF_FILEPATH, 'astute.yaml')
def delete_instance_dir(instance_id):
compute_node = get_hypervisor_hostname(instance_id)
LOG.info('Delete instance dir on compute node "%s".' % compute_node)
rm_basic_dir_cmd = 'rm -rf /var/lib/nova/instances/%s' % instance_id
rm_resize_dir_cmd = 'rm -rf /var/lib/nova/instances/%s_resize' % instance_id
rm_del_dir_cmd = 'rm -rf /var/lib/nova/instances/%s_del' % instance_id
ssh_cmd(compute_node, rm_basic_dir_cmd)
ssh_cmd(compute_node, rm_resize_dir_cmd)
ssh_cmd(compute_node, rm_del_dir_cmd)
def check_nodes(obj_name):
# node_list = get_node_list('all')
check_cmd = get_check_cmd(obj_name)
for role in ['controller', 'compute', 'mongo', 'ceph-osd']:
node_list = get_node_list(role)
(proc_list, pipe) = run_doctor_on_nodes(role, node_list, check_cmd)
for proc in proc_list:
proc.join()
LOG.info(pipe.recv(), remote=True)
def generate_conf_file(conf_dir, fuel_node_ip):
LOG.info('Generate conf file .')
if not os.path.exists(conf_dir):
os.mkdir(conf_dir)
_generate_astute_conf_file(ASTUTE_CONF_FILEPATH)
_generate_plugin_repo_conf('influxdb_grafana', INFLUXDB_REPO_CONF_FILEPATH, fuel_node_ip)
_generate_nailgun_repo_conf_file(NAILGUN_REPO_CONF_FILEPATH, fuel_node_ip)
_generate_common_conf_file(COMMON_CONF_FILEPATH)
_generate_hiera_conf_file(HIERA_CONF_FILEPATH)
def delete_snapshots(snapshots_id, volume_id):
LOG.info('Deleting snapshot %s ...' % snapshots_id)
if delete_backend_snapshots(snapshots_id, volume_id):
try:
delete_image(snapshots_id)
except Exception,ex:
LOG.error(' Delete image failed!\n %s' % ex)
update_snapshots_db(snapshots_id, volume_id)
return True
def check_nodes(obj_name):
# node_list = get_node_list('all')
check_cmd = get_check_cmd(obj_name)
for role in ['controller','compute','mongo','ceph-osd']:
node_list = get_node_list(role)
(proc_list, pipe) = run_doctor_on_nodes(role, node_list, check_cmd)
for proc in proc_list:
proc.join()
LOG.info(pipe.recv(), remote=True)
def push_conf_file_to_openstack_node(nodes_info):
LOG.info('Push conf file to openstack node.')
for node in nodes_info:
host = str(node['fuelweb_admin'].split('/')[0])
push_repo_file_to_node(host, 'lma_collector',
LMA_REPO_CONF_FILEPATH)
src_path = CONF_TMP_DIR + 'globals.yaml' + '-' + host
dst_file_name = 'globals.yaml'
push_yaml_to_node(host, src_path, dst_file_name)
def delete_snapshots(snapshots_id, volume_id):
LOG.info('Deleting snapshot %s ...' % snapshots_id)
if delete_backend_snapshots(snapshots_id, volume_id):
try:
delete_image(snapshots_id)
except Exception, ex:
LOG.error(' Delete image failed!\n %s' % ex)
update_snapshots_db(snapshots_id, volume_id)
return True
def deployment_openstack_nodes(nodes_info):
LOG.info('Deployment lma_collector on openstack nodes.')
plugin_version = get_plugin_version('lma_collector')
proc_list = []
for node in nodes_info:
proc=Process(target=deployment_openstack_node, args=(node, plugin_version,))
proc.start()
proc_list.append(proc)
for proc in proc_list:
proc.join()
def generate_conf_file(conf_dir, fuel_node_ip):
LOG.info('Generate conf file .')
if not os.path.exists(conf_dir):
os.mkdir(conf_dir)
_generate_astute_conf_file(ASTUTE_CONF_FILEPATH)
_generate_plugin_repo_conf('influxdb_grafana', INFLUXDB_REPO_CONF_FILEPATH,
fuel_node_ip)
_generate_nailgun_repo_conf_file(NAILGUN_REPO_CONF_FILEPATH, fuel_node_ip)
_generate_common_conf_file(COMMON_CONF_FILEPATH)
_generate_hiera_conf_file(HIERA_CONF_FILEPATH)
def protect_image(uuid):
'''Protect kernel image and initrd image'''
LOG.info('Image protecting...')
(stat, out) = commands.getstatusoutput(
'source %s && glance image-update --is-protected True %s' %
(env_path, uuid))
if stat != 0:
LOG.error('%s' % out)
else:
LOG.info('Protected successfully.\n')
def ssh_connect2(hostname, commands, check_all=False):
"""exec ssh command and print the result """
out, err = ssh_connect(hostname, commands)
if out and not check_all:
LOG.info(out, remote=True)
elif err and not check_all:
LOG.info(err, remote=True)
if check_all:
logging.disable(logging.INFO)
return out, err
def generate_lma_conf_file(conf_dir, nodes_info, fuel_node_ip):
LOG.info('Generate openstack node conf file.')
if not os.path.exists(conf_dir):
os.mkdir(conf_dir)
for node in nodes_info:
br_fw_mgmt = node['fuelweb_admin'].split('/')[0]
br_mgmt = str(node['management'].split('/')[0])
file_path = GLOBALS_CONF_FILEPATH + '-' + br_fw_mgmt
_generate_globals_conf_file(file_path, br_mgmt)
_generate_plugin_repo_conf('lma_collector', LMA_REPO_CONF_FILEPATH, fuel_node_ip)
def delete_backend_snapshots_eqlx(snapshots_id, volume_id):
LOG.info(' Deleting backend(eqlx) snapshots ...')
for snapshot_id in snapshots_id:
LOG.info(' [%s]Deleting backend snapshot ...' % snapshot_id)
cmd_delete_snapshot = 'volume select volume-%s snapshot delete snapshot-%s' % (volume_id, snapshot_id)
result = eqlx_ssh_execute(cmd_delete_snapshot)
if 'Snapshot deletion succeeded.' not in result:
LOG.error(' Can not delete snapshot "%s" !' % snapshot_id)
return False
else:
return True
def generate_lma_conf_file(conf_dir, nodes_info, fuel_node_ip):
LOG.info('Generate openstack node conf file.')
if not os.path.exists(conf_dir):
os.mkdir(conf_dir)
for node in nodes_info:
br_fw_mgmt = node['fuelweb_admin'].split('/')[0]
br_mgmt = str(node['management'].split('/')[0])
file_path = GLOBALS_CONF_FILEPATH + '-' + br_fw_mgmt
_generate_globals_conf_file(file_path, br_mgmt)
_generate_plugin_repo_conf('lma_collector', LMA_REPO_CONF_FILEPATH,
fuel_node_ip)
def check_services_list():
LOG.info('Checking nova & cinder service list ' 'and neutron agent list')
logging.disable(logging.INFO)
pc = PythonClient()
nova_services_list = pc.nova_services_list()
check_services(nova_services_list)
cinder_services_list = pc.cinder_services_list()
check_services(cinder_services_list)
neutron_agents_list = pc.neutron_agents_list()
logging.disable(logging.NOTSET)
check_neutron_agents(neutron_agents_list)
def delete_backend_snapshots_eqlx(snapshots_id, volume_id):
LOG.info(' Deleting backend(eqlx) snapshots ...')
for snapshot_id in snapshots_id:
LOG.info(' [%s]Deleting backend snapshot ...' % snapshot_id)
cmd_delete_snapshot = 'volume select volume-%s snapshot delete snapshot-%s' % (volume_id, snapshot_id)
result = eqlx_ssh_execute(cmd_delete_snapshot)
if 'Snapshot deletion succeeded.' not in result:
LOG.error(' Can not delete snapshot "%s" !' % snapshot_id)
return False
else:
return True
def create_symbolic_links_on_openstack_node(nodes_info):
LOG.info('Create symbolic links on openstack node.')
for node in nodes_info:
host = str(node['fuelweb_admin'].split('/')[0])
src_file = '/etc/astute.yaml'
dst_file = '/etc/hiera/astute.yaml'
cmd = 'test -h ' + dst_file + ' || ln -s ' + src_file + ' ' + dst_file
(out, err) = ssh_connect(host, cmd)
if err != '':
LOG.error('Can not run command: %s on node %s .' % (cmd, host))
else:
LOG.debug('Create symbolic links on node %s .' % host)
def restore_from_id(backup_id):
try:
db = BackupDB()
backup_path = db.read(int(backup_id))
(stat,
out) = commands.getstatusoutput('dockerctl restore %s' % backup_path)
if stat != 0:
LOG.error('%s' % out)
else:
LOG.info('Restore successfully completed!\n')
except:
LOG.error('The ID does not exist, Please try again.\n')
def delete_image(snapshots_id):
logging.disable(logging.DEBUG)
for snapshot_id in snapshots_id:
tenant_id = pc.cinder_get_tenant_id(snapshot_id)
images = pc.glance_get_images(tenant_id)
for image in images:
image_id = image.get('id')
image_block_device_mapping = image.get('block_device_mapping')
if image_block_device_mapping and snapshot_id in image_block_device_mapping:
LOG.info(' Delete image "%s".' % image_id)
pc.glance_delete_image(image_id)
logging.disable(logging.NOTSET)
def run_doctor_on_nodes(node_list, check_cmd):
pile = eventlet.GreenPile()
result = []
for node in node_list:
LOG.info('%s%s Push check cmd to %-13s (%-10s) %s%s' %
('<', '=' * 2, node['name'], node['role'], '=' * 2, '>'))
pile.spawn(run_doctor_cmd_on_node, node['role'], node['name'],
check_cmd)
for node, res in zip(node_list, pile):
result.append(res)
logging.disable(logging.NOTSET)
return result
