def check_rabbitmq(): # node role check if not NODE_ROLE.is_fuel(): if not NODE_ROLE.is_controller(): LOG.warn('This command can only run on fuel or controller node !') return if NODE_ROLE.is_fuel(): check_all_nodes('rabbitmq') return LOG.info('%s%s Checking rabbitmq cluster status' % ('=' * 5, '>')) # get all controller node hostname controllers = get_controllers_hostname() if controllers is None: LOG.error('Can not get the controllers node list !') return # get masters & slaves node list running_nodes = get_rabbitmq_nodes() if running_nodes is None: LOG.error('Can not get the running node list for rabbitmq cluster !') return # check all controller nodes in masters + slaves node list error_nodes = [] for node in controllers: if node.split('.')[0] not in running_nodes: error_nodes.append(node) if error_nodes: LOG.error('Node %s not in rabbitmq cluster !' % error_nodes) LOG.error('Rabbitmq cluster check faild !') else: LOG.info('Rabbitmq cluster check successfully !')
def check_all_nodes(check_obj): if check_obj is "all": if LOG.enable_debug: check_cmd = "sudo eayunstack --debug doctor cls --all" else: check_cmd = "sudo eayunstack doctor cls --all" else: if LOG.enable_debug: check_cmd = "sudo eayunstack --debug doctor cls -n %s" % check_obj else: check_cmd = "sudo eayunstack doctor cls -n %s" % check_obj # get controller node list node_list = get_node_list("controller") # ssh to all controller node to check obj if len(node_list) == 0: LOG.warn("Node list is null !") return else: if check_obj == "ceph": # only need to check one node for ceph cluster ceph_node = node_list[0] LOG.info("%s Role: %-10s Node: %-13s %s" % ("*" * 15, "controller", ceph_node, "*" * 15)) ssh_connect2(ceph_node, check_cmd) else: for node in node_list: LOG.info("%s Role: %-10s Node: %-13s %s" % ("*" * 15, "controller", node, "*" * 15)) ssh_connect2(node, check_cmd)
def base_delete(self, resource_name, resource_ids, delete_func): no_log_resources = [] while resource_ids: for resource_id in resource_ids: # avoid LOG delete info many times if resource_id not in no_log_resources: with log_disabled(): LOG.info('Delete %s [%s]' % (resource_name, resource_id)) no_log_resources.append(resource_id) try: delete_func(resource_id) # delete successfully, break resource_ids.remove(resource_id) break except Conflict: # retry: deal with conflict. continue except NotFound: # when call destroy_volume(), # will delete volumes and snapshots, # if snapshots NotFound, do nothing. resource_ids.remove(resource_id) break except Exception as e: LOG.warn('Can not delete %s [%s]' % (resource_name, resource_id)) LOG.error(e) # something else wrong, break, won't retry resource_ids.remove(resource_id) break
def _run(self): self.base_delete('floating ip', self.floatingips, neutronclient.delete_floatingip) for port_id in self.ports: try: with log_disabled(): LOG.info('Delete port [%s]' % port_id) neutronclient.delete_port(port_id) except Conflict as e: with log_disabled(): LOG.info(' Solving conflict: remove interface...') router_id = neutronclient.show_port( port_id)['port']['device_id'] neutronclient.remove_interface_router(router_id, {'port_id': port_id}) except Exception as e: LOG.warn('Can not delete port [%s]' % port_id) LOG.error(e) # if firewall create with target router, # CAN NOT delete router before firewall is deleted. # NOTE: already add retry self.base_delete('router', self.routers, neutronclient.delete_router) self.base_delete('subnet', self.subnets, neutronclient.delete_subnet) self.base_delete('network', self.networks, neutronclient.delete_network)
def check_mysql(): # node role check if not NODE_ROLE.is_fuel(): if not NODE_ROLE.is_controller(): LOG.warn('This command can only run on fuel or controller node !') return if NODE_ROLE.is_fuel(): check_all_nodes('mysql') return LOG.info('%s%s Checking mysql cluster status' % ('=' * 5, '>')) # get running node list for mysql cluster running_nodes = get_mysql_nodes() if running_nodes is None: LOG.error('Can not get the running node list for mysql cluster !') return # get all controller node hostname controllers = get_controllers_hostname() if controllers is None: LOG.error('Can not get the controllers node list !') return # check all controller node in mysql cluster error_nodes = [] for node in controllers: if node not in running_nodes: error_nodes.append(node) if error_nodes: LOG.error('Node %s is not running in mysql cluster !' % error_nodes) LOG.error('Mysql cluster check faild !') else: LOG.info('Mysql cluster check successfully !')
def push_conf_file_to_influxdbnode(): LOG.info('Push conf file to influxdb node.') push_repo_file_to_node(INFLUXDB_HOST, 'influxdb_grafana', INFLUXDB_REPO_CONF_FILEPATH, backup=True) push_repo_file_to_node(INFLUXDB_HOST, 'nailgun', NAILGUN_REPO_CONF_FILEPATH) push_yaml_to_node(INFLUXDB_HOST, ASTUTE_CONF_FILEPATH, 'astute.yaml')
def update(): '''update eayunstack-tools on all nodes''' node_list = get_node_list('all') update_cmd = 'yum clean all && yum -y -d 0 update eayunstack-tools' results = run_cmd_on_nodes(node_list, update_cmd) get_current_version = \ 'rpm --queryformat "%{VERSION} %{RELEASE}" -q eayunstack-tools' current_version = run_cmd_on_nodes(node_list, get_current_version) correct_version = run_command( 'rpm --queryformat "%{VERSION} %{RELEASE}" -q eayunstack-tools') for node in node_list: out = results[node][0] err = results[node][1] current_ver = current_version[node][0].split(' ')[0] + \ '-' + current_version[node][0].split(' ')[1].split('.')[0] correct_ver = correct_version[0].split(' ')[0] + \ '-' + correct_version[0].split(' ')[1].split('.')[0] if err or current_ver != correct_ver: LOG.error('Update on %s failed !' % node) LOG.error('Current version: %s' % current_ver) for l in err.split('\n'): LOG.error(l) print else: LOG.info('Update on %s successfully.' % node) LOG.info('Current version: %s' % current_ver) print
def delete_backend_snapshots_rbd(snapshots_id, volume_id): success = True rbd_pool = get_backend_pool(volume_id) LOG.info(' Deleting backend(rbd) snapshots ...') for snapshot_id in snapshots_id: LOG.info(' [%s]Deleting backend snapshot ...' % snapshot_id) (s, o) = commands.getstatusoutput( 'rbd -p %s snap unprotect --image volume-%s --snap snapshot-%s' % (rbd_pool, volume_id, snapshot_id)) if s == 0: (ss, oo) = commands.getstatusoutput( 'rbd -p %s snap rm --image volume-%s --snap snapshot-%s' % (rbd_pool, volume_id, snapshot_id)) if ss != 0: LOG.error('Can not delete backend snapshot "snapshot-%s" !' % snapshot_id) success = False elif o.find('No such file or directory') > 0: LOG.error(' This snapshot does not exist !') success = False elif o.find('Device or resource busy') > 0: LOG.error(' Unprotecting snapshot failed. Device or resource busy !') success = False else: success = False return success
def check_all_nodes(check_obj): if check_obj is 'all': if LOG.enable_debug: check_cmd = 'sudo eayunstack --debug doctor cls --all' else: check_cmd = 'sudo eayunstack doctor cls --all' else: if LOG.enable_debug: check_cmd = 'sudo eayunstack --debug doctor cls -n %s' % check_obj else: check_cmd = 'sudo eayunstack doctor cls -n %s' % check_obj # get controller node list node_list = get_node_list('controller') # ssh to all controller node to check obj if len(node_list) == 0: LOG.warn('Node list is null !') return else: if check_obj == 'ceph': # only need to check one node for ceph cluster ceph_node = node_list[0] run_doctor_cmd_on_node('controller', ceph_node, check_cmd) else: (proc_list, pipe) = run_doctor_on_nodes('controller', node_list, check_cmd) for proc in proc_list: proc.join() LOG.info(pipe.recv(), remote=True)
def delete_backend_volume_eqlx(volume_id): # get provider_location sql_get_provider_location = 'SELECT provider_location FROM volumes WHERE id =\'%s\';' % volume_id provider_location = str(db_connect(sql_get_provider_location)).split()[1] # ssh to controller and compute node to delete the iscsi connection LOG.info(' Deleting iscsi connection ...') controller_list = get_node_list('controller') compute_list = get_node_list('compute') node_list = controller_list + compute_list for node in node_list: cmd_show = 'iscsiadm -m session -o show' (out, err) = ssh_connect(node, cmd_show) if out.find(provider_location) > 0: cmd_delete = 'iscsiadm -m node -u -T %s' % provider_location (o, e) = ssh_connect(node, cmd_delete) if o.find('successful') < 0: LOG.error( ' Can not delete the iscsi connection "%s" at host %s.' % (provider_location, node)) # ssh to eqlx to delete the volume LOG.info(' Deleting backend(eqlx) volume ...') ## set eqlx volume status to offline cmd_set_eqlx_volume_offline = 'volume select volume-%s offline' % volume_id out = eqlx_ssh_execute(cmd_set_eqlx_volume_offline) if len(out) == 3: LOG.error(' ' + out[1]) return False ## delete the eqlx volume cmd_delete_eqlx_volume = 'volume delete volume-%s' % volume_id result = eqlx_ssh_execute(cmd_delete_eqlx_volume) if not result or result[1] != 'Volume deletion succeeded.': LOG.error(' Delete backend volume faild !') return False else: return True
def delete_snapshots(snapshots_id): LOG.info('Deleting snapshot %s ...' % snapshots_id) if delete_backend_snapshots(snapshots_id): update_snapshots_db(snapshots_id) return True else: return False
def check_mysql(): # node role check if not NODE_ROLE.is_fuel(): if not NODE_ROLE.is_controller(): LOG.warn('This command can only run on fuel or controller node !') return if NODE_ROLE.is_fuel(): check_all_nodes('mysql') return LOG.info('%s%s Checking mysql cluster status' %('='*5, '>')) # get running node list for mysql cluster running_nodes = get_mysql_nodes() if running_nodes is None: LOG.error('Can not get the running node list for mysql cluster !') return # get all controller node hostname controllers = get_controllers_hostname() if controllers is None: LOG.error('Can not get the controllers node list !') return # check all controller node in mysql cluster error_nodes = [] for node in controllers: if node not in running_nodes: error_nodes.append(node) if error_nodes: LOG.error('Node %s is not running in mysql cluster !' % error_nodes) LOG.error('Mysql cluster check faild !') else: LOG.info('Mysql cluster check successfully !')
def delete_volume(volume_id, volume_status): LOG.info('Deleting volume %s ...' % volume_id) if volume_status == 'creating': update_db(volume_id) else: if delete_backend_volume(volume_id): update_db(volume_id)
def check_all_nodes(check_obj): if check_obj is 'all': if LOG.enable_debug: check_cmd = 'sudo eayunstack --debug doctor cls --all' else: check_cmd = 'sudo eayunstack doctor cls --all' else: if LOG.enable_debug: check_cmd = 'sudo eayunstack --debug doctor cls -n %s' % check_obj else: check_cmd = 'sudo eayunstack doctor cls -n %s' % check_obj # get controller node list node_list = get_node_list('controller') # ssh to all controller node to check obj if len(node_list) == 0: LOG.warn('Node list is null !') return else: if check_obj == 'ceph': # only need to check one node for ceph cluster ceph_node = node_list[0] LOG.info('%s Role: %-10s Node: %-13s %s' % ('*'*15, 'controller', ceph_node, '*'*15)) ssh_connect2(ceph_node, check_cmd) else: for node in node_list: LOG.info('%s Role: %-10s Node: %-13s %s' % ('*'*15, 'controller', node, '*'*15)) ssh_connect2(node, check_cmd)
def check_all_nodes(check_obj): if check_obj is 'all': if LOG.enable_debug: check_cmd = 'sudo eayunstack --debug doctor cls --all' else: check_cmd = 'sudo eayunstack doctor cls --all' else: if LOG.enable_debug: check_cmd = 'sudo eayunstack --debug doctor cls -n %s' % check_obj else: check_cmd = 'sudo eayunstack doctor cls -n %s' % check_obj # get controller node list node_list = get_node_list('controller') # ssh to all controller node to check obj if len(node_list) == 0: LOG.warn('Node list is null !') return else: if check_obj == 'ceph': # only need to check one node for ceph cluster ceph_node = node_list[0] res = run_doctor_cmd_on_node('controller', ceph_node, check_cmd) LOG.info(res, remote=True) else: nodes = [] for node in node_list: node_info = {} node_info['role'] = 'controller' node_info['name'] = node nodes.append(node_info) result = run_doctor_on_nodes(nodes, check_cmd) for res in result: LOG.info(res, remote=True)
def check_all_openstack_node_online(nodes_info): LOG.info('Checking all openstack node is online ...') for node in nodes_info: host = str(node['fuelweb_admin'].split('/')[0]) if not check_node_online(host): return False return True
def _check_managed_status(resource): if resource['@managed'] == 'true': LOG.info('Resource %s was managed on node %s' \ % (resource['@id'], resource['node']['@name'])) else: LOG.error('Resource %s was unmanaged on node %s' \ % (resource['@id'], resource['node']['@name']))
def delete_backend_volume_eqlx(volume_id): # get provider_location sql_get_provider_location = 'SELECT provider_location FROM volumes WHERE id =\'%s\';' % volume_id provider_location = str(db_connect(sql_get_provider_location)).split()[1] # ssh to controller and compute node to delete the iscsi connection LOG.info(' Deleting iscsi connection ...') controller_list = get_node_list('controller') compute_list = get_node_list('compute') node_list = controller_list + compute_list for node in node_list: cmd_show = 'iscsiadm -m session -o show' (out, err) = ssh_connect(node, cmd_show) if out.find(provider_location) > 0: cmd_delete = 'iscsiadm -m node -u -T %s' % provider_location (o, e) = ssh_connect(node, cmd_delete) if o.find('successful') < 0: LOG.error(' Can not delete the iscsi connection "%s" at host %s.' % (provider_location, node)) # ssh to eqlx to delete the volume LOG.info(' Deleting backend(eqlx) volume ...') ## set eqlx volume status to offline cmd_set_eqlx_volume_offline = 'volume select volume-%s offline' % volume_id out = eqlx_ssh_execute(cmd_set_eqlx_volume_offline) if len(out) == 3: LOG.error(' ' + out[1]) return False ## delete the eqlx volume cmd_delete_eqlx_volume = 'volume delete volume-%s' % volume_id result = eqlx_ssh_execute(cmd_delete_eqlx_volume) if not result or result[1] != 'Volume deletion succeeded.': LOG.error(' Delete backend volume faild !') return False else: return True
def update_volume_quota(volume_id): LOG.info(' [%s]Updating volume quota ...' % volume_id) # get volume size & project id sql_get_size_project_id = 'SELECT size,project_id FROM volumes WHERE id=\'%s\';' % volume_id get_size_project_id = db_connect(sql_get_size_project_id) size = get_size_project_id[0] project_id = get_size_project_id[1] # get backend type backend_type = get_backend_type(volume_id) sql_update_gigabytes = 'UPDATE quota_usages SET in_use=in_use-%s where project_id=\'%s\' and resource=\'gigabytes\';' % ( size, project_id) sql_update_volumes = 'UPDATE quota_usages SET in_use=in_use-1 where project_id=\'%s\' and resource=\'volumes\';' % project_id db_connect(sql_update_gigabytes) db_connect(sql_update_volumes) if backend_type == 'eqlx': sql_update_gigabytes_eqlx = 'UPDATE quota_usages SET in_use=in_use-%s where project_id=\'%s\' and resource=\'gigabytes_eqlx\';' % ( size, project_id) sql_update_volumes_eqlx = 'UPDATE quota_usages SET in_use=in_use-1 where project_id=\'%s\' and resource=\'volumes_eqlx\';' % project_id db_connect(sql_update_gigabytes_eqlx) db_connect(sql_update_volumes_eqlx) elif backend_type == 'rbd': sql_update_gigabytes_rbd = 'UPDATE quota_usages SET in_use=in_use-%s where project_id=\'%s\' and resource=\'gigabytes_rbd\';' % ( size, project_id) sql_update_volumes_rbd = 'UPDATE quota_usages SET in_use=in_use-1 where project_id=\'%s\' and resource=\'volumes_rbd\';' % project_id db_connect(sql_update_gigabytes_rbd) db_connect(sql_update_volumes_rbd)
def install_packages_on_influxdbnode(): LOG.info('Install rpm packages "%s" on node %s .' % (INSTALL_PACKAGES, INFLUXDB_HOST)) (out, err) = ssh_connect(INFLUXDB_HOST, 'yum -d 0 -e 0 -y install %s' % INSTALL_PACKAGES) if out != '': log_split_output(out, 'warn')
def delete_backend_snapshots_rbd(snapshots_id, volume_id): success = True rbd_pool = get_backend_pool(volume_id) LOG.info(' Deleting backend(rbd) snapshots ...') for snapshot_id in snapshots_id: LOG.info(' [%s]Deleting backend snapshot ...' % snapshot_id) (s, o) = commands.getstatusoutput( 'rbd -p %s snap unprotect --image volume-%s --snap snapshot-%s' % (rbd_pool, volume_id, snapshot_id)) if s == 0: (ss, oo) = commands.getstatusoutput( 'rbd -p %s snap rm --image volume-%s --snap snapshot-%s' % (rbd_pool, volume_id, snapshot_id)) if ss != 0: LOG.error('Can not delete backend snapshot "snapshot-%s" !' % snapshot_id) success = False elif o.find('No such file or directory') > 0: LOG.error(' This snapshot does not exist !') success = False elif o.find('Device or resource busy') > 0: LOG.error( ' Unprotecting snapshot failed. Device or resource busy !') success = False else: success = False return success
def detach_volume(attached_servers, volume_id): LOG.info('Detaching volume "%s" .' % volume_id) volume_bootable = get_volume_info(volume_id).bootable if volume_bootable == 'false': # detach volume from instance by python sdk first logging.disable(logging.INFO) for server_id in attached_servers: pc.nova_delete_server_volume(server_id, volume_id) logging.disable(logging.NOTSET) t = 0 while t <= 14: volume_status = get_volume_info(volume_id).status if volume_status == 'available': break time.sleep(3) t+=3 # if timeout, detach-disk by virsh on compute node & update database if get_volume_info(volume_id).status != 'available': if detach_disk_on_compute_node(attached_servers, volume_id): # update database LOG.info(' Updating database.') # NOTE use UTC time detach_at = time.strftime('%Y-%m-%d %X', time.gmtime()) sql_update_cinder_db = 'UPDATE volumes SET status="available",attach_status="detached" WHERE id="%s";' % volume_id cinder_db.connect(sql_update_cinder_db) for server_id in attached_servers: sql_update_nova_db = 'UPDATE block_device_mapping SET deleted_at="%s",deleted=id WHERE instance_uuid="%s" and volume_id="%s" and deleted=0;' % (detach_at, server_id, volume_id) nova_db.connect(sql_update_nova_db) if get_volume_info(volume_id).status == 'available': return True else: LOG.warn('Can not detach root device. Please delete instance "%s" first.' % attached_servers) return False
def check_all_nodes(check_obj): if check_obj is 'all': if LOG.enable_debug: check_cmd = 'sudo eayunstack --debug doctor cls --all' else: check_cmd = 'sudo eayunstack doctor cls --all' else: if LOG.enable_debug: check_cmd = 'sudo eayunstack --debug doctor cls -n %s' % check_obj else: check_cmd = 'sudo eayunstack doctor cls -n %s' % check_obj # get controller node list node_list = get_node_list('controller') # ssh to all controller node to check obj if len(node_list) == 0: LOG.warn('Node list is null !') return else: if check_obj == 'ceph': # only need to check one node for ceph cluster ceph_node = node_list[0] run_doctor_cmd_on_node('controller', ceph_node, check_cmd) else: nodes = [] for node in node_list: node_info = {} node_info['role'] = 'controller' node_info['name'] = node nodes.append(node_info) result = run_doctor_on_nodes(nodes, check_cmd) for res in result: LOG.info(res, remote=True)
def delete_instance(instance_id, delete_disk=False): if not pc.nova_server_exist(instance_id): LOG.error('Instance "%s" is not exist !' % instance_id) return instance_status = get_instance_status(instance_id) if determine_delete_instance(instance_id, instance_status): LOG.info('Delete instance "%s".' % instance_id) instance_power_state = get_instance_power_state(instance_id) if instance_power_state == 'running': LOG.info('Instance "%s" is running, try to destroy it.' % instance_id) if destroy_instance(instance_id): delete_vnic_vbr(instance_id) delete_instance_dir(instance_id) undefine_instance(instance_id) delete_ports(instance_id) update_disk_state(instance_id) if delete_disk: delete_disks(instance_id) update_nova_db(instance_id) else: delete_vnic_vbr(instance_id) delete_instance_dir(instance_id) undefine_instance(instance_id) delete_ports(instance_id) update_disk_state(instance_id) if delete_disk: delete_disks(instance_id) update_nova_db(instance_id)
def update_nova_db(instance_id): LOG.info('Update nova database.') tenant_id = get_tenant_id(instance_id) flavor_id = get_flavor(instance_id)['id'] ram_usage = get_flavor_resource(flavor_id, 'ram') vcpu_usage = get_flavor_resource(flavor_id, 'vcpus') # update instances table vm_state power_state nova_db.connect('UPDATE instances SET vm_state="deleted",power_state=0,deleted=id WHERE uuid="%s";' % instance_id) # update quota_usages table instances nova_db.connect('UPDATE quota_usages SET in_use=in_use-1 WHERE project_id="%s" and resource="instances";' % tenant_id) # update quota_usages table nova_db.connect('UPDATE quota_usages SET in_use=in_use-%s WHERE project_id="%s" and resource="ram";' % (ram_usage, tenant_id)) # update quota_usages table nova_db.connect('UPDATE quota_usages SET in_use=in_use-%s WHERE project_id="%s" and resource="cores";' % (vcpu_usage, tenant_id)) # update instance_faults table nova_db.connect('UPDATE instance_faults SET deleted=id WHERE instance_uuid="%s";' % instance_id) # update instance_info_caches table nova_db.connect('UPDATE instance_info_caches SET deleted=id WHERE instance_uuid="%s";' % instance_id) # update security_group_instance_association table nova_db.connect('UPDATE security_group_instance_association SET deleted=id WHERE instance_uuid="%s";' % instance_id) # update block_device_mapping table nova_db.connect('UPDATE block_device_mapping SET deleted=id WHERE instance_uuid="%s";' % instance_id) # update fixed_ips table nova_db.connect('UPDATE fixed_ips SET deleted=id WHERE instance_uuid="%s";' % instance_id) # update virtual_interfaces table nova_db.connect('UPDATE virtual_interfaces SET deleted=id WHERE instance_uuid="%s";' % instance_id)
def check_rabbitmq(): # node role check if not NODE_ROLE.is_fuel(): if not NODE_ROLE.is_controller(): LOG.warn('This command can only run on fuel or controller node !') return if NODE_ROLE.is_fuel(): check_all_nodes('rabbitmq') return LOG.info('%s%s Checking rabbitmq cluster status' %('='*5, '>')) # get all controller node hostname controllers = get_controllers_hostname() if controllers is None: LOG.error('Can not get the controllers node list !') return # get masters & slaves node list running_nodes = get_rabbitmq_nodes() if running_nodes is None: LOG.error('Can not get the running node list for rabbitmq cluster !') return # check all controller nodes in masters + slaves node list error_nodes = [] for node in controllers: if node.split('.')[0] not in running_nodes: error_nodes.append(node) if error_nodes: LOG.error('Node %s not in rabbitmq cluster !' % error_nodes) LOG.error('Rabbitmq cluster check faild !') else: LOG.info('Rabbitmq cluster check successfully !')
def protect_image(uuid): '''Protect kernel image and initrd image''' LOG.info('Image protecting...') (stat, out) = commands.getstatusoutput('source %s && glance image-update --is-protected True %s' % (env_path, uuid)) if stat != 0: LOG.error('%s' % out) else: LOG.info('Protected successfully.\n')
def update_volume_table(volume_id): LOG.info(' [%s]Updating volumes table ...' % volume_id) sql_update = 'UPDATE volumes SET deleted=1,status=\'deleted\' WHERE id=\'%s\';' % volume_id db_connect(sql_update) sql_select = 'SELECT deleted,status FROM volumes WHERE id =\'%s\';' % volume_id rest = db_connect(sql_select) if rest[0] != 1 or rest[1] != 'deleted': LOG.error(' Database update faild !')
def check_node_profiles(role): component_list = eval('get_%s_component' % role)() for c in component_list: LOG.info('Checking "%s" Component' % c.capitalize()) profile_list = eval('get_%s_profiles' % c)() for p in profile_list: LOG.debug('Profile: ' + p) check_profile(p, role)
def ssh_connect2(hostname, commands): """exec ssh command and print the result """ out, err = ssh_connect(hostname, commands) if out: LOG.info(out, remote=True) elif err: LOG.info(err, remote=True) return out, err
def push_conf_file_to_openstack_node(nodes_info): LOG.info('Push conf file to openstack node.') for node in nodes_info: host = str(node['fuelweb_admin'].split('/')[0]) push_repo_file_to_node(host, 'lma_collector', LMA_REPO_CONF_FILEPATH) src_path = CONF_TMP_DIR + 'globals.yaml' + '-' + host dst_file_name = 'globals.yaml' push_yaml_to_node(host, src_path, dst_file_name)
def delete_instance_dir(instance_id): compute_node = get_hypervisor_hostname(instance_id) LOG.info('Delete instance dir on compute node "%s".' % compute_node) rm_basic_dir_cmd = 'rm -rf /var/lib/nova/instances/%s' % instance_id rm_resize_dir_cmd = 'rm -rf /var/lib/nova/instances/%s_resize' % instance_id rm_del_dir_cmd = 'rm -rf /var/lib/nova/instances/%s_del' % instance_id ssh_cmd(compute_node, rm_basic_dir_cmd) ssh_cmd(compute_node, rm_resize_dir_cmd) ssh_cmd(compute_node, rm_del_dir_cmd)
def check_nodes(obj_name): # node_list = get_node_list('all') check_cmd = get_check_cmd(obj_name) for role in ['controller', 'compute', 'mongo', 'ceph-osd']: node_list = get_node_list(role) (proc_list, pipe) = run_doctor_on_nodes(role, node_list, check_cmd) for proc in proc_list: proc.join() LOG.info(pipe.recv(), remote=True)
def generate_conf_file(conf_dir, fuel_node_ip): LOG.info('Generate conf file .') if not os.path.exists(conf_dir): os.mkdir(conf_dir) _generate_astute_conf_file(ASTUTE_CONF_FILEPATH) _generate_plugin_repo_conf('influxdb_grafana', INFLUXDB_REPO_CONF_FILEPATH, fuel_node_ip) _generate_nailgun_repo_conf_file(NAILGUN_REPO_CONF_FILEPATH, fuel_node_ip) _generate_common_conf_file(COMMON_CONF_FILEPATH) _generate_hiera_conf_file(HIERA_CONF_FILEPATH)
def delete_snapshots(snapshots_id, volume_id): LOG.info('Deleting snapshot %s ...' % snapshots_id) if delete_backend_snapshots(snapshots_id, volume_id): try: delete_image(snapshots_id) except Exception,ex: LOG.error(' Delete image failed!\n %s' % ex) update_snapshots_db(snapshots_id, volume_id) return True
def check_nodes(obj_name): # node_list = get_node_list('all') check_cmd = get_check_cmd(obj_name) for role in ['controller','compute','mongo','ceph-osd']: node_list = get_node_list(role) (proc_list, pipe) = run_doctor_on_nodes(role, node_list, check_cmd) for proc in proc_list: proc.join() LOG.info(pipe.recv(), remote=True)
def delete_snapshots(snapshots_id, volume_id): LOG.info('Deleting snapshot %s ...' % snapshots_id) if delete_backend_snapshots(snapshots_id, volume_id): try: delete_image(snapshots_id) except Exception, ex: LOG.error(' Delete image failed!\n %s' % ex) update_snapshots_db(snapshots_id, volume_id) return True
def deployment_openstack_nodes(nodes_info): LOG.info('Deployment lma_collector on openstack nodes.') plugin_version = get_plugin_version('lma_collector') proc_list = [] for node in nodes_info: proc=Process(target=deployment_openstack_node, args=(node, plugin_version,)) proc.start() proc_list.append(proc) for proc in proc_list: proc.join()
def protect_image(uuid): '''Protect kernel image and initrd image''' LOG.info('Image protecting...') (stat, out) = commands.getstatusoutput( 'source %s && glance image-update --is-protected True %s' % (env_path, uuid)) if stat != 0: LOG.error('%s' % out) else: LOG.info('Protected successfully.\n')
def ssh_connect2(hostname, commands, check_all=False): """exec ssh command and print the result """ out, err = ssh_connect(hostname, commands) if out and not check_all: LOG.info(out, remote=True) elif err and not check_all: LOG.info(err, remote=True) if check_all: logging.disable(logging.INFO) return out, err
def generate_lma_conf_file(conf_dir, nodes_info, fuel_node_ip): LOG.info('Generate openstack node conf file.') if not os.path.exists(conf_dir): os.mkdir(conf_dir) for node in nodes_info: br_fw_mgmt = node['fuelweb_admin'].split('/')[0] br_mgmt = str(node['management'].split('/')[0]) file_path = GLOBALS_CONF_FILEPATH + '-' + br_fw_mgmt _generate_globals_conf_file(file_path, br_mgmt) _generate_plugin_repo_conf('lma_collector', LMA_REPO_CONF_FILEPATH, fuel_node_ip)
def delete_backend_snapshots_eqlx(snapshots_id, volume_id): LOG.info(' Deleting backend(eqlx) snapshots ...') for snapshot_id in snapshots_id: LOG.info(' [%s]Deleting backend snapshot ...' % snapshot_id) cmd_delete_snapshot = 'volume select volume-%s snapshot delete snapshot-%s' % (volume_id, snapshot_id) result = eqlx_ssh_execute(cmd_delete_snapshot) if 'Snapshot deletion succeeded.' not in result: LOG.error(' Can not delete snapshot "%s" !' % snapshot_id) return False else: return True
def check_services_list(): LOG.info('Checking nova & cinder service list ' 'and neutron agent list') logging.disable(logging.INFO) pc = PythonClient() nova_services_list = pc.nova_services_list() check_services(nova_services_list) cinder_services_list = pc.cinder_services_list() check_services(cinder_services_list) neutron_agents_list = pc.neutron_agents_list() logging.disable(logging.NOTSET) check_neutron_agents(neutron_agents_list)
def create_symbolic_links_on_openstack_node(nodes_info): LOG.info('Create symbolic links on openstack node.') for node in nodes_info: host = str(node['fuelweb_admin'].split('/')[0]) src_file = '/etc/astute.yaml' dst_file = '/etc/hiera/astute.yaml' cmd = 'test -h ' + dst_file + ' || ln -s ' + src_file + ' ' + dst_file (out, err) = ssh_connect(host, cmd) if err != '': LOG.error('Can not run command: %s on node %s .' % (cmd, host)) else: LOG.debug('Create symbolic links on node %s .' % host)
def restore_from_id(backup_id): try: db = BackupDB() backup_path = db.read(int(backup_id)) (stat, out) = commands.getstatusoutput('dockerctl restore %s' % backup_path) if stat != 0: LOG.error('%s' % out) else: LOG.info('Restore successfully completed!\n') except: LOG.error('The ID does not exist, Please try again.\n')
def delete_image(snapshots_id): logging.disable(logging.DEBUG) for snapshot_id in snapshots_id: tenant_id = pc.cinder_get_tenant_id(snapshot_id) images = pc.glance_get_images(tenant_id) for image in images: image_id = image.get('id') image_block_device_mapping = image.get('block_device_mapping') if image_block_device_mapping and snapshot_id in image_block_device_mapping: LOG.info(' Delete image "%s".' % image_id) pc.glance_delete_image(image_id) logging.disable(logging.NOTSET)
def run_doctor_on_nodes(node_list, check_cmd): pile = eventlet.GreenPile() result = [] for node in node_list: LOG.info('%s%s Push check cmd to %-13s (%-10s) %s%s' % ('<', '=' * 2, node['name'], node['role'], '=' * 2, '>')) pile.spawn(run_doctor_cmd_on_node, node['role'], node['name'], check_cmd) for node, res in zip(node_list, pile): result.append(res) logging.disable(logging.NOTSET) return result