def network_device_fiber_check(self, target_network_device):
cmd = "cat /sys/class/net/%s/operstate" % target_network_device
cmd1 = "ethtool %s|grep 'Supported ports'|awk '{print $4}'" \
% target_network_device
cmd2 = "ethtool -m %s|grep 'Laser bias current '|" \
"awk '{print $5}'" % target_network_device
if target_network_device in self.target_network_devices:
target_network_device_state_var = shell(cmd).stdout.upper()
if target_network_device_state_var.strip('\n') == 'UP':
target_network_device_type = shell(cmd1).stdout.upper()
if target_network_device_type == 'FIBRE':
laser_bias = shell(cmd2).stdout
if not laser_bias:
LOG.info('%s bias:null' % target_network_device)
pass
elif float(laser_bias) > 12:
LOG.info('%s bias:%s' %
(target_network_device, laser_bias))
else:
LOG.debug('%s bias:%s' %
(target_network_device, laser_bias))
else:
LOG.debug('%s:TP' % target_network_device)
else:
LOG.debug('%s:down' % target_network_device)
else:
LOG.debug('%s:None' % target_network_device)
def pre_ping_check(self, device, targete_getway):
cmd = "arping -I %s %s -c 3 -b -f >/dev/null" \
% (device, targete_getway)
shell(cmd)
cmd1 = "ping -I %s %s -i 0.1 -w 2 >/dev/null" \
% (device, targete_getway)
shell(cmd1)
def is_numa_balance(self):
# numa node number
cmd1 = "lscpu | grep 'NUMA node(s)' | awk -F: '{print $2}' | " \
"sed 's/[ \t]*//g'"
num_of_numa = int(shell(cmd1).stdout)
# memmory channel number of each numa node
cmd2 = "dmidecode -t 17 | grep -A4 Size | " \
"grep -v 'No Module Installed' | grep -E 'Size|Bank' | " \
"sed -n '/Size:/,+1p' | grep 'Bank Locator' | " \
"awk -F_ '{print $3}' | sort -u | wc -l"
num_of_channel = int(shell(cmd2).stdout)
# memory number of each channel
num_of_mems = list()
for node in xrange(num_of_numa):
for channel in xrange(num_of_channel):
num_of_mem_sh = "dmidecode -t 17 | grep -A4 Size | " \
"grep -v 'No Module Installed' | " \
"grep -E 'Size|Bank' | sed -n '/Size:/,+1p'" \
" | grep 'Bank Locator' | awk -F_ " \
""" '$2=="Node%d" && $3=="Channel%d" """ \
"{print $2 $3 $4}' | wc -l" % (node, channel)
num_of_mem = int(shell(num_of_mem_sh).stdout)
num_of_mems.append(num_of_mem)
num_of_mems.sort()
if num_of_mems[0] == num_of_mems[-1]:
LOG.debug("memory numa balance: OK")
else:
LOG.error("memory numa balance: ERROR")
def os_disk_partition(self):
# 确保磁盘分区正确，如果正确则返回值为'3'
cmd = "lsblk | grep -w -E 'sda|/var|/' | wc -l"
# 分别获取磁盘sda、/、/var的存储空间
cmd1 = "lsblk | grep -w sda | awk '{print $4}'"
cmd2 = "lsblk | grep sda | grep -w / | awk '{print $4}'"
cmd3 = "lsblk | grep sda |grep -w /var | awk '{print $4}'"
disk_partition_format = shell(cmd).stdout.strip()
sda_disk_size = shell(cmd1).stdout.strip()
root_partition_size = shell(cmd2).stdout.strip()
var_partition_size = shell(cmd3).stdout.strip()
if disk_partition_format != '3':
LOG.error('Disk_Part:Error')
return
# 对比计算各个分区的大小，要求sda分两个分区：系统盘50G，剩余空间都分给/var,否则就报错
if int(float(root_partition_size.strip('G'))) == 50 \
and int(float(root_partition_size.strip('G'))) + \
int(float(var_partition_size.strip('G'))) \
== int(float(sda_disk_size.strip('G'))):
LOG.debug('sda:%s root:%s var:%s ' %
(sda_disk_size, root_partition_size, var_partition_size))
if int(float(root_partition_size.strip('G'))) != 50:
LOG.error('root_size:Error(%s)' % root_partition_size)
if int(float(root_partition_size.strip('G'))) \
+ int(float(var_partition_size.strip('G'))) \
!= int(float(sda_disk_size.strip('G'))):
LOG.error('var_size:Error(%s)' % var_partition_size)
def tcp_and_udp_port_check(self):
tcp_conf_port = [22]
udp_conf_port = []
cmd1 = "ss -lantu4 | grep LISTEN | grep -w tcp | " \
"awk '{print $5}' | awk -F ':' '{print $2}' | uniq | wc -l"
cmd2 = "ss -lantu4 | grep LISTEN | grep -w tcp | awk '{print $5}' " \
"| awk -F ':' '{print $2}' | uniq | tr '\n' ' '"
cmd3 = "ss -lantu4|grep udp|awk '{print $5}' | " \
"awk -F ':' '{print $2}'|uniq|wc -l"
cmd4 = "ss -lantu4 | grep udp | awk '{print $5}' | " \
"awk -F ':' '{print $2}' | uniq|tr '\n' ' '"
listen_tcp_port_num = shell(cmd1).stdout
listen_tcp_port = map(int, shell(cmd2).stdout.split())
listen_udp_port_num = shell(cmd3).stdout
listen_udp_port = map(int, shell(cmd4).stdout.split())
redundant_tcp_port = ' '.join(
str(s)
for s in set(listen_tcp_port).difference(set(tcp_conf_port)))
redundant_udp_port = ' '.join(
str(s)
for s in set(listen_udp_port).difference(set(udp_conf_port)))
LOG.debug('TCP_Port_Num:%d TCP_Port:%s UDP_Port_Num:%d UDP_Port:%s' %
(int(listen_tcp_port_num), listen_tcp_port,
int(listen_udp_port_num), listen_udp_port))
if set(listen_tcp_port) != set(tcp_conf_port):
LOG.warn('TCP:%s' % redundant_tcp_port)
if set(listen_udp_port) != set(udp_conf_port):
LOG.warn('UDP:%s' % redundant_udp_port)
def phy_device_state_check(self, target_device):
cmd1 = "cat /proc/net/bonding/%s | grep 'Slave Interface' " \
"| awk '{print $3}'" % target_device
bonding_slaves = shell(cmd1).stdout.replace('\n', ' ')
cmd2 = "for i in %s ;do cat /sys/class/net/$i/operstate ;done" \
% bonding_slaves
bonding_slaves_state = shell(cmd2).stdout.replace('\n', ' ').split()
slave_state_dict = dict(
zip(bonding_slaves.split(), bonding_slaves_state))
return slave_state_dict
def firewalld_service_stop(self):
cmd1 = "systemctl status firewalld" \
"|grep -w active|awk '{print $2}'"
cmd2 = "systemctl stop firewalld"
firewalld_service_state = shell(cmd1).stdout
if firewalld_service_state.strip() == 'active':
shell(cmd2)
return 0
else:
return 1
def phy_device_state_check(self, target_device):
bonding_slaves_state = []
if self.pre_check(target_device):
cmd2 = "cat /proc/net/bonding/%s | grep 'Slave Interface' " \
"| awk '{print $3}'" % target_device
bonding_slaves = shell(cmd2).stdout.replace('\n', ' ')
cmd3 = "for i in %s ;do cat /sys/class/net/$i/operstate ;done" \
% bonding_slaves
bonding_slaves_state = shell(cmd3).stdout.\
replace('\n', ' ').split()
return bonding_slaves_state
def uplinks_up(self, target_device):
cmd = "esxcli network nic up -n=%s" % target_device
shell(cmd)
cmd1 = "esxcli network nic list | grep %s | awk '{print $5}'" \
% target_device
time.sleep(4)
link_status = shell(cmd1).stdout.lower()
if link_status == "up":
LOG.debug('%s:up success!' % target_device)
elif link_status == "down":
LOG.info('%s:up failed!' % target_device)
else:
LOG.info('%s:status error!' % target_device)
def __init__(self):
if not os.path.isfile('lshw.tmp.txt'):
if not self.pre_check():
cmd1 = "yum install lshw -y"
is_installed = shell(cmd1).returncode
if is_installed == 0:
LOG.debug("installed 'lshw' successfully!")
cmd2 = "lshw > lshw.tmp.txt"
shell(cmd2)
else:
LOG.error("Can not install 'lshw'!")
else:
cmd3 = "lshw > lshw.tmp.txt"
shell(cmd3)
def check_network_number(self, counts):
cmd1 = "lspci | grep Ethernet | awk '{print $1}'"
pci_buses = shell(cmd1).stdout.split('\n')
for pci_bus in pci_buses:
for key in counts.keys():
cmd2 = "cat lshw.tmp.txt | grep -A5 %s | grep -i %s " \
% (pci_bus, key)
if shell(cmd2).returncode == 0:
counts[key] = counts[key] - 1
flag = True
for key, val in counts.iteritems():
if val != 0:
flag = False
LOG.error("number of %s network card: ERROR" % key)
return flag
def os_service_check(self, service, service_software, service_state_conf):
# 查看服务软件是否已经安装
cmd1 = "rpm -qa | grep %s >/dev/null;echo $?" % service_software
# 获取指定服务的运行状态
cmd2 = "systemctl status %s | grep Active|" \
"awk -F '(' '{print $2}' | awk -F ')' '{print $1}'" % service
service_exist = shell(cmd1).stdout.strip()
service_state = shell(cmd2).stdout.strip()
# 判断服务是否已经安装，如果返回值不为0，则代表服务未安装
if service_exist != '0':
LOG.error('%s:NotExist' % service)
return
if service_state == service_state_conf:
LOG.debug('%s:OK(%s)' % (service, service_state))
else:
LOG.error('%s:Error(%s)' % (service, service_state))
def network_device_mac_check(self, target_network_device):
cmd = "cat /sys/class/net/%s/address" % target_network_device
if target_network_device in self.target_network_devices:
target_network_device_mac_var = shell(cmd).stdout
LOG.debug('MAC:%s' % (target_network_device_mac_var).strip('\n'))
else:
LOG.warn('MAC:None')
def device_get_ip(self, device):
cmd = " ip -o -4 addr show %s | awk '{print $4}' " % device
ip = shell(cmd).stdout
if self.ip_check(ip):
return ip
else:
return None
def del_ip(self, target_network_device, ip):
cmd_del_ip = "ip addr del %s dev %s" % (ip, target_network_device)
result = shell(cmd_del_ip).stdout
LOG.debug('ip addr del %s dev %s" \n%s' %
(ip, target_network_device, result))
if result:
LOG.info(result)
def get_nvme_interrupt(self):
for nvme in self.get_nvme_numa_nodes():
cmd1 = "cat /proc/interrupts | grep %s | awk -F: " \
"'NR==1 {print $1}' | sed 's/^[ \t]*//g'" % nvme
irq = shell(cmd1).stdout
cmd2 = "cat /proc/irq/%s/node" % irq
interrupt = int(shell(cmd2).stdout)
cmd = "cat /sys/class/nvme/%s/device/numa_node | " \
"awk '{print $1}'" % nvme
numa_node = int(shell(cmd).stdout)
if interrupt == numa_node:
LOG.debug("The interrupt of nvme ssd %s is binding on "
"the same numa node" % nvme)
else:
LOG.error("The interrupt of nvme ssd %s is binding on "
"different numa nodes" % nvme)
def ip_config(self, target_network_device, ip):
cmd_set_ip = "ip addr add %s dev %s" % (ip, target_network_device)
result = shell(cmd_set_ip).stdout
LOG.debug('ip addr add %s dev %s" \n%s' %
(ip, target_network_device, result))
if result:
LOG.info(result)
def pre_check(self):
cmd = "which nvme"
is_exist = shell(cmd).returncode
if is_exist == 0:
return True
else:
return False
def __init__(self):
if not self.pre_check():
cmd = "yum install nvme-cli -y"
is_installed = shell(cmd).returncode
if is_installed == 0:
LOG.debug("installed 'nvme-cli' successfully!")
else:
LOG.error("Can not install 'nvme-cli'!")
def bond_mode_check(self, target_device):
cmd = "cat /proc/net/bonding/%s | grep 'Bonding Mode' " \
"| awk -F ':' '{print $2}'" % target_device
bond_mode = shell(cmd).stdout.strip()
for mode_name, mode_alg in self.bond_mode_list.items():
if mode_alg == bond_mode:
return mode_name
return None
def os_disk_raid_check(self):
# 查看系统磁盘是否配置了RAID,配置返回'0'，没配置返回'1'
cmd = "lspci -knn | grep 'RAID' > /dev/null && echo 0 || echo 1"
raid_conf = shell(cmd).stdout.strip()
if raid_conf == '0':
LOG.debug('raid_check:OK')
else:
LOG.error('raid_check:Error')
def get_post_ip_and_netmask(self, network):
ip_prefix = self.ip_analysis(network)[1]
ip_netmask = self.ip_analysis(network)[3]
cmd = "ip -o -4 a | grep '%s' | awk '{print $4}' " % ip_prefix
ip = shell(cmd).stdout
post_ip = self.ip_analysis(ip)[2]
post_ip_and_netmask = post_ip + ip_netmask
return post_ip_and_netmask
def kernel_release_check(self, kernel_release_conf):
# 获取linux内核版本
cmd = "uname -a | awk '{print $3}'"
kernel_release = shell(cmd).stdout.strip()
if kernel_release == kernel_release_conf:
LOG.debug('Kernel_Check:OK')
else:
LOG.error('Kernel_Check:Error(%s)' % kernel_release)
def package_check(self, package):
# 查看指定软件包是否安装，安装返回'0'，没安装返回'1'
cmd = "rpm -qa | grep -w %s >/dev/null && echo 0 || echo 1" % package
package_status_code = shell(cmd).stdout.strip()
if package_status_code == '0':
LOG.debug('%s:OK' % package)
else:
LOG.error('%s:Error' % package)
def phy_device_check(self, target_device):
bonding_slaves_set = set()
if self.pre_check(target_device):
cmd = "cat /proc/net/bonding/%s | grep 'Slave Interface' " \
"| awk '{print $3}'" % target_device
bonding_slaves_set = set(
shell(cmd).stdout.replace('\n', ' ').split())
return bonding_slaves_set
def os_release_check(self, os_release_conf):
# 获取Linux系统版本
cmd = "a=$(cat /etc/redhat-release | awk '{print $1}');" \
"b=$(cat /etc/redhat-release | awk '{print $4}');echo ${a}_${b}"
os_release = shell(cmd).stdout.strip()
if os_release == os_release_conf:
LOG.debug('OS_Check:OK')
else:
LOG.error('OS_check:Error(%s)' % os_release)
def iperf_server_client_num_check(self, device_name):
device_ipaddr = self.device_ipaddr_check(device_name)
if device_ipaddr == '':
client_num = 0
else:
client_num_cmd = "cat /tmp/iperf_server.txt|grep %s" \
"|sort|uniq|wc -l" % device_ipaddr
client_num = shell(client_num_cmd).stdout
return int(client_num)
def nvme_device_numa_check(self):
cmd1 = "ls /sys/class/block|grep nvme &>/dev/null ;echo $?"
cmd2 = "for i in /sys/class/block/nvme[0-9]n[0-9];" \
"do cat $i/device/device/numa_node;done|grep 0|wc -l"
cmd3 = "for i in /sys/class/block/nvme[0-9]n[0-9];" \
"do cat $i/device/device/numa_node;done|grep 1|wc -l"
is_exits = shell(cmd1).stdout
numa_node0_num = shell(cmd2).stdout.strip()
numa_node1_num = shell(cmd3).stdout.strip()
if int(is_exits) == 0:
if int(numa_node0_num) == int(numa_node1_num):
LOG.debug('Nvme_numa_0:%s Nvme_numa_1:%s' %
(numa_node0_num, numa_node1_num))
else:
LOG.error('Nvme_numa_0:Error(%s) Nvme_numa_1:Error(%s)' %
(numa_node0_num, numa_node1_num))
else:
LOG.warn('Nvme:None')
def check_connection(self, conf):
a = NetworkConnectionCheck()
ping_test_duration = conf.ping_test_duration
test_device_count = len(conf.target_devices.keys())
if a.network_device_check(conf.target_devices.keys()):
threads = []
for device in conf.target_devices.keys():
target_getway = conf.target_devices[device]['gateway']
t = threading.Thread(target=a.pre_ping_check,
args=(device, target_getway))
threads.append(t)
t.start()
for t in threads:
t.join()
threads = []
for device in conf.target_devices.keys():
target_getway = conf.target_devices[device]['gateway']
t = threading.Thread(target=a.ping_check,
args=(device, target_getway,
ping_test_duration))
threads.append(t)
t.start()
time.sleep(1)
for t in threads:
t.join()
cmd = "grep transmitted /tmp/*pingtxt|wc -l"
result_num = shell(cmd).stdout
second = 0
while int(result_num) != int(test_device_count):
result_num = shell(cmd).stdout
time.sleep(1)
second = second + 1
if second == conf.ping_test_timeout:
print("Timeout!")
break
else:
a._check_ping_result(conf.target_devices.keys())
a.test_file_delete(conf.target_devices.keys())
else:
LOG.warn('Device:Error')
def _check_result(self, target_devices):
for device in target_devices:
cmd1 = "cat /tmp/%s.txt |grep transmitted|" \
"awk '{print $1}'" % device
cmd2 = "cat /tmp/%s.txt |grep transmitted|" \
"awk '{print $4}'" % device
transmitted_packets = shell(cmd1).stdout
received_packets = shell(cmd2).stdout
loss_num = int(transmitted_packets) - int(received_packets)
loss_percent = (float(loss_num) / float(transmitted_packets))
if int(transmitted_packets) \
== int(received_packets) or loss_percent < 0.005:
LOG.debug('%s:OK(%d)' % (device, loss_num))
elif int(received_packets) == 0:
LOG.error('%s:Unreachable' % device)
else:
LOG.error(
'%s:Error(%d/%d/%f)' %
(device, int(transmitted_packets), loss_num, loss_percent))