def create_bonding(self, node, bonding_name, mode, device_list, new_ip):
bonding = action.IpService(node)
if not self.modify_mode:
connection = bonding.get_connection()
if self.check_bonding_exist(f'vtel_{bonding_name}', connection):
print(f"{bonding_name} already exists.")
sys.exit()
if not utils.check_ip(new_ip):
sys.exit()
lc_device_date = bonding.get_device_status()
if not self.check_device(device_list, lc_device_date):
sys.exit()
if bonding.set_bonding(bonding_name, mode):
gateway = f"{'.'.join(new_ip.split('.')[:3])}.1"
bonding.modify_ip(bonding_name, new_ip, gateway)
if mode == "802.3ad":
bonding.add_bond_options(bonding_name, "xmit_hash_policy=layer3+4")
bonding.add_bond_options(bonding_name, "miimon=100")
bonding.add_bond_options(bonding_name, "lacp_rate=fast")
for device in device_list:
bonding_slave = bonding.add_bond_slave(bonding_name, device)
if bonding_slave:
bonding.up_ip_service(bonding_slave)
else:
print(f'Failed to add bond slave about {device}')
bonding.up_ip_service(f'vtel_{bonding_name}')
def configure_bonding_by_file(self, file):
"""create or modify bonding via yaml config file"""
config = utils.ConfFile(file)
host_config = config.get_config()
self.modify_mode = True
for host in host_config:
node = host[0]
bonding_name = host[1]
mode = host[2]
device_list = host[3]
new_ip = host[4]
print('-' * 15, node, '-' * 15)
bonding = action.IpService(node)
lc_device_date = bonding.get_device_status()
if not self.check_device(device_list, lc_device_date):
continue
connection = bonding.get_connection()
if self.check_bonding_exist(f'vtel_{bonding_name}', connection):
print(f"Modify {bonding_name}.")
self.modify_bonding_slave(node, bonding_name, device_list)
self.modify_bonding_mode(node, bonding_name, mode)
self.modify_bonding_ip(node, bonding_name, new_ip)
else:
print(f"Create {bonding_name}.")
self.create_bonding(node, bonding_name, mode, device_list, new_ip)
def del_bonding(self, node, device):
bonding_name = f'vtel_{device}'
bonding = action.IpService(node)
connection = bonding.get_connection()
if connection:
slave_list = self.get_slave_via_bonding_name(device, connection)
if slave_list:
print("Started to delete bonding slave")
for slave in slave_list:
bonding.down_connect(slave)
if bonding.del_connect(slave):
print(f" Success in deleting {slave}")
else:
print(f" Failed to delete {slave}")
if self.check_bonding_exist(bonding_name, connection):
bonding.down_connect(bonding_name)
print(f"Started to delete {bonding_name}")
if bonding.del_connect(bonding_name):
print(f" Success in deleting {bonding_name}")
else:
print(f" Failed to delete {bonding_name}")
else:
print(f"There is no configuration to delete for {device}.")
else:
print("Can't get any configuration")
def modify_bonding_slave(self, node, bonding_name, device_list):
bonding = action.IpService(node)
if not self.modify_mode:
connection = bonding.get_connection()
if not self.check_bonding_exist(f'vtel_{bonding_name}', connection):
print(f"{bonding_name} does not exist. Do nothing.")
sys.exit()
lc_device_date = bonding.get_device_status()
if not self.check_device(device_list, lc_device_date):
sys.exit()
connection = bonding.get_connection()
slave_list = self.get_slave_via_bonding_name(bonding_name, connection)
device_slave_list = [f'vtel_{bonding_name}-slave-{i}' for i in device_list]
# list_retain = [i for i in device_list if i in slave_list]
list_create = [i for i in device_list if f'vtel_{bonding_name}-slave-{i}' not in slave_list]
list_delete = [i for i in slave_list if i not in device_slave_list]
if list_create or list_delete:
print("Change bonding salve device.")
for device in list_create:
bonding_slave = bonding.add_bond_slave(bonding_name, device)
bonding.up_ip_service(bonding_slave)
for delete_salve in list_delete:
bonding.down_connect(delete_salve)
bonding.del_connect(delete_salve)
bonding.up_ip_service(f'vtel_{bonding_name}')
else:
if not self.modify_mode:
print("Same Device. Do nothing.")
def up_ip_service(self):
lst_up = []
for ssh, node in zip(self.list_ssh, self.cluster['node']):
ip_service = action.IpService(ssh)
lst_up.append(
gevent.spawn(ip_service.up_ip_service,
node['private_ip']['device']))
gevent.joinall(lst_up)
def modify_ip_on_device(self):
lst_modify = []
for ssh, node in zip(self.conn.list_ssh, self.conn.cluster['node']):
ip_service = action.IpService(ssh)
lst_modify.append(
ip_service.modify_ip(node['private_ip']['device'],
node['private_ip']['ip'],
node['private_ip']['gateway']))
self.up_ip_service()
print("Finish to modify ip")
def set_ip_on_device(self):
lst_set = []
for ssh, node in zip(self.list_ssh, self.cluster['node']):
ip_service = action.IpService(ssh)
lst_set.append(
gevent.spawn(ip_service.set_ip, node['private_ip']['device'],
node['private_ip']['ip'],
node['private_ip']['gateway']))
gevent.joinall(lst_set)
self.up_ip_service()
print("Finish to set ip")
def modify_bonding_ip(self, node, device, ip):
bonding = action.IpService(node)
if not self.modify_mode:
if not utils.check_ip(ip):
sys.exit()
connection = bonding.get_connection()
if not self.check_bonding_exist(f'vtel_{device}', connection):
print(f"{device} does not exist. Do nothing.")
sys.exit()
lc_ip_data = bonding.get_bond_ip(device)
if not self.check_bond_ip(ip, lc_ip_data):
print("Change bonding IP.")
gateway = f"{'.'.join(ip.split('.')[:3])}.1"
bonding.modify_ip(device, ip, gateway)
bonding.up_ip_service(f'vtel_{device}')
else:
if not self.modify_mode:
print("Same bonding IP. Do nothing.")
def modify_bonding_mode(self, node, device, mode):
bonding = action.IpService(node)
if not self.modify_mode:
connection = bonding.get_connection()
if not self.check_bonding_exist(f'vtel_{device}', connection):
print(f"{device} does not exist. Do nothing.")
sys.exit()
lc_mode = bonding.get_mode(device)
if not self.check_mode(lc_mode, mode):
print("Change bonding mode.")
if self.check_mode(lc_mode, "802.3ad"):
bonding.delete_bond_options(device, "lacp_rate")
bonding.delete_bond_options(device, "xmit_hash_policy")
bonding.delete_bond_options(device, "miimon")
bonding.modify_bonding_mode(device, mode)
if mode == "802.3ad":
bonding.add_bond_options(device, "xmit_hash_policy=layer3+4")
bonding.add_bond_options(device, "miimon=100")
bonding.add_bond_options(device, "lacp_rate=fast")
bonding.up_ip_service(f'vtel_{device}')
else:
if not self.modify_mode:
print("Same bonding mode. Do nothing.")
def up_ip_service(self):
lst_up = []
for ssh, node in zip(self.conn.list_ssh, self.conn.cluster['node']):
ip_service = action.IpService(ssh)
lst_up.append(
ip_service.up_ip_service(node['private_ip']['device']))
def test_drbd_in_used(self):
start_time = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime())
if len(self.conn.list_vplx_ssh) != 3:
utils.prt_log(
'', f"Please make sure there are three nodes for this test", 2)
test_times = self.config.get_test_times()
device = self.config.get_device()
target = self.config.get_target()
resource = self.config.get_resource()
ip_obj = action.IpService(self.conn.list_vplx_ssh[0])
ip_node = utils.get_global_dict_value(self.conn.list_vplx_ssh[0])
for i in range(test_times):
i = i + 1
utils.set_times(i)
print(f"Number of test times --- {i}")
if not self.check_target_lun_status(target, resource,
self.conn.list_vplx_ssh[0]):
self.collect_crm_report_file(start_time,
self.conn.list_vplx_ssh[0])
self.email.send_autotest_mail()
utils.prt_log(
'', f"Finished to collect crm_report and exit testing ...",
2)
if not self.check_drbd_status(resource):
self.collect_crm_report_file(start_time,
self.conn.list_vplx_ssh[0])
self.email.send_autotest_mail()
utils.prt_log(
'', f"Finished to collect crm_report and exit testing ...",
2)
utils.prt_log(self.conn.list_vplx_ssh[0],
f"Down {device} on {ip_node} ...", 0)
ip_obj.down_device(device)
time.sleep(40)
if not self.check_target_lun_status(target, resource,
self.conn.list_vplx_ssh[1]):
ip_obj.up_device(device)
ip_obj.netplan_apply()
time.sleep(30)
self.collect_crm_report_file(start_time,
self.conn.list_vplx_ssh[0])
self.email.send_autotest_mail()
utils.prt_log(
'', f"Finished to collect crm_report and exit testing ...",
2)
utils.prt_log(self.conn.list_vplx_ssh[0],
f"Up {device} on {ip_node} ...", 0)
ip_obj.up_device(device)
ip_obj.netplan_apply()
time.sleep(30)
if not self.check_drbd_status(resource):
self.collect_crm_report_file(start_time,
self.conn.list_vplx_ssh[0])
self.email.send_autotest_mail()
utils.prt_log(
'', f"Finished to collect crm_report and exit testing ...",
2)
self.restore_resource(resource)
if i == 1:
self.collect_crm_report_file(start_time,
self.conn.list_vplx_ssh[0])
utils.prt_log(self.conn.list_vplx_ssh[0],
f"Finished to collect crm_report", 0)
utils.prt_log(
'', f"Wait 2 minutes to restore the original environment", 0)
time.sleep(120)
self.email.send_autotest_mail()
def test_drbd_quorum(self):
if len(self.conn.list_vplx_ssh) != 3:
utils.prt_log(
'', f"Please make sure there are three nodes for this test", 2)
sp = self.get_sp()
resource = "res_quorum"
test_times = self.config.get_test_times()
use_case = self.config.get_use_case()
vtel_conn = None
if None not in self.conn.list_vplx_ssh:
vtel_conn = self.conn.list_vplx_ssh[0]
self.clean_dmesg()
# utils.prt_log(None, f"Start to install software ...", 0)
# self.install_software()
# TODO 可优化，使用 LINSTOR API 代码
install_obj = action.InstallSoftware(vtel_conn)
install_obj.update_pip()
install_obj.install_vplx()
self.create_linstor_resource(vtel_conn, sp, resource)
stor_obj = action.Stor(vtel_conn)
utils.prt_log('', f"Check DRBD quorum...", 0)
if not stor_obj.check_drbd_quorum(resource):
utils.prt_log(vtel_conn, f'Abnormal quorum status of {resource}',
1)
self.get_log()
self.delete_linstor_resource(vtel_conn, sp, resource)
utils.prt_log('',
f"Finished to collect dmesg and exit testing ...", 2)
if not self.cycle_check_drbd_status(resource):
self.get_log()
self.delete_linstor_resource(vtel_conn, sp, resource)
utils.prt_log('',
f"Finished to collect dmesg and exit testing ...", 2)
device_name = stor_obj.get_device_name(resource)
device_list = [
vplx_config["private_ip"]["device"]
for vplx_config in self.vplx_configs
]
if use_case == 1:
test_conn_list = zip(
self.conn.list_vplx_ssh,
self.conn.list_vplx_ssh[1:] + self.conn.list_vplx_ssh[:1])
mode_total_test_times = 3
if use_case == 2:
test_conn_list = [
(self.conn.list_vplx_ssh[0], self.conn.list_vplx_ssh[1]),
(self.conn.list_vplx_ssh[2], self.conn.list_vplx_ssh[1])
]
mode_total_test_times = 2
device_list.pop(1)
mode_times = 0
total_times = mode_total_test_times * test_times
for conn_list in test_conn_list:
device = device_list.pop(0)
node_a = utils.get_global_dict_value(conn_list[0])
node_b = utils.get_global_dict_value(conn_list[1])
stor_a = action.Stor(conn_list[0])
stor_b = action.Stor(conn_list[1])
ip_a = action.IpService(conn_list[0])
dd_a = action.RWData(conn_list[0])
dd_b = action.RWData(conn_list[1])
mode_str = f"\nMode:({node_a}, {node_b}). Mode expect test times: {mode_total_test_times}."
utils.prt_log('', mode_str, 0)
for i in range(test_times):
times = utils.get_times() + 1
utils.set_times(times)
utils.prt_log(
'',
f"\n{mode_str} test times: {i + 1}. Current test times: {times}. Expect test times: {total_times}.",
0)
stor_a.primary_drbd(resource)
utils.prt_log(conn_list[0],
f"Primary resource on {node_a} ...", 0)
time.sleep(3)
thread1 = threading.Thread(target=dd_a.dd_operation,
args=(device_name, ),
name="thread1")
thread2 = threading.Thread(target=ip_a.down_device,
args=(device, ),
name="thread2")
thread3 = threading.Thread(target=dd_b.dd_operation,
args=(device_name, ),
name="thread3")
thread4 = threading.Thread(target=stor_a.secondary_drbd,
args=(resource, ),
name="thread4")
thread1.start()
time.sleep(20)
thread2.start()
utils.prt_log(conn_list[0], f"Down {device} on {node_a} ...",
0)
thread2.join()
time.sleep(3)
stor_b.primary_drbd(resource)
utils.prt_log(conn_list[1],
f"Primary resource on {node_b} ...", 0)
time.sleep(3)
thread3.start()
time.sleep(10)
resource_status_result = stor_a.get_drbd_status(resource)
if check_drbd_no_quorum(conn_list[0], resource_status_result):
kill_dd(conn_list[0], device_name)
if thread1.is_alive():
stop_thread(thread1)
else:
utils.prt_log(conn_list[0],
f"Configuration 'quorum:no' not exist ...",
0)
self.get_log()
self.email.send_autotest_mail()
utils.prt_log(
'', f"Finished to collect dmesg and exit testing ...",
2)
thread4.start()
utils.prt_log(conn_list[0],
f"Secondary resource on {node_a} ...", 0)
thread4.join()
thread1.join()
time.sleep(10)
kill_dd(conn_list[1], device_name)
time.sleep(5)
if thread3.is_alive():
stop_thread(thread3)
time.sleep(5)
thread3.join()
ip_a.up_device(device)
utils.prt_log(conn_list[0], f"Up {device} on {node_a} ...", 0)
ip_a.netplan_apply()
time.sleep(5)
if not self.cycle_check_drbd_status(resource):
self.get_log()
stor_b.secondary_drbd(resource)
self.delete_linstor_resource(vtel_conn, sp, resource)
self.email.send_autotest_mail()
utils.prt_log(
'', f"Finished to collect dmesg and exit testing ...",
2)
stor_b.secondary_drbd(resource)
utils.prt_log(conn_list[1],
f"Secondary resource on {node_b} ...", 0)
if times == mode_times * test_times + 1:
self.get_log()
mode_times = mode_times + 1
utils.prt_log('', f"Success. Wait 3 minutes.", 0)
time.sleep(180)
self.delete_linstor_resource(vtel_conn, sp, resource)
self.email.send_autotest_mail()
