def create_bonding(self, node, bonding_name, mode, device_list, new_ip): bonding = action.IpService(node) if not self.modify_mode: connection = bonding.get_connection() if self.check_bonding_exist(f'vtel_{bonding_name}', connection): print(f"{bonding_name} already exists.") sys.exit() if not utils.check_ip(new_ip): sys.exit() lc_device_date = bonding.get_device_status() if not self.check_device(device_list, lc_device_date): sys.exit() if bonding.set_bonding(bonding_name, mode): gateway = f"{'.'.join(new_ip.split('.')[:3])}.1" bonding.modify_ip(bonding_name, new_ip, gateway) if mode == "802.3ad": bonding.add_bond_options(bonding_name, "xmit_hash_policy=layer3+4") bonding.add_bond_options(bonding_name, "miimon=100") bonding.add_bond_options(bonding_name, "lacp_rate=fast") for device in device_list: bonding_slave = bonding.add_bond_slave(bonding_name, device) if bonding_slave: bonding.up_ip_service(bonding_slave) else: print(f'Failed to add bond slave about {device}') bonding.up_ip_service(f'vtel_{bonding_name}')
def configure_bonding_by_file(self, file): """create or modify bonding via yaml config file""" config = utils.ConfFile(file) host_config = config.get_config() self.modify_mode = True for host in host_config: node = host[0] bonding_name = host[1] mode = host[2] device_list = host[3] new_ip = host[4] print('-' * 15, node, '-' * 15) bonding = action.IpService(node) lc_device_date = bonding.get_device_status() if not self.check_device(device_list, lc_device_date): continue connection = bonding.get_connection() if self.check_bonding_exist(f'vtel_{bonding_name}', connection): print(f"Modify {bonding_name}.") self.modify_bonding_slave(node, bonding_name, device_list) self.modify_bonding_mode(node, bonding_name, mode) self.modify_bonding_ip(node, bonding_name, new_ip) else: print(f"Create {bonding_name}.") self.create_bonding(node, bonding_name, mode, device_list, new_ip)
def del_bonding(self, node, device): bonding_name = f'vtel_{device}' bonding = action.IpService(node) connection = bonding.get_connection() if connection: slave_list = self.get_slave_via_bonding_name(device, connection) if slave_list: print("Started to delete bonding slave") for slave in slave_list: bonding.down_connect(slave) if bonding.del_connect(slave): print(f" Success in deleting {slave}") else: print(f" Failed to delete {slave}") if self.check_bonding_exist(bonding_name, connection): bonding.down_connect(bonding_name) print(f"Started to delete {bonding_name}") if bonding.del_connect(bonding_name): print(f" Success in deleting {bonding_name}") else: print(f" Failed to delete {bonding_name}") else: print(f"There is no configuration to delete for {device}.") else: print("Can't get any configuration")
def modify_bonding_slave(self, node, bonding_name, device_list): bonding = action.IpService(node) if not self.modify_mode: connection = bonding.get_connection() if not self.check_bonding_exist(f'vtel_{bonding_name}', connection): print(f"{bonding_name} does not exist. Do nothing.") sys.exit() lc_device_date = bonding.get_device_status() if not self.check_device(device_list, lc_device_date): sys.exit() connection = bonding.get_connection() slave_list = self.get_slave_via_bonding_name(bonding_name, connection) device_slave_list = [f'vtel_{bonding_name}-slave-{i}' for i in device_list] # list_retain = [i for i in device_list if i in slave_list] list_create = [i for i in device_list if f'vtel_{bonding_name}-slave-{i}' not in slave_list] list_delete = [i for i in slave_list if i not in device_slave_list] if list_create or list_delete: print("Change bonding salve device.") for device in list_create: bonding_slave = bonding.add_bond_slave(bonding_name, device) bonding.up_ip_service(bonding_slave) for delete_salve in list_delete: bonding.down_connect(delete_salve) bonding.del_connect(delete_salve) bonding.up_ip_service(f'vtel_{bonding_name}') else: if not self.modify_mode: print("Same Device. Do nothing.")
def up_ip_service(self): lst_up = [] for ssh, node in zip(self.list_ssh, self.cluster['node']): ip_service = action.IpService(ssh) lst_up.append( gevent.spawn(ip_service.up_ip_service, node['private_ip']['device'])) gevent.joinall(lst_up)
def modify_ip_on_device(self): lst_modify = [] for ssh, node in zip(self.conn.list_ssh, self.conn.cluster['node']): ip_service = action.IpService(ssh) lst_modify.append( ip_service.modify_ip(node['private_ip']['device'], node['private_ip']['ip'], node['private_ip']['gateway'])) self.up_ip_service() print("Finish to modify ip")
def set_ip_on_device(self): lst_set = [] for ssh, node in zip(self.list_ssh, self.cluster['node']): ip_service = action.IpService(ssh) lst_set.append( gevent.spawn(ip_service.set_ip, node['private_ip']['device'], node['private_ip']['ip'], node['private_ip']['gateway'])) gevent.joinall(lst_set) self.up_ip_service() print("Finish to set ip")
def modify_bonding_ip(self, node, device, ip): bonding = action.IpService(node) if not self.modify_mode: if not utils.check_ip(ip): sys.exit() connection = bonding.get_connection() if not self.check_bonding_exist(f'vtel_{device}', connection): print(f"{device} does not exist. Do nothing.") sys.exit() lc_ip_data = bonding.get_bond_ip(device) if not self.check_bond_ip(ip, lc_ip_data): print("Change bonding IP.") gateway = f"{'.'.join(ip.split('.')[:3])}.1" bonding.modify_ip(device, ip, gateway) bonding.up_ip_service(f'vtel_{device}') else: if not self.modify_mode: print("Same bonding IP. Do nothing.")
def modify_bonding_mode(self, node, device, mode): bonding = action.IpService(node) if not self.modify_mode: connection = bonding.get_connection() if not self.check_bonding_exist(f'vtel_{device}', connection): print(f"{device} does not exist. Do nothing.") sys.exit() lc_mode = bonding.get_mode(device) if not self.check_mode(lc_mode, mode): print("Change bonding mode.") if self.check_mode(lc_mode, "802.3ad"): bonding.delete_bond_options(device, "lacp_rate") bonding.delete_bond_options(device, "xmit_hash_policy") bonding.delete_bond_options(device, "miimon") bonding.modify_bonding_mode(device, mode) if mode == "802.3ad": bonding.add_bond_options(device, "xmit_hash_policy=layer3+4") bonding.add_bond_options(device, "miimon=100") bonding.add_bond_options(device, "lacp_rate=fast") bonding.up_ip_service(f'vtel_{device}') else: if not self.modify_mode: print("Same bonding mode. Do nothing.")
def up_ip_service(self): lst_up = [] for ssh, node in zip(self.conn.list_ssh, self.conn.cluster['node']): ip_service = action.IpService(ssh) lst_up.append( ip_service.up_ip_service(node['private_ip']['device']))
def test_drbd_in_used(self): start_time = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime()) if len(self.conn.list_vplx_ssh) != 3: utils.prt_log( '', f"Please make sure there are three nodes for this test", 2) test_times = self.config.get_test_times() device = self.config.get_device() target = self.config.get_target() resource = self.config.get_resource() ip_obj = action.IpService(self.conn.list_vplx_ssh[0]) ip_node = utils.get_global_dict_value(self.conn.list_vplx_ssh[0]) for i in range(test_times): i = i + 1 utils.set_times(i) print(f"Number of test times --- {i}") if not self.check_target_lun_status(target, resource, self.conn.list_vplx_ssh[0]): self.collect_crm_report_file(start_time, self.conn.list_vplx_ssh[0]) self.email.send_autotest_mail() utils.prt_log( '', f"Finished to collect crm_report and exit testing ...", 2) if not self.check_drbd_status(resource): self.collect_crm_report_file(start_time, self.conn.list_vplx_ssh[0]) self.email.send_autotest_mail() utils.prt_log( '', f"Finished to collect crm_report and exit testing ...", 2) utils.prt_log(self.conn.list_vplx_ssh[0], f"Down {device} on {ip_node} ...", 0) ip_obj.down_device(device) time.sleep(40) if not self.check_target_lun_status(target, resource, self.conn.list_vplx_ssh[1]): ip_obj.up_device(device) ip_obj.netplan_apply() time.sleep(30) self.collect_crm_report_file(start_time, self.conn.list_vplx_ssh[0]) self.email.send_autotest_mail() utils.prt_log( '', f"Finished to collect crm_report and exit testing ...", 2) utils.prt_log(self.conn.list_vplx_ssh[0], f"Up {device} on {ip_node} ...", 0) ip_obj.up_device(device) ip_obj.netplan_apply() time.sleep(30) if not self.check_drbd_status(resource): self.collect_crm_report_file(start_time, self.conn.list_vplx_ssh[0]) self.email.send_autotest_mail() utils.prt_log( '', f"Finished to collect crm_report and exit testing ...", 2) self.restore_resource(resource) if i == 1: self.collect_crm_report_file(start_time, self.conn.list_vplx_ssh[0]) utils.prt_log(self.conn.list_vplx_ssh[0], f"Finished to collect crm_report", 0) utils.prt_log( '', f"Wait 2 minutes to restore the original environment", 0) time.sleep(120) self.email.send_autotest_mail()
def test_drbd_quorum(self): if len(self.conn.list_vplx_ssh) != 3: utils.prt_log( '', f"Please make sure there are three nodes for this test", 2) sp = self.get_sp() resource = "res_quorum" test_times = self.config.get_test_times() use_case = self.config.get_use_case() vtel_conn = None if None not in self.conn.list_vplx_ssh: vtel_conn = self.conn.list_vplx_ssh[0] self.clean_dmesg() # utils.prt_log(None, f"Start to install software ...", 0) # self.install_software() # TODO 可优化,使用 LINSTOR API 代码 install_obj = action.InstallSoftware(vtel_conn) install_obj.update_pip() install_obj.install_vplx() self.create_linstor_resource(vtel_conn, sp, resource) stor_obj = action.Stor(vtel_conn) utils.prt_log('', f"Check DRBD quorum...", 0) if not stor_obj.check_drbd_quorum(resource): utils.prt_log(vtel_conn, f'Abnormal quorum status of {resource}', 1) self.get_log() self.delete_linstor_resource(vtel_conn, sp, resource) utils.prt_log('', f"Finished to collect dmesg and exit testing ...", 2) if not self.cycle_check_drbd_status(resource): self.get_log() self.delete_linstor_resource(vtel_conn, sp, resource) utils.prt_log('', f"Finished to collect dmesg and exit testing ...", 2) device_name = stor_obj.get_device_name(resource) device_list = [ vplx_config["private_ip"]["device"] for vplx_config in self.vplx_configs ] if use_case == 1: test_conn_list = zip( self.conn.list_vplx_ssh, self.conn.list_vplx_ssh[1:] + self.conn.list_vplx_ssh[:1]) mode_total_test_times = 3 if use_case == 2: test_conn_list = [ (self.conn.list_vplx_ssh[0], self.conn.list_vplx_ssh[1]), (self.conn.list_vplx_ssh[2], self.conn.list_vplx_ssh[1]) ] mode_total_test_times = 2 device_list.pop(1) mode_times = 0 total_times = mode_total_test_times * test_times for conn_list in test_conn_list: device = device_list.pop(0) node_a = utils.get_global_dict_value(conn_list[0]) node_b = utils.get_global_dict_value(conn_list[1]) stor_a = action.Stor(conn_list[0]) stor_b = action.Stor(conn_list[1]) ip_a = action.IpService(conn_list[0]) dd_a = action.RWData(conn_list[0]) dd_b = action.RWData(conn_list[1]) mode_str = f"\nMode:({node_a}, {node_b}). Mode expect test times: {mode_total_test_times}." utils.prt_log('', mode_str, 0) for i in range(test_times): times = utils.get_times() + 1 utils.set_times(times) utils.prt_log( '', f"\n{mode_str} test times: {i + 1}. Current test times: {times}. Expect test times: {total_times}.", 0) stor_a.primary_drbd(resource) utils.prt_log(conn_list[0], f"Primary resource on {node_a} ...", 0) time.sleep(3) thread1 = threading.Thread(target=dd_a.dd_operation, args=(device_name, ), name="thread1") thread2 = threading.Thread(target=ip_a.down_device, args=(device, ), name="thread2") thread3 = threading.Thread(target=dd_b.dd_operation, args=(device_name, ), name="thread3") thread4 = threading.Thread(target=stor_a.secondary_drbd, args=(resource, ), name="thread4") thread1.start() time.sleep(20) thread2.start() utils.prt_log(conn_list[0], f"Down {device} on {node_a} ...", 0) thread2.join() time.sleep(3) stor_b.primary_drbd(resource) utils.prt_log(conn_list[1], f"Primary resource on {node_b} ...", 0) time.sleep(3) thread3.start() time.sleep(10) resource_status_result = stor_a.get_drbd_status(resource) if check_drbd_no_quorum(conn_list[0], resource_status_result): kill_dd(conn_list[0], device_name) if thread1.is_alive(): stop_thread(thread1) else: utils.prt_log(conn_list[0], f"Configuration 'quorum:no' not exist ...", 0) self.get_log() self.email.send_autotest_mail() utils.prt_log( '', f"Finished to collect dmesg and exit testing ...", 2) thread4.start() utils.prt_log(conn_list[0], f"Secondary resource on {node_a} ...", 0) thread4.join() thread1.join() time.sleep(10) kill_dd(conn_list[1], device_name) time.sleep(5) if thread3.is_alive(): stop_thread(thread3) time.sleep(5) thread3.join() ip_a.up_device(device) utils.prt_log(conn_list[0], f"Up {device} on {node_a} ...", 0) ip_a.netplan_apply() time.sleep(5) if not self.cycle_check_drbd_status(resource): self.get_log() stor_b.secondary_drbd(resource) self.delete_linstor_resource(vtel_conn, sp, resource) self.email.send_autotest_mail() utils.prt_log( '', f"Finished to collect dmesg and exit testing ...", 2) stor_b.secondary_drbd(resource) utils.prt_log(conn_list[1], f"Secondary resource on {node_b} ...", 0) if times == mode_times * test_times + 1: self.get_log() mode_times = mode_times + 1 utils.prt_log('', f"Success. Wait 3 minutes.", 0) time.sleep(180) self.delete_linstor_resource(vtel_conn, sp, resource) self.email.send_autotest_mail()