def test(self, leader_follower=True): """check the integrity of the old system after install (Leader Follower)""" count = 0 replication_table = None while True: NavigationBarPage(self.webdriver, self.cfg).navbar_goto("replication") replication_page = ReplicationPage(self.webdriver, self.cfg) replication_table = replication_page.get_replication_screen(leader_follower, 120) self.progress(" " + str(replication_table)) if len(replication_table["follower_table"]) == 2: break if count % 5 == 0: self.webdriver.refresh() count += 1 time.sleep(5) # head and one follower should be there: self.ui_assert(len(replication_table["follower_table"]) == 2, "UI-Test: expected 1 follower")
def jam_step_2(self): """ step 2 jamming: check the instances are gone from the table """ NavigationBarPage(self.webdriver, self.cfg).navbar_goto("cluster") cluster_page = ClusterPage(self.webdriver, self.cfg) node_count = None done = False retry_count = 0 while not done: node_count = cluster_page.cluster_dashboard_get_count() done = (node_count["dbservers"] == "3") and (node_count["coordinators"] == "3") if not done: time.sleep(3) retry_count += 1 self.ui_assert( retry_count < 10, "UI-Test: expected 3 instances each, have: DB " + node_count["dbservers"] + " C " + node_count["coordinators"], )
def check_follower_count(self, expect_follower_count=2, retry_count=10): """check the integrity of the old system after the install""" while retry_count > 0: NavigationBarPage(self.webdriver, self.cfg).navbar_goto("replication") replication_page = ReplicationPage(self.webdriver, self.cfg) replication_table = replication_page.get_replication_screen(True) print(replication_table) if len(replication_table["follower_table"]) != expect_follower_count + 1: time.sleep(5) retry_count -= 1 else: retry_count = 0 # its there! # head and two followers should be there: self.progress( " expecting %d followers, have %d followers" % (expect_follower_count, len(replication_table["follower_table"]) - 1) ) self.ui_assert( len(replication_table["follower_table"]) == expect_follower_count + 1, "UI-Test:\nexpect 1 follower in:\n%s" % pprint.pformat(replication_table), )
def after_jam_step_2(self): """check the integrity of the system after recovery from cluster failure""" version = (self.selenium_runner.new_cfg.version if self.selenium_runner.new_cfg else self.selenium_runner.cfg.version) self.check_version(version, self.is_enterprise) NavigationBarPage(self.webdriver, self.cfg).navbar_goto("nodes") nodes_page = NodesPage(self.webdriver, self.cfg) table = nodes_page.cluster_get_nodes_table() row_count = 0 for row in table: if row["state"] == "SERVING": row_count += 1 self.progress("serving instances 6 / %d" % row_count) self.ui_assert(row_count == 6, "UI-Test: expected 6 instances") nodes_page.navbar_goto("cluster") cluster_page = ClusterPage(self.webdriver, self.cfg) node_count = cluster_page.cluster_dashboard_get_count() self.ui_assert( node_count["dbservers"] == "3", "UI-Test: expected 3 dbservers, got: " + node_count["dbservers"]) self.ui_assert( node_count["coordinators"] == "3", "UI-Test: expected 3 coordinators, got: " + node_count["coordinators"]) health_state = None count = 0 while count < 10: health_state = cluster_page.get_health_state() if health_state == "NODES OK": break count += 1 time.sleep(1) self.ui_assert(health_state == "NODES OK", "UI-Test: expected all nodes to be OK")
def jam_step_1(self): """check for one set of instances to go away""" self.webdriver.refresh() time.sleep(2) NavigationBarPage(self.webdriver, self.cfg).navbar_goto("cluster") cluster_page = ClusterPage(self.webdriver, self.cfg) node_count = None done = False retry_count = 0 while not done: # the wintendo is slow to notice that the hosts are gone. timeout = 500 if self.WINVER[0] else 50 node_count = cluster_page.cluster_dashboard_get_count(timeout) done = ((node_count["dbservers"] == "2/3") and (node_count["coordinators"] == "2/3") and (cluster_page.get_health_state() != "NODES OK")) self.ui_assert( retry_count < 40, "UI-Test: Timeout: expected db + c to be 2/3, have: " + node_count["dbservers"] + ", " + node_count["coordinators"], ) if not done: time.sleep(3) retry_count += 1 self.ui_assert(node_count["dbservers"] == "2/3", "UI-Test: dbservers: " + node_count["dbservers"]) self.ui_assert(node_count["coordinators"] == "2/3", "UI-Test: coordinators: " + node_count["coordinators"]) health_state = cluster_page.get_health_state() self.ui_assert( health_state != "NODES OK", "UI-Test: expected health to be NODES OK, have: " + health_state) NavigationBarPage(self.webdriver, self.cfg).navbar_goto("nodes") nodes_page = NodesPage(self.webdriver, self.cfg) row_count = 0 retry_count = 0 while row_count != 4 and retry_count < 10: table = nodes_page.cluster_get_nodes_table(300) for row in table: if row["state"] == "SERVING": row_count += 1 retry_count += 1 if row_count != 4: self.webdriver.refresh() time.sleep(2) row_count = 0 self.progress(" serving instances 6 / %d [%d]" % (row_count, retry_count)) self.ui_assert( row_count == 4, "UI-Test: expect 2 instances to be offline have %d of 6" % row_count) health_state = None count = 0 while count < 10: health_state = nodes_page.get_health_state() if health_state != "NODES OK": break count += 1 time.sleep(1) self.ui_assert( health_state != "NODES OK", "UI-Test: wrong health stame after jam: " + health_state)