def do_test_with_updates(self, state_transitions): # start from a clean state always common.set_partition_states( params = self._params, cleanup = range(self._params["nparts"]) ) # print "Cleaning set view via compaction" common.compact_set_view(self._params) (resp, view_result) = common.query(self._params, "mapview") self.assertEqual(view_result["total_rows"], 0, "total_rows is 0 after compaction") self.assertEqual(len(view_result["rows"]), 0, "0 rows returned after compaction") info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") self.assertEqual(info["update_seqs"], {}, "right update seqs list") # print "Triggering state changes with queries (?stale=updater_after) in between" for state in state_transitions: common.set_partition_states( params = self._params, active = state["active"], passive = state["passive"], cleanup = state["cleanup"] ) time.sleep(0.5) common.query(self._params, "mapview", {"stale": "update_after", "limit": "100"}) group_stats = self.verify_final_index_state(state_transitions) self.assertTrue(group_stats["updater_interruptions"] > 0, "Got at least 1 updater interruption") self.assertEqual(group_stats["updates"], 2, "Got 2 full updates") self.assertEqual(group_stats["compactions"], 1, "Got 1 compaction") self.assertTrue(group_stats["cleanups"] >= 1, "Got at least 1 full cleanup")
def do_test_set_active_during_compaction(self): # print "Triggering view compaction" common.compact_set_view(self._params, False) # print "Marking partitions 4, 5 and 6 as active while compaction is running" common.set_partition_states(self._params, active = [3, 4, 5]) # print "Adding new partitions 7 and 8 with active state while compaction is running" common.set_partition_states(self._params, active = [6, 7]) info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") # print "Querying map view while compaction is running" (map_resp, map_view_result) = common.query(self._params, "mapview1", {"limit": "10"}) self.assertEqual(len(map_view_result["rows"]), 10, "Query returned 10 rows") common.test_keys_sorted(map_view_result) # print "Waiting for compaction to finish" compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0) self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered") # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") # print "Querying map view again" doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7]) (map_resp, map_view_result) = common.query(self._params, "mapview1") self.assertEqual(map_view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count) self.assertEqual(len(map_view_result["rows"]), doc_count, "Query returned %d rows" % doc_count) common.test_keys_sorted(map_view_result) # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3, 4, 5, 6, 7]: self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i) expected_seq = common.partition_update_seq(self._params, i) self.assertEqual(info["update_seqs"][str(i)], expected_seq, "info.update_seqs[%d] is %d" % (i, expected_seq))
def do_test_with_updates(self, state_transitions): # start from a clean state always common.set_partition_states(params=self._params, cleanup=range(self._params["nparts"])) # print "Cleaning set view via compaction" common.compact_set_view(self._params) (resp, view_result) = common.query(self._params, "mapview") self.assertEqual(view_result["total_rows"], 0, "total_rows is 0 after compaction") self.assertEqual(len(view_result["rows"]), 0, "0 rows returned after compaction") info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") self.assertEqual(info["update_seqs"], {}, "right update seqs list") # print "Triggering state changes with queries (?stale=updater_after) in between" for state in state_transitions: common.set_partition_states(params=self._params, active=state["active"], passive=state["passive"], cleanup=state["cleanup"]) time.sleep(0.5) common.query(self._params, "mapview", { "stale": "update_after", "limit": "100" }) group_stats = self.verify_final_index_state(state_transitions) self.assertTrue(group_stats["stopped_updates"] > 0, "Got at least 1 updater interruption") self.assertEqual(group_stats["full_updates"], 2, "Got 2 full updates") self.assertEqual(group_stats["compactions"], 1, "Got 1 compaction") self.assertTrue(group_stats["cleanups"] >= 1, "Got at least 1 full cleanup")
def test_compaction(self): # print "Querying map view" (map_resp, map_view_result) = common.query(self._params, "mapview1") map_etag = map_resp.getheader("ETag") self.assertEqual(map_view_result["total_rows"], self._params["ndocs"], "Query returned %d total_rows" % self._params["ndocs"]) self.assertEqual(len(map_view_result["rows"]), self._params["ndocs"], "Query returned %d rows" % self._params["ndocs"]) common.test_keys_sorted(map_view_result) # print "Verifying set view group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) # print "Querying reduce view" (red_resp, red_view_result) = common.query(self._params, "redview1") red_etag = red_resp.getheader("ETag") self.assertEqual(len(red_view_result["rows"]), 1, "Query returned 1 row") self.assertEqual(red_view_result["rows"][0]["value"], self._params["ndocs"], "Non-grouped reduce value is %d" % self._params["ndocs"]) # print "Triggering view compaction" common.compact_set_view(self._params) # print "Verifying set view group info" info2 = common.get_set_view_info(self._params) self.assertEqual(info2["active_partitions"], [0, 1, 2, 3], "right active partitions list") self.assertEqual(info2["passive_partitions"], [], "right passive partitions list") self.assertEqual(info2["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3]: self.assertEqual(info2["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertTrue(info2["disk_size"] < info["disk_size"], "Smaller disk_size after compaction") # print "Querying map view" (map_resp2, map_view_result2) = common.query(self._params, "mapview1") map_etag2 = map_resp2.getheader("ETag") self.assertEqual(map_view_result2, map_view_result, "Got same map response after compaction") self.assertEqual(map_etag2, map_etag, "Same map view etag after compaction") # print "Querying reduce view" (red_resp2, red_view_result2) = common.query(self._params, "redview1") red_etag2 = red_resp2.getheader("ETag") self.assertEqual(red_view_result2, red_view_result, "Got same reduce response after compaction") self.assertEqual(red_etag2, red_etag, "Same reduce view etag after compaction") # print "Adding 2 new documents to partition 4" server = self._params["server"] db4 = server[self._params["setname"] + "/3"] new_doc1 = {"_id": "9999999999999", "integer": 999999999999999999, "string": "999999999"} new_doc2 = {"_id": "000", "integer": -1111, "string": "000"} db4.save(new_doc1) db4.save(new_doc2) # print "Triggering view compaction" common.compact_set_view(self._params, False) # print "Verifying set view group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertEqual(info["compact_running"], True, "Compaction is flagged as running in group info") # print "Triggering set view group index update" common.query(self._params, "redview1") # print "Waiting for set view compaction to finish" compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0) self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered") # print "Verifying set view group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3]: if i == 3: seq = (self._params["ndocs"] / 4) + 2 else: seq = (self._params["ndocs"] / 4) self.assertEqual(info["update_seqs"][str(i)], seq, "right update seq for partition %d" % (i + 1)) # print "Querying map view" (map_resp3, map_view_result3) = common.query(self._params, "mapview1") map_etag3 = map_resp3.getheader("ETag") self.assertEqual(map_view_result3["total_rows"], (self._params["ndocs"] + 2), "Query returned %d total_rows" % (self._params["ndocs"] + 2)) self.assertEqual(len(map_view_result3["rows"]), (self._params["ndocs"] + 2), "Query returned %d rows" % (self._params["ndocs"] + 2)) self.assertNotEqual(map_etag3, map_etag, "Different map view etag after recompaction") common.test_keys_sorted(map_view_result3) self.assertEqual(map_view_result3["rows"][0]["id"], new_doc2["_id"], "new_doc2 in map view") self.assertEqual(map_view_result3["rows"][-1]["id"], new_doc1["_id"], "new_doc1 in map view") # print "Querying reduce view" (red_resp3, red_view_result3) = common.query(self._params, "redview1") red_etag3 = red_resp3.getheader("ETag") self.assertEqual(red_view_result3["rows"][0]["value"], (self._params["ndocs"] + 2), "Non-grouped reduce value is %d" % (self._params["ndocs"] + 2)) self.assertNotEqual(red_etag3, red_etag, "Different reduce view etag after compaction") # print "Triggering another view compaction" common.compact_set_view(self._params, False) # print "Triggering partition 4 cleanup while compaction is ongoing" common.set_partition_states(self._params, cleanup = [3]) # print "Verifying set view group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [3], "right cleanup partitions list") for i in [0, 1, 2]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertFalse("3" in info["update_seqs"], "paritition 4 not in group info update_seqs") self.assertEqual(info["compact_running"], True, "Compaction is flagged as running in group info") self.assertEqual(info["cleanup_running"], True, "Cleanup is flagged as running in group info") compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0) self.assertTrue(compaction_was_running, "Compaction was running when the cleanup was triggered") # print "Verifying set view group info" info2 = common.get_set_view_info(self._params) self.assertEqual(info2["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info2["passive_partitions"], [], "right passive partitions list") self.assertEqual(info2["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2]: self.assertEqual(info2["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertFalse("3" in info["update_seqs"], "paritition 4 not in group info update_seqs") self.assertTrue(info2["disk_size"] < info["disk_size"], "Smaller disk_size after compaction+cleanup") # print "Querying map view" (map_resp4, map_view_result4) = common.query(self._params, "mapview1") map_etag4 = map_resp4.getheader("ETag") expected = self._params["ndocs"] - (self._params["ndocs"] / 4) self.assertEqual(map_view_result4["total_rows"], expected, "Query returned %d total_rows after recompact+cleanup" % expected) self.assertEqual(len(map_view_result4["rows"]), expected, "Query returned %d rows after recompact+cleanup" % expected) self.assertNotEqual(map_etag4, map_etag, "Different map view etag after compaction+cleanup") common.test_keys_sorted(map_view_result4) all_keys = {} for r in map_view_result4["rows"]: all_keys[r["key"]] = True for key in xrange(4, self._params["ndocs"], self._params["nparts"]): self.assertFalse(key in all_keys, "Key %d not in result after partition 4 cleanup triggered" % key) self.assertFalse(new_doc1["integer"] in all_keys, "new_doc1 not included in view query result") self.assertFalse(new_doc2["integer"] in all_keys, "new_doc2 not included in view query result") self.assertNotEqual(map_view_result4, map_view_result3, "Different map view result after recompact+cleanup") # print "Querying reduce view" (red_resp4, red_view_result4) = common.query(self._params, "redview1") red_etag4 = red_resp4.getheader("ETag") expected = self._params["ndocs"] - (self._params["ndocs"] / 4) self.assertEqual(red_view_result4["rows"][0]["value"], expected, "Non-grouped reduce value is %d after recompact+cleanup" % (self._params["ndocs"] + 2)) self.assertNotEqual(red_etag4, red_etag3, "Different reduce view etag after compaction+cleanup")
def test_many_partitions(self): total_doc_count = common.set_doc_count(self._params, range(self._params["nparts"])) # print "Querying view" (resp, view_result) = common.query(self._params, "mapview") self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count) self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count) common.test_keys_sorted(view_result) # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], range(self._params["nparts"]), "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in xrange(self._params["nparts"]): expected_seq = common.partition_update_seq(self._params, i) self.assertEqual( info["update_seqs"][str(i)], expected_seq, "right update seq number (%d) for partition %d" % (expected_seq, i + 1)) # print "Adding 1 new document to each partition" new_docs = [] for i in xrange(self._params["nparts"]): server = self._params["server"] db = self._params["server"][self._params["setname"] + "/" + str(i)] value = total_doc_count + i + 1 new_doc = { "_id": str(value), "integer": value, "string": str(value) } new_docs.append(new_doc) db.save(new_doc) new_total_doc_count = common.set_doc_count( self._params, range(self._params["nparts"])) self.assertEqual(new_total_doc_count, (total_doc_count + len(new_docs)), "N documents were added") total_doc_count = new_total_doc_count # print "Querying view again" (resp, view_result) = common.query(self._params, "mapview") self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count) self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count) common.test_keys_sorted(view_result) # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], range(self._params["nparts"]), "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in xrange(self._params["nparts"]): expected_seq = common.partition_update_seq(self._params, i) self.assertEqual( info["update_seqs"][str(i)], expected_seq, "right update seq number (%d) for partition %d" % (expected_seq, i + 1)) # print "Marking half of the partitions as passive" passive = range(self._params["nparts"] / 2, self._params["nparts"]) active = range(self._params["nparts"] / 2) common.set_partition_states(self._params, passive=range(self._params["nparts"] / 2, self._params["nparts"])) # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], active, "right active partitions list") self.assertEqual(info["passive_partitions"], passive, "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in xrange(self._params["nparts"]): expected_seq = common.partition_update_seq(self._params, i) self.assertEqual( info["update_seqs"][str(i)], expected_seq, "right update seq number (%d) for partition %d" % (expected_seq, i + 1)) # print "Querying view again" (resp, view_result) = common.query(self._params, "mapview") expected_row_count = common.set_doc_count(self._params, active) self.assertEqual(len(view_result["rows"]), expected_row_count, "number of received rows is %d" % expected_row_count) common.test_keys_sorted(view_result) for row in view_result['rows']: if row["key"] >= 2001: key_part = ((row["key"] - 2000) % self._params["nparts"]) - 1 else: key_part = (row["key"] % self._params["nparts"]) - 1 self.assertTrue( key_part in active, "Key %d from passive partition not in result set" % row["key"]) # print "Marking half of the partitions for cleanup" common.set_partition_states(self._params, cleanup=passive) cleanup = passive common.compact_set_view(self._params) # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], active, "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") self.assertEqual(info["stats"]["compactions"], 1, "1 compaction") self.assertEqual(info["stats"]["cleanups"], 1, "1 full cleanup") for i in active: expected_seq = common.partition_update_seq(self._params, i) self.assertEqual( info["update_seqs"][str(i)], expected_seq, "right update seq number (%d) for partition %d" % (expected_seq, i + 1))
def test_filter_partitions(self): # print "Querying map view with ?partitions=[0,1,3]" (map_resp, map_view_result) = common.query( self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])}) map_etag = map_resp.getheader("ETag") expected = self._params["ndocs"] - (self._params["ndocs"] / 4) self.assertEqual(map_view_result["total_rows"], self._params["ndocs"], "Query returned %d total_rows" % self._params["ndocs"]) self.assertEqual(len(map_view_result["rows"]), expected, "Query returned %d rows" % expected) common.test_keys_sorted(map_view_result) all_keys = {} for r in map_view_result["rows"]: all_keys[r["key"]] = True for key in xrange(3, self._params["ndocs"], self._params["nparts"]): self.assertFalse(key in all_keys, "Key %d from partition 3 is not in the result" % key) # print "Disabling (making it passive) partition 2" common.set_partition_states(self._params, passive = [1]) # print "Querying map view again with ?partitions=[0,1,3]" (map_resp2, map_view_result2) = common.query( self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])}) map_etag2 = map_resp2.getheader("ETag") self.assertEqual(map_view_result2, map_view_result, "Same result as before") self.assertEqual(map_etag2, map_etag, "Same Etag as before") # print "Marking partition 2 for cleanup" common.set_partition_states(self._params, cleanup = [1]) # print "Querying map view again with ?partitions=[0,1,3]" (map_resp3, map_view_result3) = common.query( self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])}) map_etag3 = map_resp3.getheader("ETag") expected = self._params["ndocs"] / 2 self.assertEqual(len(map_view_result3["rows"]), expected, "Query returned %d rows" % expected) self.assertNotEqual(map_etag3, map_etag2, "Different Etag from before") common.test_keys_sorted(map_view_result3) all_keys = {} for r in map_view_result3["rows"]: all_keys[r["key"]] = True for key in xrange(2, self._params["ndocs"], self._params["nparts"]): self.assertFalse(key in all_keys, "Key %d from partition 2 is not in the result" % key) for key in xrange(3, self._params["ndocs"], self._params["nparts"]): self.assertFalse(key in all_keys, "Key %d from partition 3 is not in the result" % key) # print "Triggering view compaction (to guarantee cleanup is complete)" common.compact_set_view(self._params) # print "Querying map view again with ?partitions=[0,1,3]" (map_resp4, map_view_result4) = common.query( self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])}) map_etag4 = map_resp4.getheader("ETag") # total_rows is different after cleanup self.assertEquals(map_view_result4["rows"], map_view_result3["rows"], "Same result as before") self.assertEquals(map_etag4, map_etag3, "Same Etag as before")
def do_test_simple_cleanup(self): # print "Querying view in steady state" (resp, view_result) = common.query(self._params, "mapview1") etag = resp.getheader("ETag") self.assertEqual(view_result["total_rows"], self._params["ndocs"], "Query returned %d total_rows" % self._params["ndocs"]) self.assertEqual(len(view_result["rows"]), self._params["ndocs"], "Query returned %d rows" % self._params["ndocs"]) common.test_keys_sorted(view_result) info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) # print "Triggering partition 4 cleanup" common.set_partition_states(self._params, cleanup = [3]) info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [3], "right cleanup partitions list") for i in [0, 1, 2]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs") self.assertEqual(info["cleanup_running"], True, "cleanup process is running") # print "Querying view again" (resp2, view_result2) = common.query(self._params, "mapview1") etag2 = resp2.getheader("ETag") expected = self._params["ndocs"] - (self._params["ndocs"] / 4) self.assertEqual(view_result2["total_rows"], self._params["ndocs"], "Query returned %d total_rows" % self._params["ndocs"]) self.assertEqual(len(view_result2["rows"]), expected, "Query returned %d rows" % expected) self.assertNotEqual(etag2, etag, "Different Etag after cleanup triggered") common.test_keys_sorted(view_result2) all_keys = {} for r in view_result2["rows"]: all_keys[r["key"]] = True for key in xrange(4, self._params["ndocs"], self._params["nparts"]): self.assertFalse(key in all_keys, "Key %d not in result after partition 4 cleanup triggered" % key) # print "Triggering view compaction and querying view again" common.compact_set_view(self._params) (resp3, view_result3) = common.query(self._params, "mapview1") etag3 = resp3.getheader("ETag") expected = self._params["ndocs"] - (self._params["ndocs"] / 4) self.assertEqual(view_result3["total_rows"], expected, "Query returned %d total_rows" % expected) self.assertEqual(len(view_result3["rows"]), expected, "Query returned %d rows" % expected) self.assertEqual(etag2, etag3, "Same Etag after cleanup finished") common.test_keys_sorted(view_result3) all_keys = {} for r in view_result3["rows"]: all_keys[r["key"]] = True for key in xrange(4, self._params["ndocs"], self._params["nparts"]): self.assertFalse(key in all_keys, "Key %d not in result after partition 4 cleanup finished" % key) info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs") # print "Adding 2 new documents to partition 4" server = self._params["server"] db4 = server[self._params["setname"] + "/3"] new_doc1 = {"_id": "999999999", "integer": 999999999, "string": "999999999"} new_doc2 = {"_id": "000", "integer": -1111, "string": "000"} db4.save(new_doc1) db4.save(new_doc2) # print "Querying view again" (resp4, view_result4) = common.query(self._params, "mapview1") etag4 = resp4.getheader("ETag") expected = self._params["ndocs"] - (self._params["ndocs"] / 4) self.assertEqual(view_result4["total_rows"], expected, "Query returned %d total_rows" % expected) self.assertEqual(len(view_result4["rows"]), expected, "Query returned %d rows" % expected) self.assertEqual(etag4, etag3, "Same etag after adding new documents to cleaned partition") common.test_keys_sorted(view_result4) all_keys = {} for r in view_result4["rows"]: all_keys[r["key"]] = True for key in xrange(4, self._params["ndocs"], self._params["nparts"]): self.assertFalse(key in all_keys, "Key %d not in result after partition 4 cleanup finished" % key) self.assertFalse(new_doc1["integer"] in all_keys, "new_doc1 not in query result after cleanup") self.assertFalse(new_doc2["integer"] in all_keys, "new_doc2 not in query result after cleanup") info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs") # print "Triggering compaction again and verifying it doesn't crash" common.compact_set_view(self._params) (resp5, view_result5) = common.query(self._params, "mapview1") etag5 = resp5.getheader("ETag") self.assertEqual(etag5, etag4, "Same etag after second compaction") self.assertEqual(view_result5, view_result4, "Same query results after second compaction") info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs")
def do_test_set_cleanup_partitions_when_updater_is_running(self): # print "Marking all partitions for cleanup" common.set_partition_states(self._params, cleanup = range(self._params["nparts"])) # print "Compacting the set view group" common.compact_set_view(self._params) info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") # print "Querying view" (resp, view_result) = common.query(self._params, "mapview1") self.assertEqual(view_result["total_rows"], 0, "Empty view result") self.assertEqual(len(view_result["rows"]), 0, "Empty view result") # print "Marking all partitions as active" common.set_partition_states(self._params, active = range(self._params["nparts"])) # print "Querying view with ?stale=update_after" (resp, view_result) = common.query(self._params, "mapview1", {"stale": "update_after"}) self.assertEqual(view_result["total_rows"], 0, "Empty view result") self.assertEqual(len(view_result["rows"]), 0, "Empty view result") # print "Marking partition 2 for cleanup while the updater is running" common.set_partition_states(self._params, cleanup = [1]) info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [1], "right cleanup partitions list") self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs") self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs") # print "Waiting for the set view updater to finish" iterations = 0 while True: info = common.get_set_view_info(self._params) if info["updater_running"]: iterations += 1 else: break self.assertTrue(iterations > 0, "Updater was running when partition 2 was marked for cleanup") # print "Verifying set view group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertTrue(["cleanup_partitions"] == [1] or info["cleanup_partitions"] == [], "cleanup partitions list is not wrong") self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs") self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs") # print "Querying view" (resp, view_result) = common.query(self._params, "mapview1") doc_count = common.set_doc_count(self._params, [0, 2, 3]) self.assertEqual(len(view_result["rows"]), doc_count, "Query returned %d rows" % doc_count) common.test_keys_sorted(view_result) all_keys = {} for r in view_result["rows"]: all_keys[r["key"]] = True for key in xrange(2, self._params["ndocs"], self._params["nparts"]): self.assertFalse(key in all_keys, "Key %d not in result after partition 2 marked for cleanup" % key) # print "Verifying set view group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs") self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs")
def do_test_set_passive_during_compaction(self): # print "Querying map view" (map_resp, map_view_result) = common.query(self._params, "mapview1") doc_count = common.set_doc_count(self._params, [0, 1, 2, 3]) self.assertEqual(map_view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count) self.assertEqual(len(map_view_result["rows"]), doc_count, "Query returned %d rows" % doc_count) common.test_keys_sorted(map_view_result) # print "Verifying set view group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3]: db = self._params["server"][self._params["setname"] + "/" + str(i)] seq = db.info()["update_seq"] self.assertEqual(info["update_seqs"][str(i)], seq, "right update seq for partition %d" % (i + 1)) # print "Triggering view compaction" common.compact_set_view(self._params, False) # print "Marking partition 4 as passive" common.set_partition_states(self._params, passive = [3]) info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [3], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") # print "Waiting for compaction to finish" count = 0 while info["stats"]["compactions"] < 1: time.sleep(0.5) count += 1 info = common.get_set_view_info(self._params) self.assertTrue((count > 0), "Compaction was running when the partition states were updated") # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [3], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") # print "Querying map view again" (map_resp, map_view_result) = common.query(self._params, "mapview1") expected = common.set_doc_count(self._params, [0, 1, 2]) self.assertEqual(map_view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count) self.assertEqual(len(map_view_result["rows"]), expected, "Query returned %d rows" % expected) common.test_keys_sorted(map_view_result) all_keys = {} for r in map_view_result["rows"]: all_keys[r["key"]] = True for key in xrange(4, self._params["ndocs"], self._params["nparts"]): self.assertFalse(key in all_keys, "Key %d not in result after partition 4 set to passive" % key) # print "Triggering view compaction" common.compact_set_view(self._params, False) # print "Adding two new partitions, 5 and 6, as passive while compaction is running" common.set_partition_states(self._params, passive = [4, 5]) # print "Waiting for compaction to finish" info = common.get_set_view_info(self._params) count = 0 while info["stats"]["compactions"] < 2: time.sleep(0.5) count += 1 info = common.get_set_view_info(self._params) self.assertTrue((count > 0), "Compaction was running when the partition states were updated") # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [3, 4, 5], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3, 4, 5]: self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i) for i in [6, 7]: self.assertFalse(str(i) in info["update_seqs"], "%d not in info.update_seqs" % i) # print "Querying map view again" (map_resp, map_view_result2) = common.query(self._params, "mapview1") self.assertEqual(map_view_result2["rows"], map_view_result["rows"], "Same result set as before") total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7]) # print "Adding 1 new document to each partition" new_docs = [] for i in [0, 1, 2, 3, 4, 5, 6, 7]: server = self._params["server"] db = self._params["server"][self._params["setname"] + "/" + str(i)] value = total_doc_count + i + 1 new_doc = { "_id": str(value), "integer": value, "string": str(value) } new_docs.append(new_doc) db.save(new_doc) new_total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7]) self.assertEqual(new_total_doc_count, (total_doc_count + 8), "8 documents added") self.assertEqual(len(new_docs), 8, "8 documents added") info = common.get_set_view_info(self._params) if info["updater_running"]: # print "Waiting for updater to finish" self.assertEqual(info["updater_state"], "updating_passive", "updater state is updating_passive") while True: info = common.get_set_view_info(self._params) if info["updater_running"]: self.assertEqual(info["updater_state"], "updating_passive", "updater state is updating_passive") time.sleep(3) else: break expected_row_count = common.set_doc_count(self._params, [0, 1, 2]) expected_total_rows = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5]) # print "Querying map view again" (map_resp, map_view_result) = common.query(self._params, "mapview1") self.assertEqual(len(map_view_result["rows"]), expected_row_count, "len(rows) is %d" % expected_row_count) common.test_keys_sorted(map_view_result) # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [3, 4, 5], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3, 4, 5]: self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i) expected_seq = common.partition_update_seq(self._params, i) self.assertEqual(info["update_seqs"][str(i)], expected_seq, "info.update_seqs[%d] is %d" % (i, expected_seq))
def test_filter_partitions(self): # print "Querying map view with ?partitions=[0,1,3]" (map_resp, map_view_result) = common.query(self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])}) map_etag = map_resp.getheader("ETag") expected = self._params["ndocs"] - (self._params["ndocs"] / 4) self.assertEqual( map_view_result["total_rows"], self._params["ndocs"], "Query returned %d total_rows" % self._params["ndocs"]) self.assertEqual(len(map_view_result["rows"]), expected, "Query returned %d rows" % expected) common.test_keys_sorted(map_view_result) all_keys = {} for r in map_view_result["rows"]: all_keys[r["key"]] = True for key in xrange(3, self._params["ndocs"], self._params["nparts"]): self.assertFalse( key in all_keys, "Key %d from partition 3 is not in the result" % key) # print "Disabling (making it passive) partition 2" common.set_partition_states(self._params, passive=[1]) # print "Querying map view again with ?partitions=[0,1,3]" (map_resp2, map_view_result2) = common.query( self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])}) map_etag2 = map_resp2.getheader("ETag") self.assertEqual(map_view_result2, map_view_result, "Same result as before") self.assertEqual(map_etag2, map_etag, "Same Etag as before") # print "Marking partition 2 for cleanup" common.set_partition_states(self._params, cleanup=[1]) # print "Querying map view again with ?partitions=[0,1,3]" (map_resp3, map_view_result3) = common.query( self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])}) map_etag3 = map_resp3.getheader("ETag") expected = self._params["ndocs"] / 2 self.assertEqual(len(map_view_result3["rows"]), expected, "Query returned %d rows" % expected) self.assertNotEqual(map_etag3, map_etag2, "Different Etag from before") common.test_keys_sorted(map_view_result3) all_keys = {} for r in map_view_result3["rows"]: all_keys[r["key"]] = True for key in xrange(2, self._params["ndocs"], self._params["nparts"]): self.assertFalse( key in all_keys, "Key %d from partition 2 is not in the result" % key) for key in xrange(3, self._params["ndocs"], self._params["nparts"]): self.assertFalse( key in all_keys, "Key %d from partition 3 is not in the result" % key) # print "Triggering view compaction (to guarantee cleanup is complete)" common.compact_set_view(self._params) # print "Querying map view again with ?partitions=[0,1,3]" (map_resp4, map_view_result4) = common.query( self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])}) map_etag4 = map_resp4.getheader("ETag") # total_rows is different after cleanup self.assertEquals(map_view_result4["rows"], map_view_result3["rows"], "Same result as before") self.assertEquals(map_etag4, map_etag3, "Same Etag as before")
def do_test_simple_cleanup(self): # print "Querying view in steady state" (resp, view_result) = common.query(self._params, "mapview1") etag = resp.getheader("ETag") self.assertEqual( view_result["total_rows"], self._params["ndocs"], "Query returned %d total_rows" % self._params["ndocs"]) self.assertEqual(len(view_result["rows"]), self._params["ndocs"], "Query returned %d rows" % self._params["ndocs"]) common.test_keys_sorted(view_result) info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) # print "Triggering partition 4 cleanup" common.set_partition_states(self._params, cleanup=[3]) info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [3], "right cleanup partitions list") for i in [0, 1, 2]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs") self.assertEqual(info["cleanup_running"], True, "cleanup process is running") # print "Querying view again" (resp2, view_result2) = common.query(self._params, "mapview1") etag2 = resp2.getheader("ETag") expected = self._params["ndocs"] - (self._params["ndocs"] / 4) self.assertEqual( view_result2["total_rows"], self._params["ndocs"], "Query returned %d total_rows" % self._params["ndocs"]) self.assertEqual(len(view_result2["rows"]), expected, "Query returned %d rows" % expected) self.assertNotEqual(etag2, etag, "Different Etag after cleanup triggered") common.test_keys_sorted(view_result2) all_keys = {} for r in view_result2["rows"]: all_keys[r["key"]] = True for key in xrange(4, self._params["ndocs"], self._params["nparts"]): self.assertFalse( key in all_keys, "Key %d not in result after partition 4 cleanup triggered" % key) # print "Triggering view compaction and querying view again" common.compact_set_view(self._params) (resp3, view_result3) = common.query(self._params, "mapview1") etag3 = resp3.getheader("ETag") expected = self._params["ndocs"] - (self._params["ndocs"] / 4) self.assertEqual(view_result3["total_rows"], expected, "Query returned %d total_rows" % expected) self.assertEqual(len(view_result3["rows"]), expected, "Query returned %d rows" % expected) self.assertEqual(etag2, etag3, "Same Etag after cleanup finished") common.test_keys_sorted(view_result3) all_keys = {} for r in view_result3["rows"]: all_keys[r["key"]] = True for key in xrange(4, self._params["ndocs"], self._params["nparts"]): self.assertFalse( key in all_keys, "Key %d not in result after partition 4 cleanup finished" % key) info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs") # print "Adding 2 new documents to partition 4" server = self._params["server"] db4 = server[self._params["setname"] + "/3"] new_doc1 = { "_id": "999999999", "integer": 999999999, "string": "999999999" } new_doc2 = {"_id": "000", "integer": -1111, "string": "000"} db4.save(new_doc1) db4.save(new_doc2) # print "Querying view again" (resp4, view_result4) = common.query(self._params, "mapview1") etag4 = resp4.getheader("ETag") expected = self._params["ndocs"] - (self._params["ndocs"] / 4) self.assertEqual(view_result4["total_rows"], expected, "Query returned %d total_rows" % expected) self.assertEqual(len(view_result4["rows"]), expected, "Query returned %d rows" % expected) self.assertEqual( etag4, etag3, "Same etag after adding new documents to cleaned partition") common.test_keys_sorted(view_result4) all_keys = {} for r in view_result4["rows"]: all_keys[r["key"]] = True for key in xrange(4, self._params["ndocs"], self._params["nparts"]): self.assertFalse( key in all_keys, "Key %d not in result after partition 4 cleanup finished" % key) self.assertFalse(new_doc1["integer"] in all_keys, "new_doc1 not in query result after cleanup") self.assertFalse(new_doc2["integer"] in all_keys, "new_doc2 not in query result after cleanup") info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs") # print "Triggering compaction again and verifying it doesn't crash" common.compact_set_view(self._params) (resp5, view_result5) = common.query(self._params, "mapview1") etag5 = resp5.getheader("ETag") self.assertEqual(etag5, etag4, "Same etag after second compaction") self.assertEqual(view_result5, view_result4, "Same query results after second compaction") info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs")
def do_test_set_cleanup_partitions_when_updater_is_running(self): # print "Marking all partitions for cleanup" common.set_partition_states(self._params, cleanup=range(self._params["nparts"])) # print "Compacting the set view group" common.compact_set_view(self._params) info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") # print "Querying view" (resp, view_result) = common.query(self._params, "mapview1") self.assertEqual(view_result["total_rows"], 0, "Empty view result") self.assertEqual(len(view_result["rows"]), 0, "Empty view result") # print "Marking all partitions as active" common.set_partition_states(self._params, active=range(self._params["nparts"])) # print "Querying view with ?stale=update_after" (resp, view_result) = common.query(self._params, "mapview1", {"stale": "update_after"}) self.assertEqual(view_result["total_rows"], 0, "Empty view result") self.assertEqual(len(view_result["rows"]), 0, "Empty view result") # print "Marking partition 2 for cleanup while the updater is running" common.set_partition_states(self._params, cleanup=[1]) info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [1], "right cleanup partitions list") self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs") self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs") # print "Waiting for the set view updater to finish" iterations = 0 while True: info = common.get_set_view_info(self._params) if info["updater_running"]: iterations += 1 else: break self.assertTrue( iterations > 0, "Updater was running when partition 2 was marked for cleanup") # print "Verifying set view group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertTrue(["cleanup_partitions"] == [1] or info["cleanup_partitions"] == [], "cleanup partitions list is not wrong") self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs") self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs") # print "Querying view" (resp, view_result) = common.query(self._params, "mapview1") doc_count = common.set_doc_count(self._params, [0, 2, 3]) self.assertEqual(len(view_result["rows"]), doc_count, "Query returned %d rows" % doc_count) common.test_keys_sorted(view_result) all_keys = {} for r in view_result["rows"]: all_keys[r["key"]] = True for key in xrange(2, self._params["ndocs"], self._params["nparts"]): self.assertFalse( key in all_keys, "Key %d not in result after partition 2 marked for cleanup" % key) # print "Verifying set view group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs") self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs")
def test_many_partitions(self): total_doc_count = common.set_doc_count(self._params, range(self._params["nparts"])) # print "Querying view" (resp, view_result) = common.query(self._params, "mapview") self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count) self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count) common.test_keys_sorted(view_result) # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], range(self._params["nparts"]), "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in xrange(self._params["nparts"]): expected_seq = common.partition_update_seq(self._params, i) self.assertEqual(info["update_seqs"][str(i)], expected_seq, "right update seq number (%d) for partition %d" % (expected_seq, i + 1)) # print "Adding 1 new document to each partition" new_docs = [] for i in xrange(self._params["nparts"]): server = self._params["server"] db = self._params["server"][self._params["setname"] + "/" + str(i)] value = total_doc_count + i + 1 new_doc = { "_id": str(value), "integer": value, "string": str(value) } new_docs.append(new_doc) db.save(new_doc) new_total_doc_count = common.set_doc_count(self._params, range(self._params["nparts"])) self.assertEqual(new_total_doc_count, (total_doc_count + len(new_docs)), "N documents were added") total_doc_count = new_total_doc_count # print "Querying view again" (resp, view_result) = common.query(self._params, "mapview") self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count) self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count) common.test_keys_sorted(view_result) # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], range(self._params["nparts"]), "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in xrange(self._params["nparts"]): expected_seq = common.partition_update_seq(self._params, i) self.assertEqual(info["update_seqs"][str(i)], expected_seq, "right update seq number (%d) for partition %d" % (expected_seq, i + 1)) # print "Marking half of the partitions as passive" passive = range(self._params["nparts"] / 2, self._params["nparts"]) active = range(self._params["nparts"] / 2) common.set_partition_states( self._params, passive = range(self._params["nparts"] / 2, self._params["nparts"]) ) # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], active, "right active partitions list") self.assertEqual(info["passive_partitions"], passive, "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in xrange(self._params["nparts"]): expected_seq = common.partition_update_seq(self._params, i) self.assertEqual(info["update_seqs"][str(i)], expected_seq, "right update seq number (%d) for partition %d" % (expected_seq, i + 1)) # print "Querying view again" (resp, view_result) = common.query(self._params, "mapview") expected_row_count = common.set_doc_count(self._params, active) self.assertEqual(len(view_result["rows"]), expected_row_count, "number of received rows is %d" % expected_row_count) common.test_keys_sorted(view_result) for row in view_result['rows']: if row["key"] >= 2001: key_part = ((row["key"] - 2000) % self._params["nparts"]) - 1 else: key_part = (row["key"] % self._params["nparts"]) - 1 self.assertTrue(key_part in active, "Key %d from passive partition not in result set" % row["key"]) # print "Marking half of the partitions for cleanup" common.set_partition_states( self._params, cleanup = passive ) cleanup = passive common.compact_set_view(self._params) # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], active, "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") self.assertEqual(info["stats"]["compactions"], 1, "1 compaction") self.assertEqual(info["stats"]["cleanups"], 1, "1 full cleanup") for i in active: expected_seq = common.partition_update_seq(self._params, i) self.assertEqual(info["update_seqs"][str(i)], expected_seq, "right update seq number (%d) for partition %d" % (expected_seq, i + 1))