def do_test_set_active_during_compaction(self): # print "Triggering view compaction" common.compact_set_view(self._params, False) # print "Marking partitions 4, 5 and 6 as active while compaction is running" common.set_partition_states(self._params, active = [3, 4, 5]) # print "Adding new partitions 7 and 8 with active state while compaction is running" common.set_partition_states(self._params, active = [6, 7]) info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") # print "Querying map view while compaction is running" (map_resp, map_view_result) = common.query(self._params, "mapview1", {"limit": "10"}) self.assertEqual(len(map_view_result["rows"]), 10, "Query returned 10 rows") common.test_keys_sorted(map_view_result) # print "Waiting for compaction to finish" compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0) self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered") # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") # print "Querying map view again" doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7]) (map_resp, map_view_result) = common.query(self._params, "mapview1") self.assertEqual(map_view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count) self.assertEqual(len(map_view_result["rows"]), doc_count, "Query returned %d rows" % doc_count) common.test_keys_sorted(map_view_result) # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3, 4, 5, 6, 7]: self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i) expected_seq = common.partition_update_seq(self._params, i) self.assertEqual(info["update_seqs"][str(i)], expected_seq, "info.update_seqs[%d] is %d" % (i, expected_seq))
def test_compaction(self): # print "Querying map view" (map_resp, map_view_result) = common.query(self._params, "mapview1") map_etag = map_resp.getheader("ETag") self.assertEqual(map_view_result["total_rows"], self._params["ndocs"], "Query returned %d total_rows" % self._params["ndocs"]) self.assertEqual(len(map_view_result["rows"]), self._params["ndocs"], "Query returned %d rows" % self._params["ndocs"]) common.test_keys_sorted(map_view_result) # print "Verifying set view group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) # print "Querying reduce view" (red_resp, red_view_result) = common.query(self._params, "redview1") red_etag = red_resp.getheader("ETag") self.assertEqual(len(red_view_result["rows"]), 1, "Query returned 1 row") self.assertEqual(red_view_result["rows"][0]["value"], self._params["ndocs"], "Non-grouped reduce value is %d" % self._params["ndocs"]) # print "Triggering view compaction" common.compact_set_view(self._params) # print "Verifying set view group info" info2 = common.get_set_view_info(self._params) self.assertEqual(info2["active_partitions"], [0, 1, 2, 3], "right active partitions list") self.assertEqual(info2["passive_partitions"], [], "right passive partitions list") self.assertEqual(info2["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3]: self.assertEqual(info2["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertTrue(info2["disk_size"] < info["disk_size"], "Smaller disk_size after compaction") # print "Querying map view" (map_resp2, map_view_result2) = common.query(self._params, "mapview1") map_etag2 = map_resp2.getheader("ETag") self.assertEqual(map_view_result2, map_view_result, "Got same map response after compaction") self.assertEqual(map_etag2, map_etag, "Same map view etag after compaction") # print "Querying reduce view" (red_resp2, red_view_result2) = common.query(self._params, "redview1") red_etag2 = red_resp2.getheader("ETag") self.assertEqual(red_view_result2, red_view_result, "Got same reduce response after compaction") self.assertEqual(red_etag2, red_etag, "Same reduce view etag after compaction") # print "Adding 2 new documents to partition 4" server = self._params["server"] db4 = server[self._params["setname"] + "/3"] new_doc1 = {"_id": "9999999999999", "integer": 999999999999999999, "string": "999999999"} new_doc2 = {"_id": "000", "integer": -1111, "string": "000"} db4.save(new_doc1) db4.save(new_doc2) # print "Triggering view compaction" common.compact_set_view(self._params, False) # print "Verifying set view group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertEqual(info["compact_running"], True, "Compaction is flagged as running in group info") # print "Triggering set view group index update" common.query(self._params, "redview1") # print "Waiting for set view compaction to finish" compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0) self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered") # print "Verifying set view group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3]: if i == 3: seq = (self._params["ndocs"] / 4) + 2 else: seq = (self._params["ndocs"] / 4) self.assertEqual(info["update_seqs"][str(i)], seq, "right update seq for partition %d" % (i + 1)) # print "Querying map view" (map_resp3, map_view_result3) = common.query(self._params, "mapview1") map_etag3 = map_resp3.getheader("ETag") self.assertEqual(map_view_result3["total_rows"], (self._params["ndocs"] + 2), "Query returned %d total_rows" % (self._params["ndocs"] + 2)) self.assertEqual(len(map_view_result3["rows"]), (self._params["ndocs"] + 2), "Query returned %d rows" % (self._params["ndocs"] + 2)) self.assertNotEqual(map_etag3, map_etag, "Different map view etag after recompaction") common.test_keys_sorted(map_view_result3) self.assertEqual(map_view_result3["rows"][0]["id"], new_doc2["_id"], "new_doc2 in map view") self.assertEqual(map_view_result3["rows"][-1]["id"], new_doc1["_id"], "new_doc1 in map view") # print "Querying reduce view" (red_resp3, red_view_result3) = common.query(self._params, "redview1") red_etag3 = red_resp3.getheader("ETag") self.assertEqual(red_view_result3["rows"][0]["value"], (self._params["ndocs"] + 2), "Non-grouped reduce value is %d" % (self._params["ndocs"] + 2)) self.assertNotEqual(red_etag3, red_etag, "Different reduce view etag after compaction") # print "Triggering another view compaction" common.compact_set_view(self._params, False) # print "Triggering partition 4 cleanup while compaction is ongoing" common.set_partition_states(self._params, cleanup = [3]) # print "Verifying set view group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [3], "right cleanup partitions list") for i in [0, 1, 2]: self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertFalse("3" in info["update_seqs"], "paritition 4 not in group info update_seqs") self.assertEqual(info["compact_running"], True, "Compaction is flagged as running in group info") self.assertEqual(info["cleanup_running"], True, "Cleanup is flagged as running in group info") compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0) self.assertTrue(compaction_was_running, "Compaction was running when the cleanup was triggered") # print "Verifying set view group info" info2 = common.get_set_view_info(self._params) self.assertEqual(info2["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info2["passive_partitions"], [], "right passive partitions list") self.assertEqual(info2["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2]: self.assertEqual(info2["update_seqs"][str(i)], (self._params["ndocs"] / 4), "right update seq for partition %d" % (i + 1)) self.assertFalse("3" in info["update_seqs"], "paritition 4 not in group info update_seqs") self.assertTrue(info2["disk_size"] < info["disk_size"], "Smaller disk_size after compaction+cleanup") # print "Querying map view" (map_resp4, map_view_result4) = common.query(self._params, "mapview1") map_etag4 = map_resp4.getheader("ETag") expected = self._params["ndocs"] - (self._params["ndocs"] / 4) self.assertEqual(map_view_result4["total_rows"], expected, "Query returned %d total_rows after recompact+cleanup" % expected) self.assertEqual(len(map_view_result4["rows"]), expected, "Query returned %d rows after recompact+cleanup" % expected) self.assertNotEqual(map_etag4, map_etag, "Different map view etag after compaction+cleanup") common.test_keys_sorted(map_view_result4) all_keys = {} for r in map_view_result4["rows"]: all_keys[r["key"]] = True for key in xrange(4, self._params["ndocs"], self._params["nparts"]): self.assertFalse(key in all_keys, "Key %d not in result after partition 4 cleanup triggered" % key) self.assertFalse(new_doc1["integer"] in all_keys, "new_doc1 not included in view query result") self.assertFalse(new_doc2["integer"] in all_keys, "new_doc2 not included in view query result") self.assertNotEqual(map_view_result4, map_view_result3, "Different map view result after recompact+cleanup") # print "Querying reduce view" (red_resp4, red_view_result4) = common.query(self._params, "redview1") red_etag4 = red_resp4.getheader("ETag") expected = self._params["ndocs"] - (self._params["ndocs"] / 4) self.assertEqual(red_view_result4["rows"][0]["value"], expected, "Non-grouped reduce value is %d after recompact+cleanup" % (self._params["ndocs"] + 2)) self.assertNotEqual(red_etag4, red_etag3, "Different reduce view etag after compaction+cleanup")
def do_test_set_passive_during_compaction(self): # print "Querying map view" (map_resp, map_view_result) = common.query(self._params, "mapview1") doc_count = common.set_doc_count(self._params, [0, 1, 2, 3]) self.assertEqual(map_view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count) self.assertEqual(len(map_view_result["rows"]), doc_count, "Query returned %d rows" % doc_count) common.test_keys_sorted(map_view_result) # print "Verifying set view group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list") self.assertEqual(info["passive_partitions"], [], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3]: db = self._params["server"][self._params["setname"] + "/" + str(i)] seq = db.info()["update_seq"] self.assertEqual(info["update_seqs"][str(i)], seq, "right update seq for partition %d" % (i + 1)) # print "Triggering view compaction" common.compact_set_view(self._params, False) # print "Marking partition 4 as passive" common.set_partition_states(self._params, passive = [3]) info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [3], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") # print "Waiting for compaction to finish" compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0) self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered") # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [3], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") # print "Querying map view again" (map_resp, map_view_result) = common.query(self._params, "mapview1") expected = common.set_doc_count(self._params, [0, 1, 2]) self.assertEqual(map_view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count) self.assertEqual(len(map_view_result["rows"]), expected, "Query returned %d rows" % expected) common.test_keys_sorted(map_view_result) all_keys = {} for r in map_view_result["rows"]: all_keys[r["key"]] = True for key in xrange(4, self._params["ndocs"], self._params["nparts"]): self.assertFalse(key in all_keys, "Key %d not in result after partition 4 set to passive" % key) # print "Triggering view compaction" common.compact_set_view(self._params, False) # print "Adding two new partitions, 5 and 6, as passive while compaction is running" common.set_partition_states(self._params, passive = [4, 5]) # print "Waiting for compaction to finish" compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0) self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered") # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [3, 4, 5], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3, 4, 5]: self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i) for i in [6, 7]: self.assertFalse(str(i) in info["update_seqs"], "%d not in info.update_seqs" % i) # print "Querying map view again" (map_resp, map_view_result2) = common.query(self._params, "mapview1") self.assertEqual(map_view_result2["rows"], map_view_result["rows"], "Same result set as before") total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7]) # print "Adding 1 new document to each partition" new_docs = [] for i in [0, 1, 2, 3, 4, 5, 6, 7]: server = self._params["server"] db = self._params["server"][self._params["setname"] + "/" + str(i)] value = total_doc_count + i + 1 new_doc = { "_id": str(value), "integer": value, "string": str(value) } new_docs.append(new_doc) db.save(new_doc) new_total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7]) self.assertEqual(new_total_doc_count, (total_doc_count + 8), "8 documents added") self.assertEqual(len(new_docs), 8, "8 documents added") info = common.get_set_view_info(self._params) if info["updater_running"]: # print "Waiting for updater to finish" self.assertEqual(info["updater_state"], "updating_passive", "updater state is updating_passive") while True: info = common.get_set_view_info(self._params) if info["updater_running"]: self.assertEqual(info["updater_state"], "updating_passive", "updater state is updating_passive") time.sleep(3) else: break expected_row_count = common.set_doc_count(self._params, [0, 1, 2]) expected_total_rows = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5]) # print "Querying map view again" (map_resp, map_view_result) = common.query(self._params, "mapview1") self.assertEqual(len(map_view_result["rows"]), expected_row_count, "len(rows) is %d" % expected_row_count) common.test_keys_sorted(map_view_result) # print "Verifying group info" info = common.get_set_view_info(self._params) self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list") self.assertEqual(info["passive_partitions"], [3, 4, 5], "right passive partitions list") self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list") for i in [0, 1, 2, 3, 4, 5]: self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i) expected_seq = common.partition_update_seq(self._params, i) self.assertEqual(info["update_seqs"][str(i)], expected_seq, "info.update_seqs[%d] is %d" % (i, expected_seq))