def do_test_set_active_during_compaction(self):
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Marking partitions 4, 5 and 6 as active while compaction is running"
common.set_partition_states(self._params, active = [3, 4, 5])
# print "Adding new partitions 7 and 8 with active state while compaction is running"
common.set_partition_states(self._params, active = [6, 7])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying map view while compaction is running"
(map_resp, map_view_result) = common.query(self._params, "mapview1", {"limit": "10"})
self.assertEqual(len(map_view_result["rows"]), 10, "Query returned 10 rows")
common.test_keys_sorted(map_view_result)
# print "Waiting for compaction to finish"
compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0)
self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying map view again"
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
(map_resp, map_view_result) = common.query(self._params, "mapview1")
self.assertEqual(map_view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(map_view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(map_view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5, 6, 7]:
self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i)
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"info.update_seqs[%d] is %d" % (i, expected_seq))
def test_compaction(self):
# print "Querying map view"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
map_etag = map_resp.getheader("ETag")
self.assertEqual(map_view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(map_view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(map_view_result)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying reduce view"
(red_resp, red_view_result) = common.query(self._params, "redview1")
red_etag = red_resp.getheader("ETag")
self.assertEqual(len(red_view_result["rows"]), 1, "Query returned 1 row")
self.assertEqual(red_view_result["rows"][0]["value"], self._params["ndocs"],
"Non-grouped reduce value is %d" % self._params["ndocs"])
# print "Triggering view compaction"
common.compact_set_view(self._params)
# print "Verifying set view group info"
info2 = common.get_set_view_info(self._params)
self.assertEqual(info2["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info2["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info2["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info2["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertTrue(info2["disk_size"] < info["disk_size"], "Smaller disk_size after compaction")
# print "Querying map view"
(map_resp2, map_view_result2) = common.query(self._params, "mapview1")
map_etag2 = map_resp2.getheader("ETag")
self.assertEqual(map_view_result2, map_view_result, "Got same map response after compaction")
self.assertEqual(map_etag2, map_etag, "Same map view etag after compaction")
# print "Querying reduce view"
(red_resp2, red_view_result2) = common.query(self._params, "redview1")
red_etag2 = red_resp2.getheader("ETag")
self.assertEqual(red_view_result2, red_view_result, "Got same reduce response after compaction")
self.assertEqual(red_etag2, red_etag, "Same reduce view etag after compaction")
# print "Adding 2 new documents to partition 4"
server = self._params["server"]
db4 = server[self._params["setname"] + "/3"]
new_doc1 = {"_id": "9999999999999", "integer": 999999999999999999, "string": "999999999"}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db4.save(new_doc1)
db4.save(new_doc2)
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertEqual(info["compact_running"], True, "Compaction is flagged as running in group info")
# print "Triggering set view group index update"
common.query(self._params, "redview1")
# print "Waiting for set view compaction to finish"
compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0)
self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
if i == 3:
seq = (self._params["ndocs"] / 4) + 2
else:
seq = (self._params["ndocs"] / 4)
self.assertEqual(info["update_seqs"][str(i)], seq, "right update seq for partition %d" % (i + 1))
# print "Querying map view"
(map_resp3, map_view_result3) = common.query(self._params, "mapview1")
map_etag3 = map_resp3.getheader("ETag")
self.assertEqual(map_view_result3["total_rows"], (self._params["ndocs"] + 2),
"Query returned %d total_rows" % (self._params["ndocs"] + 2))
self.assertEqual(len(map_view_result3["rows"]), (self._params["ndocs"] + 2),
"Query returned %d rows" % (self._params["ndocs"] + 2))
self.assertNotEqual(map_etag3, map_etag, "Different map view etag after recompaction")
common.test_keys_sorted(map_view_result3)
self.assertEqual(map_view_result3["rows"][0]["id"], new_doc2["_id"], "new_doc2 in map view")
self.assertEqual(map_view_result3["rows"][-1]["id"], new_doc1["_id"], "new_doc1 in map view")
# print "Querying reduce view"
(red_resp3, red_view_result3) = common.query(self._params, "redview1")
red_etag3 = red_resp3.getheader("ETag")
self.assertEqual(red_view_result3["rows"][0]["value"], (self._params["ndocs"] + 2),
"Non-grouped reduce value is %d" % (self._params["ndocs"] + 2))
self.assertNotEqual(red_etag3, red_etag, "Different reduce view etag after compaction")
# print "Triggering another view compaction"
common.compact_set_view(self._params, False)
# print "Triggering partition 4 cleanup while compaction is ongoing"
common.set_partition_states(self._params, cleanup = [3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [3], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "paritition 4 not in group info update_seqs")
self.assertEqual(info["compact_running"], True, "Compaction is flagged as running in group info")
self.assertEqual(info["cleanup_running"], True, "Cleanup is flagged as running in group info")
compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0)
self.assertTrue(compaction_was_running, "Compaction was running when the cleanup was triggered")
# print "Verifying set view group info"
info2 = common.get_set_view_info(self._params)
self.assertEqual(info2["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info2["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info2["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info2["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "paritition 4 not in group info update_seqs")
self.assertTrue(info2["disk_size"] < info["disk_size"], "Smaller disk_size after compaction+cleanup")
# print "Querying map view"
(map_resp4, map_view_result4) = common.query(self._params, "mapview1")
map_etag4 = map_resp4.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(map_view_result4["total_rows"], expected,
"Query returned %d total_rows after recompact+cleanup" % expected)
self.assertEqual(len(map_view_result4["rows"]), expected,
"Query returned %d rows after recompact+cleanup" % expected)
self.assertNotEqual(map_etag4, map_etag, "Different map view etag after compaction+cleanup")
common.test_keys_sorted(map_view_result4)
all_keys = {}
for r in map_view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 cleanup triggered" % key)
self.assertFalse(new_doc1["integer"] in all_keys, "new_doc1 not included in view query result")
self.assertFalse(new_doc2["integer"] in all_keys, "new_doc2 not included in view query result")
self.assertNotEqual(map_view_result4, map_view_result3, "Different map view result after recompact+cleanup")
# print "Querying reduce view"
(red_resp4, red_view_result4) = common.query(self._params, "redview1")
red_etag4 = red_resp4.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(red_view_result4["rows"][0]["value"], expected,
"Non-grouped reduce value is %d after recompact+cleanup" % (self._params["ndocs"] + 2))
self.assertNotEqual(red_etag4, red_etag3, "Different reduce view etag after compaction+cleanup")
def do_test_set_passive_during_compaction(self):
# print "Querying map view"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
self.assertEqual(map_view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(map_view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(map_view_result)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
db = self._params["server"][self._params["setname"] + "/" + str(i)]
seq = db.info()["update_seq"]
self.assertEqual(info["update_seqs"][str(i)], seq,
"right update seq for partition %d" % (i + 1))
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Marking partition 4 as passive"
common.set_partition_states(self._params, passive = [3])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Waiting for compaction to finish"
compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0)
self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying map view again"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
expected = common.set_doc_count(self._params, [0, 1, 2])
self.assertEqual(map_view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(map_view_result["rows"]), expected,
"Query returned %d rows" % expected)
common.test_keys_sorted(map_view_result)
all_keys = {}
for r in map_view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 set to passive" % key)
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Adding two new partitions, 5 and 6, as passive while compaction is running"
common.set_partition_states(self._params, passive = [4, 5])
# print "Waiting for compaction to finish"
compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0)
self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3, 4, 5], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5]:
self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i)
for i in [6, 7]:
self.assertFalse(str(i) in info["update_seqs"], "%d not in info.update_seqs" % i)
# print "Querying map view again"
(map_resp, map_view_result2) = common.query(self._params, "mapview1")
self.assertEqual(map_view_result2["rows"], map_view_result["rows"], "Same result set as before")
total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
# print "Adding 1 new document to each partition"
new_docs = []
for i in [0, 1, 2, 3, 4, 5, 6, 7]:
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
self.assertEqual(new_total_doc_count, (total_doc_count + 8), "8 documents added")
self.assertEqual(len(new_docs), 8, "8 documents added")
info = common.get_set_view_info(self._params)
if info["updater_running"]:
# print "Waiting for updater to finish"
self.assertEqual(info["updater_state"], "updating_passive",
"updater state is updating_passive")
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
self.assertEqual(info["updater_state"], "updating_passive",
"updater state is updating_passive")
time.sleep(3)
else:
break
expected_row_count = common.set_doc_count(self._params, [0, 1, 2])
expected_total_rows = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5])
# print "Querying map view again"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
self.assertEqual(len(map_view_result["rows"]), expected_row_count, "len(rows) is %d" % expected_row_count)
common.test_keys_sorted(map_view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3, 4, 5], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5]:
self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i)
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"info.update_seqs[%d] is %d" % (i, expected_seq))
