def do_test_with_updates(self, state_transitions):
# start from a clean state always
common.set_partition_states(
params = self._params,
cleanup = range(self._params["nparts"])
)
# print "Cleaning set view via compaction"
common.compact_set_view(self._params)
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(view_result["total_rows"], 0, "total_rows is 0 after compaction")
self.assertEqual(len(view_result["rows"]), 0, "0 rows returned after compaction")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
self.assertEqual(info["update_seqs"], {}, "right update seqs list")
# print "Triggering state changes with queries (?stale=updater_after) in between"
for state in state_transitions:
common.set_partition_states(
params = self._params,
active = state["active"],
passive = state["passive"],
cleanup = state["cleanup"]
)
time.sleep(0.5)
common.query(self._params, "mapview", {"stale": "update_after", "limit": "100"})
group_stats = self.verify_final_index_state(state_transitions)
self.assertTrue(group_stats["updater_interruptions"] > 0, "Got at least 1 updater interruption")
self.assertEqual(group_stats["updates"], 2, "Got 2 full updates")
self.assertEqual(group_stats["compactions"], 1, "Got 1 compaction")
self.assertTrue(group_stats["cleanups"] >= 1, "Got at least 1 full cleanup")
def do_test_set_active_during_compaction(self):
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Marking partitions 4, 5 and 6 as active while compaction is running"
common.set_partition_states(self._params, active = [3, 4, 5])
# print "Adding new partitions 7 and 8 with active state while compaction is running"
common.set_partition_states(self._params, active = [6, 7])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying map view while compaction is running"
(map_resp, map_view_result) = common.query(self._params, "mapview1", {"limit": "10"})
self.assertEqual(len(map_view_result["rows"]), 10, "Query returned 10 rows")
common.test_keys_sorted(map_view_result)
# print "Waiting for compaction to finish"
compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0)
self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying map view again"
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
(map_resp, map_view_result) = common.query(self._params, "mapview1")
self.assertEqual(map_view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(map_view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(map_view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5, 6, 7]:
self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i)
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"info.update_seqs[%d] is %d" % (i, expected_seq))
def do_test_with_updates(self, state_transitions):
# start from a clean state always
common.set_partition_states(params=self._params,
cleanup=range(self._params["nparts"]))
# print "Cleaning set view via compaction"
common.compact_set_view(self._params)
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(view_result["total_rows"], 0,
"total_rows is 0 after compaction")
self.assertEqual(len(view_result["rows"]), 0,
"0 rows returned after compaction")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
self.assertEqual(info["update_seqs"], {}, "right update seqs list")
# print "Triggering state changes with queries (?stale=updater_after) in between"
for state in state_transitions:
common.set_partition_states(params=self._params,
active=state["active"],
passive=state["passive"],
cleanup=state["cleanup"])
time.sleep(0.5)
common.query(self._params, "mapview", {
"stale": "update_after",
"limit": "100"
})
group_stats = self.verify_final_index_state(state_transitions)
self.assertTrue(group_stats["stopped_updates"] > 0,
"Got at least 1 updater interruption")
self.assertEqual(group_stats["full_updates"], 2, "Got 2 full updates")
self.assertEqual(group_stats["compactions"], 1, "Got 1 compaction")
self.assertTrue(group_stats["cleanups"] >= 1,
"Got at least 1 full cleanup")
def test_compaction(self):
# print "Querying map view"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
map_etag = map_resp.getheader("ETag")
self.assertEqual(map_view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(map_view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(map_view_result)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying reduce view"
(red_resp, red_view_result) = common.query(self._params, "redview1")
red_etag = red_resp.getheader("ETag")
self.assertEqual(len(red_view_result["rows"]), 1, "Query returned 1 row")
self.assertEqual(red_view_result["rows"][0]["value"], self._params["ndocs"],
"Non-grouped reduce value is %d" % self._params["ndocs"])
# print "Triggering view compaction"
common.compact_set_view(self._params)
# print "Verifying set view group info"
info2 = common.get_set_view_info(self._params)
self.assertEqual(info2["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info2["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info2["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info2["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertTrue(info2["disk_size"] < info["disk_size"], "Smaller disk_size after compaction")
# print "Querying map view"
(map_resp2, map_view_result2) = common.query(self._params, "mapview1")
map_etag2 = map_resp2.getheader("ETag")
self.assertEqual(map_view_result2, map_view_result, "Got same map response after compaction")
self.assertEqual(map_etag2, map_etag, "Same map view etag after compaction")
# print "Querying reduce view"
(red_resp2, red_view_result2) = common.query(self._params, "redview1")
red_etag2 = red_resp2.getheader("ETag")
self.assertEqual(red_view_result2, red_view_result, "Got same reduce response after compaction")
self.assertEqual(red_etag2, red_etag, "Same reduce view etag after compaction")
# print "Adding 2 new documents to partition 4"
server = self._params["server"]
db4 = server[self._params["setname"] + "/3"]
new_doc1 = {"_id": "9999999999999", "integer": 999999999999999999, "string": "999999999"}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db4.save(new_doc1)
db4.save(new_doc2)
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertEqual(info["compact_running"], True, "Compaction is flagged as running in group info")
# print "Triggering set view group index update"
common.query(self._params, "redview1")
# print "Waiting for set view compaction to finish"
compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0)
self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
if i == 3:
seq = (self._params["ndocs"] / 4) + 2
else:
seq = (self._params["ndocs"] / 4)
self.assertEqual(info["update_seqs"][str(i)], seq, "right update seq for partition %d" % (i + 1))
# print "Querying map view"
(map_resp3, map_view_result3) = common.query(self._params, "mapview1")
map_etag3 = map_resp3.getheader("ETag")
self.assertEqual(map_view_result3["total_rows"], (self._params["ndocs"] + 2),
"Query returned %d total_rows" % (self._params["ndocs"] + 2))
self.assertEqual(len(map_view_result3["rows"]), (self._params["ndocs"] + 2),
"Query returned %d rows" % (self._params["ndocs"] + 2))
self.assertNotEqual(map_etag3, map_etag, "Different map view etag after recompaction")
common.test_keys_sorted(map_view_result3)
self.assertEqual(map_view_result3["rows"][0]["id"], new_doc2["_id"], "new_doc2 in map view")
self.assertEqual(map_view_result3["rows"][-1]["id"], new_doc1["_id"], "new_doc1 in map view")
# print "Querying reduce view"
(red_resp3, red_view_result3) = common.query(self._params, "redview1")
red_etag3 = red_resp3.getheader("ETag")
self.assertEqual(red_view_result3["rows"][0]["value"], (self._params["ndocs"] + 2),
"Non-grouped reduce value is %d" % (self._params["ndocs"] + 2))
self.assertNotEqual(red_etag3, red_etag, "Different reduce view etag after compaction")
# print "Triggering another view compaction"
common.compact_set_view(self._params, False)
# print "Triggering partition 4 cleanup while compaction is ongoing"
common.set_partition_states(self._params, cleanup = [3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [3], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "paritition 4 not in group info update_seqs")
self.assertEqual(info["compact_running"], True, "Compaction is flagged as running in group info")
self.assertEqual(info["cleanup_running"], True, "Cleanup is flagged as running in group info")
compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0)
self.assertTrue(compaction_was_running, "Compaction was running when the cleanup was triggered")
# print "Verifying set view group info"
info2 = common.get_set_view_info(self._params)
self.assertEqual(info2["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info2["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info2["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info2["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "paritition 4 not in group info update_seqs")
self.assertTrue(info2["disk_size"] < info["disk_size"], "Smaller disk_size after compaction+cleanup")
# print "Querying map view"
(map_resp4, map_view_result4) = common.query(self._params, "mapview1")
map_etag4 = map_resp4.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(map_view_result4["total_rows"], expected,
"Query returned %d total_rows after recompact+cleanup" % expected)
self.assertEqual(len(map_view_result4["rows"]), expected,
"Query returned %d rows after recompact+cleanup" % expected)
self.assertNotEqual(map_etag4, map_etag, "Different map view etag after compaction+cleanup")
common.test_keys_sorted(map_view_result4)
all_keys = {}
for r in map_view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 cleanup triggered" % key)
self.assertFalse(new_doc1["integer"] in all_keys, "new_doc1 not included in view query result")
self.assertFalse(new_doc2["integer"] in all_keys, "new_doc2 not included in view query result")
self.assertNotEqual(map_view_result4, map_view_result3, "Different map view result after recompact+cleanup")
# print "Querying reduce view"
(red_resp4, red_view_result4) = common.query(self._params, "redview1")
red_etag4 = red_resp4.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(red_view_result4["rows"][0]["value"], expected,
"Non-grouped reduce value is %d after recompact+cleanup" % (self._params["ndocs"] + 2))
self.assertNotEqual(red_etag4, red_etag3, "Different reduce view etag after compaction+cleanup")
def test_many_partitions(self):
total_doc_count = common.set_doc_count(self._params,
range(self._params["nparts"]))
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count,
"number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count,
"total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"],
range(self._params["nparts"]),
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
# print "Adding 1 new document to each partition"
new_docs = []
for i in xrange(self._params["nparts"]):
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(
self._params, range(self._params["nparts"]))
self.assertEqual(new_total_doc_count,
(total_doc_count + len(new_docs)),
"N documents were added")
total_doc_count = new_total_doc_count
# print "Querying view again"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count,
"number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count,
"total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"],
range(self._params["nparts"]),
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
# print "Marking half of the partitions as passive"
passive = range(self._params["nparts"] / 2, self._params["nparts"])
active = range(self._params["nparts"] / 2)
common.set_partition_states(self._params,
passive=range(self._params["nparts"] / 2,
self._params["nparts"]))
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], active,
"right active partitions list")
self.assertEqual(info["passive_partitions"], passive,
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
# print "Querying view again"
(resp, view_result) = common.query(self._params, "mapview")
expected_row_count = common.set_doc_count(self._params, active)
self.assertEqual(len(view_result["rows"]), expected_row_count,
"number of received rows is %d" % expected_row_count)
common.test_keys_sorted(view_result)
for row in view_result['rows']:
if row["key"] >= 2001:
key_part = ((row["key"] - 2000) % self._params["nparts"]) - 1
else:
key_part = (row["key"] % self._params["nparts"]) - 1
self.assertTrue(
key_part in active,
"Key %d from passive partition not in result set" % row["key"])
# print "Marking half of the partitions for cleanup"
common.set_partition_states(self._params, cleanup=passive)
cleanup = passive
common.compact_set_view(self._params)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], active,
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
self.assertEqual(info["stats"]["compactions"], 1, "1 compaction")
self.assertEqual(info["stats"]["cleanups"], 1, "1 full cleanup")
for i in active:
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
def test_filter_partitions(self):
# print "Querying map view with ?partitions=[0,1,3]"
(map_resp, map_view_result) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag = map_resp.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(map_view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(map_view_result["rows"]), expected, "Query returned %d rows" % expected)
common.test_keys_sorted(map_view_result)
all_keys = {}
for r in map_view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys, "Key %d from partition 3 is not in the result" % key)
# print "Disabling (making it passive) partition 2"
common.set_partition_states(self._params, passive = [1])
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp2, map_view_result2) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag2 = map_resp2.getheader("ETag")
self.assertEqual(map_view_result2, map_view_result, "Same result as before")
self.assertEqual(map_etag2, map_etag, "Same Etag as before")
# print "Marking partition 2 for cleanup"
common.set_partition_states(self._params, cleanup = [1])
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp3, map_view_result3) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag3 = map_resp3.getheader("ETag")
expected = self._params["ndocs"] / 2
self.assertEqual(len(map_view_result3["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(map_etag3, map_etag2, "Different Etag from before")
common.test_keys_sorted(map_view_result3)
all_keys = {}
for r in map_view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d from partition 2 is not in the result" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d from partition 3 is not in the result" % key)
# print "Triggering view compaction (to guarantee cleanup is complete)"
common.compact_set_view(self._params)
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp4, map_view_result4) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag4 = map_resp4.getheader("ETag")
# total_rows is different after cleanup
self.assertEquals(map_view_result4["rows"], map_view_result3["rows"], "Same result as before")
self.assertEquals(map_etag4, map_etag3, "Same Etag as before")
def do_test_simple_cleanup(self):
# print "Querying view in steady state"
(resp, view_result) = common.query(self._params, "mapview1")
etag = resp.getheader("ETag")
self.assertEqual(view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(view_result)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Triggering partition 4 cleanup"
common.set_partition_states(self._params, cleanup = [3])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [3], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs")
self.assertEqual(info["cleanup_running"], True, "cleanup process is running")
# print "Querying view again"
(resp2, view_result2) = common.query(self._params, "mapview1")
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result2["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result2["rows"]), expected, "Query returned %d rows" % expected)
self.assertNotEqual(etag2, etag, "Different Etag after cleanup triggered")
common.test_keys_sorted(view_result2)
all_keys = {}
for r in view_result2["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 cleanup triggered" % key)
# print "Triggering view compaction and querying view again"
common.compact_set_view(self._params)
(resp3, view_result3) = common.query(self._params, "mapview1")
etag3 = resp3.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result3["total_rows"], expected, "Query returned %d total_rows" % expected)
self.assertEqual(len(view_result3["rows"]), expected, "Query returned %d rows" % expected)
self.assertEqual(etag2, etag3, "Same Etag after cleanup finished")
common.test_keys_sorted(view_result3)
all_keys = {}
for r in view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 cleanup finished" % key)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs")
# print "Adding 2 new documents to partition 4"
server = self._params["server"]
db4 = server[self._params["setname"] + "/3"]
new_doc1 = {"_id": "999999999", "integer": 999999999, "string": "999999999"}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db4.save(new_doc1)
db4.save(new_doc2)
# print "Querying view again"
(resp4, view_result4) = common.query(self._params, "mapview1")
etag4 = resp4.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result4["total_rows"], expected, "Query returned %d total_rows" % expected)
self.assertEqual(len(view_result4["rows"]), expected, "Query returned %d rows" % expected)
self.assertEqual(etag4, etag3, "Same etag after adding new documents to cleaned partition")
common.test_keys_sorted(view_result4)
all_keys = {}
for r in view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 cleanup finished" % key)
self.assertFalse(new_doc1["integer"] in all_keys, "new_doc1 not in query result after cleanup")
self.assertFalse(new_doc2["integer"] in all_keys, "new_doc2 not in query result after cleanup")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs")
# print "Triggering compaction again and verifying it doesn't crash"
common.compact_set_view(self._params)
(resp5, view_result5) = common.query(self._params, "mapview1")
etag5 = resp5.getheader("ETag")
self.assertEqual(etag5, etag4, "Same etag after second compaction")
self.assertEqual(view_result5, view_result4, "Same query results after second compaction")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs")
def do_test_set_cleanup_partitions_when_updater_is_running(self):
# print "Marking all partitions for cleanup"
common.set_partition_states(self._params, cleanup = range(self._params["nparts"]))
# print "Compacting the set view group"
common.compact_set_view(self._params)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
self.assertEqual(view_result["total_rows"], 0, "Empty view result")
self.assertEqual(len(view_result["rows"]), 0, "Empty view result")
# print "Marking all partitions as active"
common.set_partition_states(self._params, active = range(self._params["nparts"]))
# print "Querying view with ?stale=update_after"
(resp, view_result) = common.query(self._params, "mapview1", {"stale": "update_after"})
self.assertEqual(view_result["total_rows"], 0, "Empty view result")
self.assertEqual(len(view_result["rows"]), 0, "Empty view result")
# print "Marking partition 2 for cleanup while the updater is running"
common.set_partition_states(self._params, cleanup = [1])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [1], "right cleanup partitions list")
self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs")
# print "Waiting for the set view updater to finish"
iterations = 0
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
iterations += 1
else:
break
self.assertTrue(iterations > 0, "Updater was running when partition 2 was marked for cleanup")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertTrue(["cleanup_partitions"] == [1] or info["cleanup_partitions"] == [],
"cleanup partitions list is not wrong")
self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs")
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 2, 3])
self.assertEqual(len(view_result["rows"]), doc_count, "Query returned %d rows" % doc_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 2 marked for cleanup" % key)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs")
def do_test_set_passive_during_compaction(self):
# print "Querying map view"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
self.assertEqual(map_view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(map_view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(map_view_result)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
db = self._params["server"][self._params["setname"] + "/" + str(i)]
seq = db.info()["update_seq"]
self.assertEqual(info["update_seqs"][str(i)], seq,
"right update seq for partition %d" % (i + 1))
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Marking partition 4 as passive"
common.set_partition_states(self._params, passive = [3])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Waiting for compaction to finish"
count = 0
while info["stats"]["compactions"] < 1:
time.sleep(0.5)
count += 1
info = common.get_set_view_info(self._params)
self.assertTrue((count > 0), "Compaction was running when the partition states were updated")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying map view again"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
expected = common.set_doc_count(self._params, [0, 1, 2])
self.assertEqual(map_view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(map_view_result["rows"]), expected,
"Query returned %d rows" % expected)
common.test_keys_sorted(map_view_result)
all_keys = {}
for r in map_view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 set to passive" % key)
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Adding two new partitions, 5 and 6, as passive while compaction is running"
common.set_partition_states(self._params, passive = [4, 5])
# print "Waiting for compaction to finish"
info = common.get_set_view_info(self._params)
count = 0
while info["stats"]["compactions"] < 2:
time.sleep(0.5)
count += 1
info = common.get_set_view_info(self._params)
self.assertTrue((count > 0), "Compaction was running when the partition states were updated")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3, 4, 5], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5]:
self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i)
for i in [6, 7]:
self.assertFalse(str(i) in info["update_seqs"], "%d not in info.update_seqs" % i)
# print "Querying map view again"
(map_resp, map_view_result2) = common.query(self._params, "mapview1")
self.assertEqual(map_view_result2["rows"], map_view_result["rows"], "Same result set as before")
total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
# print "Adding 1 new document to each partition"
new_docs = []
for i in [0, 1, 2, 3, 4, 5, 6, 7]:
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
self.assertEqual(new_total_doc_count, (total_doc_count + 8), "8 documents added")
self.assertEqual(len(new_docs), 8, "8 documents added")
info = common.get_set_view_info(self._params)
if info["updater_running"]:
# print "Waiting for updater to finish"
self.assertEqual(info["updater_state"], "updating_passive",
"updater state is updating_passive")
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
self.assertEqual(info["updater_state"], "updating_passive",
"updater state is updating_passive")
time.sleep(3)
else:
break
expected_row_count = common.set_doc_count(self._params, [0, 1, 2])
expected_total_rows = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5])
# print "Querying map view again"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
self.assertEqual(len(map_view_result["rows"]), expected_row_count, "len(rows) is %d" % expected_row_count)
common.test_keys_sorted(map_view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3, 4, 5], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5]:
self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i)
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"info.update_seqs[%d] is %d" % (i, expected_seq))
def test_filter_partitions(self):
# print "Querying map view with ?partitions=[0,1,3]"
(map_resp,
map_view_result) = common.query(self._params, "mapview1",
{"partitions": json.dumps([0, 1, 3])})
map_etag = map_resp.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(
map_view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(map_view_result["rows"]), expected,
"Query returned %d rows" % expected)
common.test_keys_sorted(map_view_result)
all_keys = {}
for r in map_view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d from partition 3 is not in the result" % key)
# print "Disabling (making it passive) partition 2"
common.set_partition_states(self._params, passive=[1])
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp2, map_view_result2) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag2 = map_resp2.getheader("ETag")
self.assertEqual(map_view_result2, map_view_result,
"Same result as before")
self.assertEqual(map_etag2, map_etag, "Same Etag as before")
# print "Marking partition 2 for cleanup"
common.set_partition_states(self._params, cleanup=[1])
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp3, map_view_result3) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag3 = map_resp3.getheader("ETag")
expected = self._params["ndocs"] / 2
self.assertEqual(len(map_view_result3["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(map_etag3, map_etag2, "Different Etag from before")
common.test_keys_sorted(map_view_result3)
all_keys = {}
for r in map_view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d from partition 2 is not in the result" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d from partition 3 is not in the result" % key)
# print "Triggering view compaction (to guarantee cleanup is complete)"
common.compact_set_view(self._params)
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp4, map_view_result4) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag4 = map_resp4.getheader("ETag")
# total_rows is different after cleanup
self.assertEquals(map_view_result4["rows"], map_view_result3["rows"],
"Same result as before")
self.assertEquals(map_etag4, map_etag3, "Same Etag as before")
def do_test_simple_cleanup(self):
# print "Querying view in steady state"
(resp, view_result) = common.query(self._params, "mapview1")
etag = resp.getheader("ETag")
self.assertEqual(
view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(view_result)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Triggering partition 4 cleanup"
common.set_partition_states(self._params, cleanup=[3])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [3],
"right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"],
"partition 3 not in info.update_seqs")
self.assertEqual(info["cleanup_running"], True,
"cleanup process is running")
# print "Querying view again"
(resp2, view_result2) = common.query(self._params, "mapview1")
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(
view_result2["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result2["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(etag2, etag,
"Different Etag after cleanup triggered")
common.test_keys_sorted(view_result2)
all_keys = {}
for r in view_result2["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 4 cleanup triggered" %
key)
# print "Triggering view compaction and querying view again"
common.compact_set_view(self._params)
(resp3, view_result3) = common.query(self._params, "mapview1")
etag3 = resp3.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result3["total_rows"], expected,
"Query returned %d total_rows" % expected)
self.assertEqual(len(view_result3["rows"]), expected,
"Query returned %d rows" % expected)
self.assertEqual(etag2, etag3, "Same Etag after cleanup finished")
common.test_keys_sorted(view_result3)
all_keys = {}
for r in view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 4 cleanup finished" %
key)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"],
"partition 3 not in info.update_seqs")
# print "Adding 2 new documents to partition 4"
server = self._params["server"]
db4 = server[self._params["setname"] + "/3"]
new_doc1 = {
"_id": "999999999",
"integer": 999999999,
"string": "999999999"
}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db4.save(new_doc1)
db4.save(new_doc2)
# print "Querying view again"
(resp4, view_result4) = common.query(self._params, "mapview1")
etag4 = resp4.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result4["total_rows"], expected,
"Query returned %d total_rows" % expected)
self.assertEqual(len(view_result4["rows"]), expected,
"Query returned %d rows" % expected)
self.assertEqual(
etag4, etag3,
"Same etag after adding new documents to cleaned partition")
common.test_keys_sorted(view_result4)
all_keys = {}
for r in view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 4 cleanup finished" %
key)
self.assertFalse(new_doc1["integer"] in all_keys,
"new_doc1 not in query result after cleanup")
self.assertFalse(new_doc2["integer"] in all_keys,
"new_doc2 not in query result after cleanup")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"],
"partition 3 not in info.update_seqs")
# print "Triggering compaction again and verifying it doesn't crash"
common.compact_set_view(self._params)
(resp5, view_result5) = common.query(self._params, "mapview1")
etag5 = resp5.getheader("ETag")
self.assertEqual(etag5, etag4, "Same etag after second compaction")
self.assertEqual(view_result5, view_result4,
"Same query results after second compaction")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"],
"partition 3 not in info.update_seqs")
def do_test_set_cleanup_partitions_when_updater_is_running(self):
# print "Marking all partitions for cleanup"
common.set_partition_states(self._params,
cleanup=range(self._params["nparts"]))
# print "Compacting the set view group"
common.compact_set_view(self._params)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
self.assertEqual(view_result["total_rows"], 0, "Empty view result")
self.assertEqual(len(view_result["rows"]), 0, "Empty view result")
# print "Marking all partitions as active"
common.set_partition_states(self._params,
active=range(self._params["nparts"]))
# print "Querying view with ?stale=update_after"
(resp, view_result) = common.query(self._params, "mapview1",
{"stale": "update_after"})
self.assertEqual(view_result["total_rows"], 0, "Empty view result")
self.assertEqual(len(view_result["rows"]), 0, "Empty view result")
# print "Marking partition 2 for cleanup while the updater is running"
common.set_partition_states(self._params, cleanup=[1])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [1],
"right cleanup partitions list")
self.assertFalse("1" in info["update_seqs"],
"partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"],
"partition 1 not in info.update_seqs")
# print "Waiting for the set view updater to finish"
iterations = 0
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
iterations += 1
else:
break
self.assertTrue(
iterations > 0,
"Updater was running when partition 2 was marked for cleanup")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertTrue(["cleanup_partitions"] == [1]
or info["cleanup_partitions"] == [],
"cleanup partitions list is not wrong")
self.assertFalse("1" in info["update_seqs"],
"partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"],
"partition 1 not in info.update_seqs")
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 2, 3])
self.assertEqual(len(view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 2 marked for cleanup" %
key)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
self.assertFalse("1" in info["update_seqs"],
"partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"],
"partition 1 not in info.update_seqs")
def test_many_partitions(self):
total_doc_count = common.set_doc_count(self._params, range(self._params["nparts"]))
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], range(self._params["nparts"]), "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
# print "Adding 1 new document to each partition"
new_docs = []
for i in xrange(self._params["nparts"]):
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(self._params, range(self._params["nparts"]))
self.assertEqual(new_total_doc_count, (total_doc_count + len(new_docs)), "N documents were added")
total_doc_count = new_total_doc_count
# print "Querying view again"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], range(self._params["nparts"]), "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
# print "Marking half of the partitions as passive"
passive = range(self._params["nparts"] / 2, self._params["nparts"])
active = range(self._params["nparts"] / 2)
common.set_partition_states(
self._params,
passive = range(self._params["nparts"] / 2, self._params["nparts"])
)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], active, "right active partitions list")
self.assertEqual(info["passive_partitions"], passive, "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
# print "Querying view again"
(resp, view_result) = common.query(self._params, "mapview")
expected_row_count = common.set_doc_count(self._params, active)
self.assertEqual(len(view_result["rows"]), expected_row_count, "number of received rows is %d" % expected_row_count)
common.test_keys_sorted(view_result)
for row in view_result['rows']:
if row["key"] >= 2001:
key_part = ((row["key"] - 2000) % self._params["nparts"]) - 1
else:
key_part = (row["key"] % self._params["nparts"]) - 1
self.assertTrue(key_part in active, "Key %d from passive partition not in result set" % row["key"])
# print "Marking half of the partitions for cleanup"
common.set_partition_states(
self._params,
cleanup = passive
)
cleanup = passive
common.compact_set_view(self._params)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], active, "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
self.assertEqual(info["stats"]["compactions"], 1, "1 compaction")
self.assertEqual(info["stats"]["cleanups"], 1, "1 full cleanup")
for i in active:
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
