def do_test_with_updates(self, state_transitions):
# start from a clean state always
common.set_partition_states(
params = self._params,
cleanup = range(self._params["nparts"])
)
# print "Cleaning set view via compaction"
common.compact_set_view(self._params)
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(view_result["total_rows"], 0, "total_rows is 0 after compaction")
self.assertEqual(len(view_result["rows"]), 0, "0 rows returned after compaction")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
self.assertEqual(info["update_seqs"], {}, "right update seqs list")
# print "Triggering state changes with queries (?stale=updater_after) in between"
for state in state_transitions:
common.set_partition_states(
params = self._params,
active = state["active"],
passive = state["passive"],
cleanup = state["cleanup"]
)
time.sleep(0.5)
common.query(self._params, "mapview", {"stale": "update_after", "limit": "100"})
group_stats = self.verify_final_index_state(state_transitions)
self.assertTrue(group_stats["updater_interruptions"] > 0, "Got at least 1 updater interruption")
self.assertEqual(group_stats["updates"], 2, "Got 2 full updates")
self.assertEqual(group_stats["compactions"], 1, "Got 1 compaction")
self.assertTrue(group_stats["cleanups"] >= 1, "Got at least 1 full cleanup")
def do_test_set_active_during_compaction(self):
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Marking partitions 4, 5 and 6 as active while compaction is running"
common.set_partition_states(self._params, active = [3, 4, 5])
# print "Adding new partitions 7 and 8 with active state while compaction is running"
common.set_partition_states(self._params, active = [6, 7])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying map view while compaction is running"
(map_resp, map_view_result) = common.query(self._params, "mapview1", {"limit": "10"})
self.assertEqual(len(map_view_result["rows"]), 10, "Query returned 10 rows")
common.test_keys_sorted(map_view_result)
# print "Waiting for compaction to finish"
compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0)
self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying map view again"
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
(map_resp, map_view_result) = common.query(self._params, "mapview1")
self.assertEqual(map_view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(map_view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(map_view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5, 6, 7]:
self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i)
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"info.update_seqs[%d] is %d" % (i, expected_seq))
def do_test_without_updates(self, state_transitions):
# print "Triggering state changes without queries in between"
for state in state_transitions:
common.set_partition_states(params=self._params,
active=state["active"],
passive=state["passive"],
cleanup=state["cleanup"])
group_stats = self.verify_final_index_state(state_transitions)
self.assertEqual(group_stats["stopped_updates"], 0,
"0 updater interruptions")
self.assertEqual(group_stats["full_updates"], 1, "Got 1 full update")
def do_test_without_updates(self, state_transitions):
# print "Triggering state changes without queries in between"
for state in state_transitions:
common.set_partition_states(
params = self._params,
active = state["active"],
passive = state["passive"],
cleanup = state["cleanup"]
)
group_stats = self.verify_final_index_state(state_transitions)
self.assertEqual(group_stats["updater_interruptions"], 0, "0 updater interruptions")
self.assertEqual(group_stats["updates"], 1, "Got 1 full update")
def do_test_with_updates(self, state_transitions):
# start from a clean state always
common.set_partition_states(params=self._params,
cleanup=range(self._params["nparts"]))
# print "Cleaning set view via compaction"
common.compact_set_view(self._params)
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(view_result["total_rows"], 0,
"total_rows is 0 after compaction")
self.assertEqual(len(view_result["rows"]), 0,
"0 rows returned after compaction")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
self.assertEqual(info["update_seqs"], {}, "right update seqs list")
# print "Triggering state changes with queries (?stale=updater_after) in between"
for state in state_transitions:
common.set_partition_states(params=self._params,
active=state["active"],
passive=state["passive"],
cleanup=state["cleanup"])
time.sleep(0.5)
common.query(self._params, "mapview", {
"stale": "update_after",
"limit": "100"
})
group_stats = self.verify_final_index_state(state_transitions)
self.assertTrue(group_stats["stopped_updates"] > 0,
"Got at least 1 updater interruption")
self.assertEqual(group_stats["full_updates"], 2, "Got 2 full updates")
self.assertEqual(group_stats["compactions"], 1, "Got 1 compaction")
self.assertTrue(group_stats["cleanups"] >= 1,
"Got at least 1 full cleanup")
def do_test_set_cleanup_partitions_when_updater_is_running(self):
# print "Marking all partitions for cleanup"
common.set_partition_states(self._params,
cleanup=range(self._params["nparts"]))
# print "Compacting the set view group"
common.compact_set_view(self._params)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
self.assertEqual(view_result["total_rows"], 0, "Empty view result")
self.assertEqual(len(view_result["rows"]), 0, "Empty view result")
# print "Marking all partitions as active"
common.set_partition_states(self._params,
active=range(self._params["nparts"]))
# print "Querying view with ?stale=update_after"
(resp, view_result) = common.query(self._params, "mapview1",
{"stale": "update_after"})
self.assertEqual(view_result["total_rows"], 0, "Empty view result")
self.assertEqual(len(view_result["rows"]), 0, "Empty view result")
# print "Marking partition 2 for cleanup while the updater is running"
common.set_partition_states(self._params, cleanup=[1])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [1],
"right cleanup partitions list")
self.assertFalse("1" in info["update_seqs"],
"partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"],
"partition 1 not in info.update_seqs")
# print "Waiting for the set view updater to finish"
iterations = 0
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
iterations += 1
else:
break
self.assertTrue(
iterations > 0,
"Updater was running when partition 2 was marked for cleanup")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertTrue(["cleanup_partitions"] == [1]
or info["cleanup_partitions"] == [],
"cleanup partitions list is not wrong")
self.assertFalse("1" in info["update_seqs"],
"partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"],
"partition 1 not in info.update_seqs")
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 2, 3])
self.assertEqual(len(view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 2 marked for cleanup" %
key)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
self.assertFalse("1" in info["update_seqs"],
"partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"],
"partition 1 not in info.update_seqs")
def test_updates(self):
total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
# print "Querying view in steady state with ?stale=update_after"
(resp, view_result) = common.query(self._params, "mapview", {"stale": "update_after"})
self.assertEqual(len(view_result["rows"]), 0, "Received empty row set")
self.assertEqual(view_result["total_rows"], 0, "Received empty row set")
# print "Adding partitions 7 and 8 as active while updater is running"
common.set_partition_states(self._params, active = [6, 7])
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Waiting for updater to finish"
count = 0
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
count += 1
else:
break
self.assertTrue(count > 0, "Updater was running when partitions 7 and 8 were added as active")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5]:
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
# print "Querying view again (steady state)"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Adding 1 new document to each partition"
new_docs = []
for i in [0, 1, 2, 3, 4, 5, 6, 7]:
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
self.assertEqual(new_total_doc_count, (total_doc_count + len(new_docs)), "8 documents were added")
total_doc_count = new_total_doc_count
# print "Querying view again (steady state)"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
i = len(new_docs) - 1
j = len(view_result["rows"]) - 1
while i >= 0:
self.assertEqual(view_result["rows"][j]["key"], new_docs[i]["integer"],
"new document %s in view result set" % new_docs[i]["_id"])
self.assertEqual(view_result["rows"][j]["id"], new_docs[i]["_id"],
"new document %s in view result set" % new_docs[i]["_id"])
i -= 1
j -= 1
# print "Deleting the documents that were added before"
i = 0
for doc in new_docs:
db = self._params["setname"] + "/" + str(i)
common.delete_doc(self._params, db, doc["_id"])
i = (i + 1) % self._params["nparts"]
new_total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
self.assertEqual(new_total_doc_count, (total_doc_count - len(new_docs)), "8 documents were deleted")
# print "Querying view again (steady state)"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), new_total_doc_count,
"number of received rows is %d" % new_total_doc_count)
self.assertEqual(view_result["total_rows"], new_total_doc_count,
"total_rows is %d" % new_total_doc_count)
common.test_keys_sorted(view_result)
all_ids = {}
all_keys = {}
for row in view_result["rows"]:
all_ids[row["id"]] = True
all_keys[row["key"]] = True
for doc in new_docs:
self.assertFalse(doc["_id"] in all_ids, "deleted doc %s not in view results anymore")
self.assertFalse(doc["integer"] in all_keys, "deleted doc %s not in view results anymore")
def test_many_partitions(self):
total_doc_count = common.set_doc_count(self._params,
range(self._params["nparts"]))
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count,
"number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count,
"total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"],
range(self._params["nparts"]),
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
# print "Adding 1 new document to each partition"
new_docs = []
for i in xrange(self._params["nparts"]):
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(
self._params, range(self._params["nparts"]))
self.assertEqual(new_total_doc_count,
(total_doc_count + len(new_docs)),
"N documents were added")
total_doc_count = new_total_doc_count
# print "Querying view again"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count,
"number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count,
"total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"],
range(self._params["nparts"]),
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
# print "Marking half of the partitions as passive"
passive = range(self._params["nparts"] / 2, self._params["nparts"])
active = range(self._params["nparts"] / 2)
common.set_partition_states(self._params,
passive=range(self._params["nparts"] / 2,
self._params["nparts"]))
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], active,
"right active partitions list")
self.assertEqual(info["passive_partitions"], passive,
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
# print "Querying view again"
(resp, view_result) = common.query(self._params, "mapview")
expected_row_count = common.set_doc_count(self._params, active)
self.assertEqual(len(view_result["rows"]), expected_row_count,
"number of received rows is %d" % expected_row_count)
common.test_keys_sorted(view_result)
for row in view_result['rows']:
if row["key"] >= 2001:
key_part = ((row["key"] - 2000) % self._params["nparts"]) - 1
else:
key_part = (row["key"] % self._params["nparts"]) - 1
self.assertTrue(
key_part in active,
"Key %d from passive partition not in result set" % row["key"])
# print "Marking half of the partitions for cleanup"
common.set_partition_states(self._params, cleanup=passive)
cleanup = passive
common.compact_set_view(self._params)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], active,
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
self.assertEqual(info["stats"]["compactions"], 1, "1 compaction")
self.assertEqual(info["stats"]["cleanups"], 1, "1 full cleanup")
for i in active:
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
def do_test_change_partition_states_while_cleanup_running(self):
# print "Marking all partitions as active"
common.set_partition_states(self._params,
active=range(self._params["nparts"]))
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
# print "Updating view"
(resp, view_result) = common.query(self._params, "mapview1",
{"limit": "100"})
self.assertEqual(view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), 100,
"Query returned 100 rows")
common.test_keys_sorted(view_result)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
# print "Marking partitions 1 and 2 for cleanup"
common.set_partition_states(self._params, cleanup=[0, 1])
info = common.get_set_view_info(self._params)
self.assertEqual(info["cleanup_running"], True, "cleanup is running")
self.assertEqual(info["active_partitions"], [2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [0, 1],
"right cleanup partitions list")
# print "Marking partitions 1 and 2 as active while cleanup is ongoing"
common.set_partition_states(self._params, active=[0, 1])
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
info = common.get_set_view_info(self._params)
self.assertEqual(view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d in result after partition 1 activated" % key)
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d in result after partition 2 activated" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 3) in result set" % key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 4) in result set" % key)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
def do_test_maps(self):
# print "Querying map view in steady state"
(resp, view_result) = common.query(self._params, "mapview1")
etag = resp.getheader("ETag")
self.assertEqual(
view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(view_result)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Setting partition 4 as passive"
common.set_partition_states(self._params, passive=[3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [3],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp2, view_result2) = common.query(self._params, "mapview1")
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(
view_result2["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result2["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(etag2, etag,
"Different Etag after setting passive partition")
common.test_keys_sorted(view_result2)
all_keys = {}
for r in view_result2["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 4 was made passive" %
key)
# print "Setting partition 4 state to active"
common.set_partition_states(self._params, active=[3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp3, view_result3) = common.query(self._params, "mapview1")
etag3 = resp3.getheader("ETag")
self.assertEqual(
view_result3["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result3["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
self.assertEqual(etag3, etag, "ETag is same as first query response")
common.test_keys_sorted(view_result3)
# print "Setting partitions 1 and 4 to passive state"
common.set_partition_states(self._params, passive=[0, 3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [0, 3],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp4, view_result4) = common.query(self._params, "mapview1")
etag4 = resp4.getheader("ETag")
expected = self._params["ndocs"] / 2
self.assertEqual(
view_result4["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result4["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(etag4, etag,
"ETag is different from all previous responses")
self.assertNotEqual(etag4, etag2,
"ETag is different from all previous responses")
common.test_keys_sorted(view_result4)
all_keys = {}
for r in view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 1 was made passive" %
key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 4 was made passive" %
key)
# print "Marking all partitions as passive and querying view again"
common.set_partition_states(self._params,
passive=range(self._params["nparts"]))
(resp5, view_result5) = common.query(self._params, "mapview1")
etag5 = resp5.getheader("ETag")
self.assertEqual(
view_result5["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result5["rows"]), 0, "Query returned 0 rows")
self.assertNotEqual(etag5, etag,
"ETag is different from all previous responses")
self.assertNotEqual(etag5, etag2,
"ETag is different from all previous responses")
self.assertNotEqual(etag5, etag3,
"ETag is different from all previous responses")
self.assertNotEqual(etag5, etag4,
"ETag is different from all previous responses")
# print "Setting all partitions to active state"
common.set_partition_states(self._params,
active=range(self._params["nparts"]))
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp6, view_result6) = common.query(self._params, "mapview1")
etag6 = resp6.getheader("ETag")
self.assertEqual(
view_result6["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result6["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
self.assertEqual(etag6, etag,
"ETag is the same from first view query response")
common.test_keys_sorted(view_result6)
def do_test_reduce(self, viewname):
# print "Querying reduce view in steady state"
(resp, view_result) = common.query(self._params, viewname)
etag = resp.getheader("ETag")
self.assertEqual(len(view_result["rows"]), 1, "Query returned 1 row")
self.assertEqual(
view_result["rows"][0]["value"], self._params["ndocs"],
"Non-grouped reduce value is %d" % self._params["ndocs"])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Setting partition 3 to passive state"
common.set_partition_states(self._params, passive=[2])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [2],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying reduce view again"
(resp2, view_result2) = common.query(self._params, viewname)
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(len(view_result2["rows"]), 1, "Query returned 1 row")
self.assertEqual(view_result2["rows"][0]["value"], expected,
"Non-grouped reduce value is %d" % expected)
self.assertNotEqual(etag2, etag, "Different ETags")
# print "Querying view with ?group=true"
(resp3, view_result3) = common.query(self._params, viewname,
{"group": "true"})
etag3 = resp3.getheader("ETag")
self.assertEqual(len(view_result3["rows"]), expected,
"Query returned % rows" % expected)
self.assertNotEqual(etag3, etag, "Different ETags")
self.assertEqual(etag3, etag2, "Equal ETags for responses 2 and 3")
common.test_keys_sorted(view_result3)
all_keys = {}
for r in view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 3 was made passive" %
key)
# print "Querying view with ?group=true&descending=true"
(resp4, view_result4) = common.query(self._params, viewname, {
"group": "true",
"descending": "true"
})
etag4 = resp4.getheader("ETag")
self.assertEqual(len(view_result4["rows"]), expected,
"Query returned % rows" % expected)
self.assertNotEqual(etag4, etag, "Different ETags")
self.assertEqual(etag4, etag3, "Equal ETags for responses 3 and 4")
common.test_keys_sorted(view_result4, lambda a, b: a > b)
all_keys = {}
for r in view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 3 was made passive" %
key)
# print "Querying view with ?group=true&startkey=3333&endkey=44781"
(resp5, view_result5) = common.query(self._params, viewname, {
"group": "true",
"startkey": "3333",
"endkey": "44781"
})
etag5 = resp5.getheader("ETag")
self.assertNotEqual(etag5, etag, "Different ETags")
self.assertEqual(etag5, etag4, "Equal ETags for responses 4 and 5")
common.test_keys_sorted(view_result5)
self.assertTrue(view_result5["rows"][0]["key"] >= 3333,
"First key is >= 3333")
self.assertTrue(view_result5["rows"][-1]["key"] <= 44781,
"Last key is <= 44781")
all_keys = {}
for r in view_result5["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 3 was made passive" %
key)
# print "Querying view with ?group=true&startkey=44781&endkey=3333&descending=true"
(resp6, view_result6) = common.query(
self._params, viewname, {
"group": "true",
"startkey": "44781",
"endkey": "3333",
"descending": "true"
})
etag6 = resp6.getheader("ETag")
self.assertNotEqual(etag6, etag, "Different ETags")
self.assertEqual(etag6, etag5, "Equal ETags for responses 5 and 6")
common.test_keys_sorted(view_result6, lambda a, b: a > b)
self.assertTrue(view_result6["rows"][0]["key"] <= 44781,
"First key is <= 44781")
self.assertTrue(view_result6["rows"][-1]["key"] >= 3333,
"Last key is >= 3333")
self.assertEqual(len(view_result6["rows"]), len(view_result5["rows"]),
"Same number of rows for responses 5 and 6")
all_keys = {}
for r in view_result6["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 3 was made passive" %
key)
# print "Setting partition 3 to active state"
common.set_partition_states(self._params, active=[2])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view with ?group=true"
(resp7, view_result7) = common.query(self._params, viewname,
{"group": "true"})
etag7 = resp7.getheader("ETag")
self.assertEqual(len(view_result7["rows"]), self._params["ndocs"],
"Query returned % rows" % self._params["ndocs"])
self.assertEqual(etag7, etag, "Same etags for responses 1 and 7")
common.test_keys_sorted(view_result7)
# print "Querying view with ?group=true&descending=true"
(resp8, view_result8) = common.query(self._params, viewname, {
"group": "true",
"descending": "true"
})
etag8 = resp8.getheader("ETag")
self.assertEqual(len(view_result8["rows"]), self._params["ndocs"],
"Query returned % rows" % self._params["ndocs"])
self.assertEqual(etag7, etag8, "Same etags for responses 7 and 8")
common.test_keys_sorted(view_result8, lambda a, b: a > b)
# print "Querying view with ?group=true&startkey=3333&endkey=44781"
(resp9, view_result9) = common.query(self._params, viewname, {
"group": "true",
"startkey": "3333",
"endkey": "44781"
})
etag9 = resp9.getheader("ETag")
self.assertEqual(etag9, etag8, "Equal ETags for responses 8 and 9")
common.test_keys_sorted(view_result9)
self.assertTrue(view_result9["rows"][0]["key"] >= 3333,
"First key is >= 3333")
self.assertTrue(view_result9["rows"][-1]["key"] <= 44781,
"Last key is <= 44781")
# print "Querying view with ?group=true&startkey=44781&endkey=3333&descending=true"
(resp10, view_result10) = common.query(
self._params, viewname, {
"group": "true",
"startkey": "44781",
"endkey": "3333",
"descending": "true"
})
etag10 = resp10.getheader("ETag")
self.assertEqual(etag10, etag9, "Equal ETags for responses 9 and 10")
common.test_keys_sorted(view_result10, lambda a, b: a > b)
self.assertTrue(view_result10["rows"][0]["key"] <= 44781,
"First key is <= 44781")
self.assertTrue(view_result10["rows"][-1]["key"] >= 3333,
"Last key is >= 3333")
self.assertEqual(len(view_result10["rows"]), len(view_result9["rows"]),
"Same number of rows for responses 9 and 10")
def do_test_view_updates(self):
# print "Setting partition 2 state to passive"
common.set_partition_states(self._params, passive = [1])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [1], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Adding 2 new documents to partition 2"
server = self._params["server"]
db2 = server[self._params["setname"] + "/1"]
new_doc1 = {"_id": "999999999", "integer": 999999999, "string": "999999999"}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db2.save(new_doc1)
db2.save(new_doc2)
# print "Querying map view"
(resp, view_result) = common.query(self._params, "mapview1")
etag = resp.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(len(view_result["rows"]), expected, "Query returned %d rows" % expected)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 2 was made passive" % key)
self.assertFalse(new_doc1["integer"] in all_keys, "new_doc1 not reflected in view")
self.assertFalse(new_doc2["integer"] in all_keys, "new_doc2 not reflected in view")
# print "Setting partition 2 state to active"
common.set_partition_states(self._params, active = [1])
# print "Querying map view again"
(resp2, view_result2) = common.query(self._params, "mapview1")
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] + 2
self.assertEqual(view_result2["total_rows"], expected, "Query returned %d total_rows" % expected)
self.assertEqual(len(view_result2["rows"]), expected, "Query returned %d rows" % expected)
common.test_keys_sorted(view_result2)
all_keys = {}
for r in view_result2["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d in result after partition 2 was re-enabled" % key)
self.assertTrue(new_doc1["integer"] in all_keys, "new_doc1 reflected in view")
self.assertTrue(new_doc2["integer"] in all_keys, "new_doc2 reflected in view")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
if i == 1:
seq = (self._params["ndocs"] / 4) + 2
else:
seq = (self._params["ndocs"] / 4)
self.assertEqual(info["update_seqs"][str(i)], seq, "right update seq for partition %d" % (i + 1))
def test_updates(self):
total_doc_count = common.set_doc_count(self._params,
[0, 1, 2, 3, 4, 5, 6, 7])
# print "Querying view in steady state with ?stale=update_after"
(resp, view_result) = common.query(self._params, "mapview",
{"stale": "update_after"})
self.assertEqual(len(view_result["rows"]), 0, "Received empty row set")
self.assertEqual(view_result["total_rows"], 0,
"Received empty row set")
# print "Adding partitions 7 and 8 as active while updater is running"
common.set_partition_states(self._params, active=[6, 7])
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
# print "Waiting for updater to finish"
count = 0
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
count += 1
else:
break
self.assertTrue(
count > 0,
"Updater was running when partitions 7 and 8 were added as active")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5]:
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
# print "Querying view again (steady state)"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count,
"number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count,
"total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Adding 1 new document to each partition"
new_docs = []
for i in [0, 1, 2, 3, 4, 5, 6, 7]:
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(self._params,
[0, 1, 2, 3, 4, 5, 6, 7])
self.assertEqual(new_total_doc_count,
(total_doc_count + len(new_docs)),
"8 documents were added")
total_doc_count = new_total_doc_count
# print "Querying view again (steady state)"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count,
"number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count,
"total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
i = len(new_docs) - 1
j = len(view_result["rows"]) - 1
while i >= 0:
self.assertEqual(
view_result["rows"][j]["key"], new_docs[i]["integer"],
"new document %s in view result set" % new_docs[i]["_id"])
self.assertEqual(
view_result["rows"][j]["id"], new_docs[i]["_id"],
"new document %s in view result set" % new_docs[i]["_id"])
i -= 1
j -= 1
# print "Deleting the documents that were added before"
i = 0
for doc in new_docs:
db = self._params["setname"] + "/" + str(i)
common.delete_doc(self._params, db, doc["_id"])
i = (i + 1) % self._params["nparts"]
new_total_doc_count = common.set_doc_count(self._params,
[0, 1, 2, 3, 4, 5, 6, 7])
self.assertEqual(new_total_doc_count,
(total_doc_count - len(new_docs)),
"8 documents were deleted")
# print "Querying view again (steady state)"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), new_total_doc_count,
"number of received rows is %d" % new_total_doc_count)
self.assertEqual(view_result["total_rows"], new_total_doc_count,
"total_rows is %d" % new_total_doc_count)
common.test_keys_sorted(view_result)
all_ids = {}
all_keys = {}
for row in view_result["rows"]:
all_ids[row["id"]] = True
all_keys[row["key"]] = True
for doc in new_docs:
self.assertFalse(doc["_id"] in all_ids,
"deleted doc %s not in view results anymore")
self.assertFalse(doc["integer"] in all_keys,
"deleted doc %s not in view results anymore")
def do_test_change_partition_states_while_cleanup_running(self):
# print "Marking all partitions as active"
common.set_partition_states(self._params,
active=range(self._params["nparts"]))
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
# print "Updating view"
(resp, view_result) = common.query(self._params, "mapview1",
{"limit": "100"})
self.assertEqual(view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), 100,
"Query returned 100 rows")
common.test_keys_sorted(view_result)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
# print "Marking partitions 1 and 2 for cleanup"
common.set_partition_states(self._params, cleanup=[0, 1])
info = common.get_set_view_info(self._params)
self.assertEqual(info["cleanup_running"], True, "cleanup is running")
self.assertEqual(info["active_partitions"], [2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [0, 1],
"right cleanup partitions list")
# print "Marking partitions 1 and 2 as active while cleanup is ongoing"
common.set_partition_states(self._params, active=[0, 1])
info = common.get_set_view_info(self._params)
self.assertEqual(type(info["pending_transition"]), dict,
"pending_transition is an object")
self.assertEqual(sorted(info["pending_transition"]["active"]), [0, 1],
"pending_transition active list is [0, 1]")
self.assertEqual(info["pending_transition"]["passive"], [],
"pending_transition passive list is []")
self.assertEqual(info["pending_transition"]["cleanup"], [],
"pending_transition cleanup list is []")
# print "Waiting for pending transition to be applied"
iterations = 0
while True:
if iterations > 600:
raise (Exception(
"timeout waiting for pending transition to be applied"))
info = common.get_set_view_info(self._params)
if info["pending_transition"] is None:
break
else:
time.sleep(1)
iterations += 1
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
info = common.get_set_view_info(self._params)
self.assertEqual(view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d in result after partition 1 activated" % key)
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d in result after partition 2 activated" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 3) in result set" % key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 4) in result set" % key)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
def do_test_replica_support(self):
server = couchdb.Server(url = "http://" + HOST)
local_params = {
"host": HOST,
"ddoc": deepcopy(DDOC),
"nparts": NUM_PARTS,
"ndocs": 50000,
"setname": SET_NAME_LOCAL + "_2",
"server": server
}
remote_params = {
"host": HOST,
"ddoc": deepcopy(DDOC),
"nparts": NUM_PARTS,
"start_id": 50000,
"ndocs": 50000,
"setname": SET_NAME_REMOTE + "_2",
"server": server
}
# print "Creating databases"
common.create_dbs(local_params)
common.create_dbs(remote_params)
common.populate(local_params)
common.populate(remote_params)
# print "Databases created"
common.define_set_view(local_params, [0, 1], [], True)
common.define_set_view(remote_params, [2, 3], [], True)
common.add_replica_partitions(local_params, [2, 3])
common.add_replica_partitions(remote_params, [0, 1])
local_spec = self.set_spec(local_params["setname"], "mapview", [0, 1])
remote_spec = self.set_spec(remote_params["setname"], "mapview", [2, 3])
remote_merge = self.merge_spec(remote_params["host"], [], [remote_spec])
# print "Triggering view update"
full_spec = self.views_spec([remote_merge], [local_spec])
_, result = self.query(local_params["host"], full_spec, {"limit": "1", "stale": "false"})
info = common.get_set_view_info(local_params)
seconds = 0
while info["replica_group_info"]["stats"]["full_updates"] < 1:
time.sleep(3)
seconds += 3
if seconds > 900:
raise(Exception("timeout waiting for replica group full update"))
info = common.get_set_view_info(local_params)
info = common.get_set_view_info(remote_params)
seconds = 0
while info["replica_group_info"]["stats"]["full_updates"] < 1:
time.sleep(3)
seconds += 3
if seconds > 900:
raise(Exception("timeout waiting for replica group full update"))
info = common.get_set_view_info(remote_params)
local_spec = self.set_spec(local_params["setname"], "mapview", [0, 1])
remote_spec = self.set_spec(remote_params["setname"], "mapview", [2, 3])
remote_merge = self.merge_spec(remote_params["host"], [], [remote_spec])
full_spec = self.views_spec([remote_merge], [local_spec])
_, result = self.query(local_params["host"], full_spec, {"stale": "false"})
self.assertEqual(len(result["rows"]), 50000, "response has 50000 rows")
# print "Activating replica partitions"
common.set_partition_states(local_params, active = [2, 3])
common.set_partition_states(remote_params, active = [0, 1])
# print "Querying with ?debug=true"
local_spec = self.set_spec(local_params["setname"], "mapview", [0, 1, 2, 3])
remote_spec = self.set_spec(remote_params["setname"], "mapview", [0, 1, 2, 3])
remote_merge = self.merge_spec(remote_params["host"], [], [remote_spec])
full_spec = self.views_spec([remote_merge], [local_spec])
_, result = self.query(local_params["host"], full_spec, {"stale": "ok", "debug": "true"})
self.assertEqual(len(result["rows"]), 100000, "response has 100000 rows")
self.assertTrue("debug_info" in result, "result has debug info")
self.assertTrue("local" in result["debug_info"], "result have debug info for local node")
remote_node = "http://%s/_view_merge/" % remote_params["host"]
self.assertTrue(remote_node in result["debug_info"], "result have debug info for remote node")
next_key = 1
next_part = 0
i = 0
while next_key < 100000:
row = result["rows"][i]
self.assertEqual(row["key"], next_key, "correct row key at iteration %d" % (i + 1))
self.assertEqual(row["value"], str(next_key), "correct row value at iteration %d" % (i + 1))
self.assertEqual(row["partition"], next_part, "correct row partition at iteration %d" % (i + 1))
node = "local"
if next_key > 50000:
node = "http://%s/_view_merge/" % remote_params["host"]
self.assertEqual(row["node"], node, "correct row node at iteration %d" % (i + 1))
i += 1
next_key += 1
next_part = (next_part + 1) % 4
# print "Deleting databases"
common.create_dbs(local_params, True)
common.create_dbs(remote_params, True)
def do_test_change_partition_states_while_cleanup_running(self):
# print "Marking all partitions as active"
common.set_partition_states(self._params, active = range(self._params["nparts"]))
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
# print "Updating view"
(resp, view_result) = common.query(self._params, "mapview1", {"limit": "100"})
self.assertEqual(view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), 100, "Query returned 100 rows")
common.test_keys_sorted(view_result)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
# print "Marking partitions 1 and 2 for cleanup"
common.set_partition_states(self._params, cleanup = [0, 1])
info = common.get_set_view_info(self._params)
self.assertEqual(info["cleanup_running"], True, "cleanup is running")
self.assertEqual(info["active_partitions"], [2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [0, 1], "right cleanup partitions list")
# print "Marking partitions 1 and 2 as active while cleanup is ongoing"
common.set_partition_states(self._params, active = [0, 1])
info = common.get_set_view_info(self._params)
self.assertEqual(type(info["pending_transition"]), dict, "pending_transition is an object")
self.assertEqual(sorted(info["pending_transition"]["active"]),
[0, 1],
"pending_transition active list is [0, 1]")
self.assertEqual(info["pending_transition"]["passive"],
[],
"pending_transition passive list is []")
self.assertEqual(info["pending_transition"]["cleanup"],
[],
"pending_transition cleanup list is []")
# print "Waiting for pending transition to be applied"
iterations = 0
while True:
if iterations > 600:
raise(Exception("timeout waiting for pending transition to be applied"))
info = common.get_set_view_info(self._params)
if info["pending_transition"] is None:
break
else:
time.sleep(1)
iterations += 1
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
info = common.get_set_view_info(self._params)
self.assertEqual(view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), doc_count, "Query returned %d rows" % doc_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d in result after partition 1 activated" % key)
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d in result after partition 2 activated" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 3) in result set" % key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 4) in result set" % key)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
def test_filter_partitions(self):
# print "Querying map view with ?partitions=[0,1,3]"
(map_resp,
map_view_result) = common.query(self._params, "mapview1",
{"partitions": json.dumps([0, 1, 3])})
map_etag = map_resp.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(
map_view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(map_view_result["rows"]), expected,
"Query returned %d rows" % expected)
common.test_keys_sorted(map_view_result)
all_keys = {}
for r in map_view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d from partition 3 is not in the result" % key)
# print "Disabling (making it passive) partition 2"
common.set_partition_states(self._params, passive=[1])
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp2, map_view_result2) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag2 = map_resp2.getheader("ETag")
self.assertEqual(map_view_result2, map_view_result,
"Same result as before")
self.assertEqual(map_etag2, map_etag, "Same Etag as before")
# print "Marking partition 2 for cleanup"
common.set_partition_states(self._params, cleanup=[1])
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp3, map_view_result3) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag3 = map_resp3.getheader("ETag")
expected = self._params["ndocs"] / 2
self.assertEqual(len(map_view_result3["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(map_etag3, map_etag2, "Different Etag from before")
common.test_keys_sorted(map_view_result3)
all_keys = {}
for r in map_view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d from partition 2 is not in the result" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d from partition 3 is not in the result" % key)
# print "Triggering view compaction (to guarantee cleanup is complete)"
common.compact_set_view(self._params)
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp4, map_view_result4) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag4 = map_resp4.getheader("ETag")
# total_rows is different after cleanup
self.assertEquals(map_view_result4["rows"], map_view_result3["rows"],
"Same result as before")
self.assertEquals(map_etag4, map_etag3, "Same Etag as before")
def do_test_set_passive_partitions_when_updater_is_running(self):
# print "Re-creating databases"
common.create_dbs(self._params)
common.populate(self._params)
# print "Configuring set view with all partitions active"
common.define_set_view(self._params, [0, 1, 2, 3], [])
# print "Querying map view in steady state with ?stale=update_after"
(resp, view_result) = common.query(self._params, "mapview", {"stale": "update_after"})
self.assertEqual(len(view_result["rows"]), 0, "Received empty row set")
self.assertEqual(view_result["total_rows"], 0, "Received empty row set")
# print "Marking partition 4 as passive"
common.set_partition_states(self._params, passive = [3])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Waiting for the set view updater to finish"
iterations = 0
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
iterations += 1
else:
break
self.assertTrue(iterations > 0, "Updater was running when partition 4 was set to passive")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying map view again"
(resp, view_result) = common.query(self._params, "mapview")
doc_count = common.set_doc_count(self._params)
expected_row_count = common.set_doc_count(self._params, [0, 1, 2])
self.assertEqual(view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), expected_row_count, "Query returned %d rows" % expected_row_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, doc_count, self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 was set to passive" % key)
# print "Adding 2 new documents to partition 4"
server = self._params["server"]
db4 = server[self._params["setname"] + "/3"]
new_doc1 = {"_id": "999999999", "integer": 999999999, "string": "999999999"}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db4.save(new_doc1)
db4.save(new_doc2)
# print "Querying map view again"
(resp, view_result2) = common.query(self._params, "mapview")
self.assertEqual(view_result2["rows"], view_result["rows"], "Same result set as before")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
total_doc_count = common.set_doc_count(self._params)
# print "Changing partition 4 from passive to active"
common.set_partition_states(self._params, active = [3])
# print "Querying map view again"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count)
self.assertEqual(len(view_result["rows"]), total_doc_count, "number of rows returned is %d" % total_doc_count)
common.test_keys_sorted(view_result)
self.assertEqual(view_result["rows"][0]["key"], new_doc2["integer"], "new_doc2 reflected in result set at first row")
self.assertEqual(view_result["rows"][-1]["key"], new_doc1["integer"], "new_doc1 reflected in result set at last row")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
def do_test_simple_cleanup(self):
# print "Querying view in steady state"
(resp, view_result) = common.query(self._params, "mapview1")
etag = resp.getheader("ETag")
self.assertEqual(
view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(view_result)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Triggering partition 4 cleanup"
common.set_partition_states(self._params, cleanup=[3])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [3],
"right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"],
"partition 3 not in info.update_seqs")
self.assertEqual(info["cleanup_running"], True,
"cleanup process is running")
# print "Querying view again"
(resp2, view_result2) = common.query(self._params, "mapview1")
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(
view_result2["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result2["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(etag2, etag,
"Different Etag after cleanup triggered")
common.test_keys_sorted(view_result2)
all_keys = {}
for r in view_result2["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 4 cleanup triggered" %
key)
# print "Triggering view compaction and querying view again"
common.compact_set_view(self._params)
(resp3, view_result3) = common.query(self._params, "mapview1")
etag3 = resp3.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result3["total_rows"], expected,
"Query returned %d total_rows" % expected)
self.assertEqual(len(view_result3["rows"]), expected,
"Query returned %d rows" % expected)
self.assertEqual(etag2, etag3, "Same Etag after cleanup finished")
common.test_keys_sorted(view_result3)
all_keys = {}
for r in view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 4 cleanup finished" %
key)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"],
"partition 3 not in info.update_seqs")
# print "Adding 2 new documents to partition 4"
server = self._params["server"]
db4 = server[self._params["setname"] + "/3"]
new_doc1 = {
"_id": "999999999",
"integer": 999999999,
"string": "999999999"
}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db4.save(new_doc1)
db4.save(new_doc2)
# print "Querying view again"
(resp4, view_result4) = common.query(self._params, "mapview1")
etag4 = resp4.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result4["total_rows"], expected,
"Query returned %d total_rows" % expected)
self.assertEqual(len(view_result4["rows"]), expected,
"Query returned %d rows" % expected)
self.assertEqual(
etag4, etag3,
"Same etag after adding new documents to cleaned partition")
common.test_keys_sorted(view_result4)
all_keys = {}
for r in view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 4 cleanup finished" %
key)
self.assertFalse(new_doc1["integer"] in all_keys,
"new_doc1 not in query result after cleanup")
self.assertFalse(new_doc2["integer"] in all_keys,
"new_doc2 not in query result after cleanup")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"],
"partition 3 not in info.update_seqs")
# print "Triggering compaction again and verifying it doesn't crash"
common.compact_set_view(self._params)
(resp5, view_result5) = common.query(self._params, "mapview1")
etag5 = resp5.getheader("ETag")
self.assertEqual(etag5, etag4, "Same etag after second compaction")
self.assertEqual(view_result5, view_result4,
"Same query results after second compaction")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"],
"partition 3 not in info.update_seqs")
def test_update_cleanup(self):
# print "Triggering initial view update"
t0 = time.time()
(resp, view_result) = common.query(self._params, "mapview",
{"limit": "1"})
# print "Update took %.2f seconds" % (time.time() - t0)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
stats = info["stats"]
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
self.assertEqual(stats["full_updates"], 1, "1 full update done so far")
self.assertEqual(stats["stopped_updates"], 0,
"no updater interruptions so far")
self.assertEqual(stats["cleanups"], 0, "0 cleanups done")
self.assertEqual(stats["cleanup_interruptions"], 0,
"no cleanup interruptions so far")
# print "Adding new partitions 5, 6, 7 and 8 and marking partitions 1 and 4 for cleanup"
common.set_partition_states(self._params,
active=[4, 5, 6, 7],
cleanup=[0, 3])
# print "Querying view (should trigger update + cleanup)"
t0 = time.time()
(resp, view_result) = common.query(self._params, "mapview")
t1 = time.time()
# print "Verifying query results"
expected_total = common.set_doc_count(self._params, [1, 2, 4, 5, 6, 7])
self.assertEqual(view_result["total_rows"], expected_total,
"total rows is %d" % expected_total)
self.assertEqual(len(view_result["rows"]), expected_total,
"got %d tows" % expected_total)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d (partition 1) not in query result after update+cleanup"
% key)
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d (partition 2) in query result after update+cleanup" %
key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d (partition 3) in query result after update+cleanup" %
key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d (partition 4) not in query result after update+cleanup"
% key)
for key in xrange(5, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d (partition 5) in query result after update+cleanup" %
key)
for key in xrange(6, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d (partition 6) in query result after update+cleanup" %
key)
for key in xrange(7, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d (partition 7) in query result after update+cleanup" %
key)
for key in xrange(8, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d (partition 8) in query result after update+cleanup" %
key)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
stats = info["stats"]
self.assertEqual(info["active_partitions"], [1, 2, 4, 5, 6, 7],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
self.assertEqual(stats["full_updates"], 2,
"2 full updates done so far")
self.assertEqual(stats["stopped_updates"], 0,
"no updater interruptions so far")
self.assertEqual(stats["cleanups"], 1, "1 full cleanup done")
self.assertEqual(stats["cleanup_interruptions"], 1,
"1 cleanup interruption done")
for i in info["active_partitions"]:
expected_seq = common.partition_update_seq(self._params, i)
self.assertTrue(
str(i) in info["update_seqs"],
"partition %d in info.update_seqs" % (i + 1))
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right seq in info.update_seqs[%d]" % i)
def do_test_set_passive_during_compaction(self):
# print "Querying map view"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
self.assertEqual(map_view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(map_view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(map_view_result)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
db = self._params["server"][self._params["setname"] + "/" + str(i)]
seq = db.info()["update_seq"]
self.assertEqual(info["update_seqs"][str(i)], seq,
"right update seq for partition %d" % (i + 1))
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Marking partition 4 as passive"
common.set_partition_states(self._params, passive = [3])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Waiting for compaction to finish"
count = 0
while info["stats"]["compactions"] < 1:
time.sleep(0.5)
count += 1
info = common.get_set_view_info(self._params)
self.assertTrue((count > 0), "Compaction was running when the partition states were updated")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying map view again"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
expected = common.set_doc_count(self._params, [0, 1, 2])
self.assertEqual(map_view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(map_view_result["rows"]), expected,
"Query returned %d rows" % expected)
common.test_keys_sorted(map_view_result)
all_keys = {}
for r in map_view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 set to passive" % key)
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Adding two new partitions, 5 and 6, as passive while compaction is running"
common.set_partition_states(self._params, passive = [4, 5])
# print "Waiting for compaction to finish"
info = common.get_set_view_info(self._params)
count = 0
while info["stats"]["compactions"] < 2:
time.sleep(0.5)
count += 1
info = common.get_set_view_info(self._params)
self.assertTrue((count > 0), "Compaction was running when the partition states were updated")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3, 4, 5], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5]:
self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i)
for i in [6, 7]:
self.assertFalse(str(i) in info["update_seqs"], "%d not in info.update_seqs" % i)
# print "Querying map view again"
(map_resp, map_view_result2) = common.query(self._params, "mapview1")
self.assertEqual(map_view_result2["rows"], map_view_result["rows"], "Same result set as before")
total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
# print "Adding 1 new document to each partition"
new_docs = []
for i in [0, 1, 2, 3, 4, 5, 6, 7]:
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
self.assertEqual(new_total_doc_count, (total_doc_count + 8), "8 documents added")
self.assertEqual(len(new_docs), 8, "8 documents added")
info = common.get_set_view_info(self._params)
if info["updater_running"]:
# print "Waiting for updater to finish"
self.assertEqual(info["updater_state"], "updating_passive",
"updater state is updating_passive")
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
self.assertEqual(info["updater_state"], "updating_passive",
"updater state is updating_passive")
time.sleep(3)
else:
break
expected_row_count = common.set_doc_count(self._params, [0, 1, 2])
expected_total_rows = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5])
# print "Querying map view again"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
self.assertEqual(len(map_view_result["rows"]), expected_row_count, "len(rows) is %d" % expected_row_count)
common.test_keys_sorted(map_view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3, 4, 5], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5]:
self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i)
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"info.update_seqs[%d] is %d" % (i, expected_seq))
def test_update_cleanup(self):
# print "Triggering initial view update"
t0 = time.time()
(resp, view_result) = common.query(self._params, "mapview", {"limit": "1"})
# print "Update took %.2f seconds" % (time.time() - t0)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
stats = info["stats"]
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
self.assertEqual(stats["full_updates"], 1, "1 full update done so far")
self.assertEqual(stats["stopped_updates"], 0, "no updater interruptions so far")
self.assertEqual(stats["cleanups"], 0, "0 cleanups done")
self.assertEqual(stats["cleanup_interruptions"], 0, "no cleanup interruptions so far")
# print "Adding new partitions 5, 6, 7 and 8 and marking partitions 1 and 4 for cleanup"
common.set_partition_states(self._params, active=[4, 5, 6, 7], cleanup=[0, 3])
# print "Querying view (should trigger update + cleanup)"
t0 = time.time()
(resp, view_result) = common.query(self._params, "mapview")
t1 = time.time()
# print "Verifying query results"
expected_total = common.set_doc_count(self._params, [1, 2, 4, 5, 6, 7])
self.assertEqual(view_result["total_rows"], expected_total, "total rows is %d" % expected_total)
self.assertEqual(len(view_result["rows"]), expected_total, "got %d tows" % expected_total)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys, "Key %d (partition 1) not in query result after update+cleanup" % key)
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys, "Key %d (partition 2) in query result after update+cleanup" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys, "Key %d (partition 3) in query result after update+cleanup" % key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys, "Key %d (partition 4) not in query result after update+cleanup" % key)
for key in xrange(5, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys, "Key %d (partition 5) in query result after update+cleanup" % key)
for key in xrange(6, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys, "Key %d (partition 6) in query result after update+cleanup" % key)
for key in xrange(7, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys, "Key %d (partition 7) in query result after update+cleanup" % key)
for key in xrange(8, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys, "Key %d (partition 8) in query result after update+cleanup" % key)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
stats = info["stats"]
self.assertEqual(info["active_partitions"], [1, 2, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
self.assertEqual(stats["full_updates"], 2, "2 full updates done so far")
self.assertEqual(stats["stopped_updates"], 0, "no updater interruptions so far")
self.assertEqual(stats["cleanups"], 1, "1 full cleanup done")
self.assertEqual(stats["cleanup_interruptions"], 1, "1 cleanup interruption done")
for i in info["active_partitions"]:
expected_seq = common.partition_update_seq(self._params, i)
self.assertTrue(str(i) in info["update_seqs"], "partition %d in info.update_seqs" % (i + 1))
self.assertEqual(info["update_seqs"][str(i)], expected_seq, "right seq in info.update_seqs[%d]" % i)
def do_test_set_cleanup_partitions_when_updater_is_running(self):
# print "Marking all partitions for cleanup"
common.set_partition_states(self._params, cleanup = range(self._params["nparts"]))
# print "Compacting the set view group"
common.compact_set_view(self._params)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
self.assertEqual(view_result["total_rows"], 0, "Empty view result")
self.assertEqual(len(view_result["rows"]), 0, "Empty view result")
# print "Marking all partitions as active"
common.set_partition_states(self._params, active = range(self._params["nparts"]))
# print "Querying view with ?stale=update_after"
(resp, view_result) = common.query(self._params, "mapview1", {"stale": "update_after"})
self.assertEqual(view_result["total_rows"], 0, "Empty view result")
self.assertEqual(len(view_result["rows"]), 0, "Empty view result")
# print "Marking partition 2 for cleanup while the updater is running"
common.set_partition_states(self._params, cleanup = [1])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [1], "right cleanup partitions list")
self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs")
# print "Waiting for the set view updater to finish"
iterations = 0
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
iterations += 1
else:
break
self.assertTrue(iterations > 0, "Updater was running when partition 2 was marked for cleanup")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertTrue(["cleanup_partitions"] == [1] or info["cleanup_partitions"] == [],
"cleanup partitions list is not wrong")
self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs")
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 2, 3])
self.assertEqual(len(view_result["rows"]), doc_count, "Query returned %d rows" % doc_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 2 marked for cleanup" % key)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs")
def do_test_reduce(self, viewname):
# print "Querying reduce view in steady state"
(resp, view_result) = common.query(self._params, viewname)
etag = resp.getheader("ETag")
self.assertEqual(len(view_result["rows"]), 1, "Query returned 1 row")
self.assertEqual(view_result["rows"][0]["value"], self._params["ndocs"],
"Non-grouped reduce value is %d" % self._params["ndocs"])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Setting partition 3 to passive state"
common.set_partition_states(self._params, passive = [2])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [2], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying reduce view again"
(resp2, view_result2) = common.query(self._params, viewname)
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(len(view_result2["rows"]), 1, "Query returned 1 row")
self.assertEqual(view_result2["rows"][0]["value"], expected,
"Non-grouped reduce value is %d" % expected)
self.assertNotEqual(etag2, etag, "Different ETags")
# print "Querying view with ?group=true"
(resp3, view_result3) = common.query(self._params, viewname, {"group": "true"})
etag3 = resp3.getheader("ETag")
self.assertEqual(len(view_result3["rows"]), expected, "Query returned % rows" % expected)
self.assertNotEqual(etag3, etag, "Different ETags")
self.assertEqual(etag3, etag2, "Equal ETags for responses 2 and 3")
common.test_keys_sorted(view_result3)
all_keys = {}
for r in view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 3 was made passive" % key)
# print "Querying view with ?group=true&descending=true"
(resp4, view_result4) = common.query(
self._params, viewname, {"group": "true", "descending": "true"})
etag4 = resp4.getheader("ETag")
self.assertEqual(len(view_result4["rows"]), expected, "Query returned % rows" % expected)
self.assertNotEqual(etag4, etag, "Different ETags")
self.assertEqual(etag4, etag3, "Equal ETags for responses 3 and 4")
common.test_keys_sorted(view_result4, lambda a, b: a > b)
all_keys = {}
for r in view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 3 was made passive" % key)
# print "Querying view with ?group=true&startkey=3333&endkey=44781"
(resp5, view_result5) = common.query(
self._params, viewname,
{"group": "true", "startkey": "3333", "endkey": "44781"})
etag5 = resp5.getheader("ETag")
self.assertNotEqual(etag5, etag, "Different ETags")
self.assertEqual(etag5, etag4, "Equal ETags for responses 4 and 5")
common.test_keys_sorted(view_result5)
self.assertTrue(view_result5["rows"][0]["key"] >= 3333, "First key is >= 3333")
self.assertTrue(view_result5["rows"][-1]["key"] <= 44781, "Last key is <= 44781")
all_keys = {}
for r in view_result5["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 3 was made passive" % key)
# print "Querying view with ?group=true&startkey=44781&endkey=3333&descending=true"
(resp6, view_result6) = common.query(
self._params, viewname,
{"group": "true", "startkey": "44781", "endkey": "3333", "descending": "true"})
etag6 = resp6.getheader("ETag")
self.assertNotEqual(etag6, etag, "Different ETags")
self.assertEqual(etag6, etag5, "Equal ETags for responses 5 and 6")
common.test_keys_sorted(view_result6, lambda a, b: a > b)
self.assertTrue(view_result6["rows"][0]["key"] <= 44781, "First key is <= 44781")
self.assertTrue(view_result6["rows"][-1]["key"] >= 3333, "Last key is >= 3333")
self.assertEqual(len(view_result6["rows"]), len(view_result5["rows"]),
"Same number of rows for responses 5 and 6")
all_keys = {}
for r in view_result6["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 3 was made passive" % key)
# print "Setting partition 3 to active state"
common.set_partition_states(self._params, active = [2])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view with ?group=true"
(resp7, view_result7) = common.query(self._params, viewname, {"group": "true"})
etag7 = resp7.getheader("ETag")
self.assertEqual(len(view_result7["rows"]), self._params["ndocs"],
"Query returned % rows" % self._params["ndocs"])
self.assertEqual(etag7, etag, "Same etags for responses 1 and 7")
common.test_keys_sorted(view_result7)
# print "Querying view with ?group=true&descending=true"
(resp8, view_result8) = common.query(
self._params, viewname, {"group": "true", "descending": "true"})
etag8 = resp8.getheader("ETag")
self.assertEqual(len(view_result8["rows"]), self._params["ndocs"],
"Query returned % rows" % self._params["ndocs"])
self.assertEqual(etag7, etag8, "Same etags for responses 7 and 8")
common.test_keys_sorted(view_result8, lambda a, b: a > b)
# print "Querying view with ?group=true&startkey=3333&endkey=44781"
(resp9, view_result9) = common.query(
self._params, viewname,
{"group": "true", "startkey": "3333", "endkey": "44781"})
etag9 = resp9.getheader("ETag")
self.assertEqual(etag9, etag8, "Equal ETags for responses 8 and 9")
common.test_keys_sorted(view_result9)
self.assertTrue(view_result9["rows"][0]["key"] >= 3333, "First key is >= 3333")
self.assertTrue(view_result9["rows"][-1]["key"] <= 44781, "Last key is <= 44781")
# print "Querying view with ?group=true&startkey=44781&endkey=3333&descending=true"
(resp10, view_result10) = common.query(
self._params, viewname,
{"group": "true", "startkey": "44781", "endkey": "3333", "descending": "true"})
etag10 = resp10.getheader("ETag")
self.assertEqual(etag10, etag9, "Equal ETags for responses 9 and 10")
common.test_keys_sorted(view_result10, lambda a, b: a > b)
self.assertTrue(view_result10["rows"][0]["key"] <= 44781, "First key is <= 44781")
self.assertTrue(view_result10["rows"][-1]["key"] >= 3333, "Last key is >= 3333")
self.assertEqual(len(view_result10["rows"]), len(view_result9["rows"]),
"Same number of rows for responses 9 and 10")
def do_test_change_partition_states_while_cleanup_running(self):
# print "Marking all partitions as active"
common.set_partition_states(self._params, active = range(self._params["nparts"]))
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
# print "Updating view"
(resp, view_result) = common.query(self._params, "mapview1", {"limit": "100"})
self.assertEqual(view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), 100, "Query returned 100 rows")
common.test_keys_sorted(view_result)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
# print "Marking partitions 1 and 2 for cleanup"
common.set_partition_states(self._params, cleanup = [0, 1])
info = common.get_set_view_info(self._params)
self.assertEqual(info["cleanup_running"], True, "cleanup is running")
self.assertEqual(info["active_partitions"], [2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [0, 1], "right cleanup partitions list")
# print "Marking partitions 1 and 2 as active while cleanup is ongoing"
common.set_partition_states(self._params, active = [0, 1])
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
info = common.get_set_view_info(self._params)
self.assertEqual(view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), doc_count, "Query returned %d rows" % doc_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d in result after partition 1 activated" % key)
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d in result after partition 2 activated" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 3) in result set" % key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 4) in result set" % key)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
def do_test_maps(self):
# print "Querying map view in steady state"
(resp, view_result) = common.query(self._params, "mapview1")
etag = resp.getheader("ETag")
self.assertEqual(view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(view_result)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Setting partition 4 as passive"
common.set_partition_states(self._params, passive = [3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp2, view_result2) = common.query(self._params, "mapview1")
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result2["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result2["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(etag2, etag, "Different Etag after setting passive partition")
common.test_keys_sorted(view_result2)
all_keys = {}
for r in view_result2["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys, "Key %d not in result after partition 4 was made passive" % key)
# print "Setting partition 4 state to active"
common.set_partition_states(self._params, active = [3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp3, view_result3) = common.query(self._params, "mapview1")
etag3 = resp3.getheader("ETag")
self.assertEqual(view_result3["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result3["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
self.assertEqual(etag3, etag, "ETag is same as first query response")
common.test_keys_sorted(view_result3)
# print "Setting partitions 1 and 4 to passive state"
common.set_partition_states(self._params, passive = [0, 3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [0, 3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp4, view_result4) = common.query(self._params, "mapview1")
etag4 = resp4.getheader("ETag")
expected = self._params["ndocs"] / 2
self.assertEqual(view_result4["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result4["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(etag4, etag, "ETag is different from all previous responses")
self.assertNotEqual(etag4, etag2, "ETag is different from all previous responses")
common.test_keys_sorted(view_result4)
all_keys = {}
for r in view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 1 was made passive" % key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 was made passive" % key)
# print "Marking all partitions as passive and querying view again"
common.set_partition_states(self._params, passive = range(self._params["nparts"]))
(resp5, view_result5) = common.query(self._params, "mapview1")
etag5 = resp5.getheader("ETag")
self.assertEqual(view_result5["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result5["rows"]), 0, "Query returned 0 rows")
self.assertNotEqual(etag5, etag, "ETag is different from all previous responses")
self.assertNotEqual(etag5, etag2, "ETag is different from all previous responses")
self.assertNotEqual(etag5, etag3, "ETag is different from all previous responses")
self.assertNotEqual(etag5, etag4, "ETag is different from all previous responses")
# print "Setting all partitions to active state"
common.set_partition_states(self._params, active = range(self._params["nparts"]))
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp6, view_result6) = common.query(self._params, "mapview1")
etag6 = resp6.getheader("ETag")
self.assertEqual(view_result6["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result6["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
self.assertEqual(etag6, etag, "ETag is the same from first view query response")
common.test_keys_sorted(view_result6)
def do_test_simple_cleanup(self):
# print "Querying view in steady state"
(resp, view_result) = common.query(self._params, "mapview1")
etag = resp.getheader("ETag")
self.assertEqual(view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(view_result)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Triggering partition 4 cleanup"
common.set_partition_states(self._params, cleanup = [3])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [3], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs")
self.assertEqual(info["cleanup_running"], True, "cleanup process is running")
# print "Querying view again"
(resp2, view_result2) = common.query(self._params, "mapview1")
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result2["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result2["rows"]), expected, "Query returned %d rows" % expected)
self.assertNotEqual(etag2, etag, "Different Etag after cleanup triggered")
common.test_keys_sorted(view_result2)
all_keys = {}
for r in view_result2["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 cleanup triggered" % key)
# print "Triggering view compaction and querying view again"
common.compact_set_view(self._params)
(resp3, view_result3) = common.query(self._params, "mapview1")
etag3 = resp3.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result3["total_rows"], expected, "Query returned %d total_rows" % expected)
self.assertEqual(len(view_result3["rows"]), expected, "Query returned %d rows" % expected)
self.assertEqual(etag2, etag3, "Same Etag after cleanup finished")
common.test_keys_sorted(view_result3)
all_keys = {}
for r in view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 cleanup finished" % key)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs")
# print "Adding 2 new documents to partition 4"
server = self._params["server"]
db4 = server[self._params["setname"] + "/3"]
new_doc1 = {"_id": "999999999", "integer": 999999999, "string": "999999999"}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db4.save(new_doc1)
db4.save(new_doc2)
# print "Querying view again"
(resp4, view_result4) = common.query(self._params, "mapview1")
etag4 = resp4.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result4["total_rows"], expected, "Query returned %d total_rows" % expected)
self.assertEqual(len(view_result4["rows"]), expected, "Query returned %d rows" % expected)
self.assertEqual(etag4, etag3, "Same etag after adding new documents to cleaned partition")
common.test_keys_sorted(view_result4)
all_keys = {}
for r in view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 cleanup finished" % key)
self.assertFalse(new_doc1["integer"] in all_keys, "new_doc1 not in query result after cleanup")
self.assertFalse(new_doc2["integer"] in all_keys, "new_doc2 not in query result after cleanup")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs")
# print "Triggering compaction again and verifying it doesn't crash"
common.compact_set_view(self._params)
(resp5, view_result5) = common.query(self._params, "mapview1")
etag5 = resp5.getheader("ETag")
self.assertEqual(etag5, etag4, "Same etag after second compaction")
self.assertEqual(view_result5, view_result4, "Same query results after second compaction")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs")
def do_test_view_updates(self):
# print "Setting partition 2 state to passive"
common.set_partition_states(self._params, passive=[1])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [1],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Adding 2 new documents to partition 2"
server = self._params["server"]
db2 = server[self._params["setname"] + "/1"]
new_doc1 = {
"_id": "999999999",
"integer": 999999999,
"string": "999999999"
}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db2.save(new_doc1)
db2.save(new_doc2)
# print "Querying map view"
(resp, view_result) = common.query(self._params, "mapview1")
etag = resp.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(len(view_result["rows"]), expected,
"Query returned %d rows" % expected)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 2 was made passive" %
key)
self.assertFalse(new_doc1["integer"] in all_keys,
"new_doc1 not reflected in view")
self.assertFalse(new_doc2["integer"] in all_keys,
"new_doc2 not reflected in view")
# print "Setting partition 2 state to active"
common.set_partition_states(self._params, active=[1])
# print "Querying map view again"
(resp2, view_result2) = common.query(self._params, "mapview1")
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] + 2
self.assertEqual(view_result2["total_rows"], expected,
"Query returned %d total_rows" % expected)
self.assertEqual(len(view_result2["rows"]), expected,
"Query returned %d rows" % expected)
common.test_keys_sorted(view_result2)
all_keys = {}
for r in view_result2["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d in result after partition 2 was re-enabled" % key)
self.assertTrue(new_doc1["integer"] in all_keys,
"new_doc1 reflected in view")
self.assertTrue(new_doc2["integer"] in all_keys,
"new_doc2 reflected in view")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
if i == 1:
seq = (self._params["ndocs"] / 4) + 2
else:
seq = (self._params["ndocs"] / 4)
self.assertEqual(info["update_seqs"][str(i)], seq,
"right update seq for partition %d" % (i + 1))
def test_filter_partitions(self):
# print "Querying map view with ?partitions=[0,1,3]"
(map_resp, map_view_result) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag = map_resp.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(map_view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(map_view_result["rows"]), expected, "Query returned %d rows" % expected)
common.test_keys_sorted(map_view_result)
all_keys = {}
for r in map_view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys, "Key %d from partition 3 is not in the result" % key)
# print "Disabling (making it passive) partition 2"
common.set_partition_states(self._params, passive = [1])
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp2, map_view_result2) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag2 = map_resp2.getheader("ETag")
self.assertEqual(map_view_result2, map_view_result, "Same result as before")
self.assertEqual(map_etag2, map_etag, "Same Etag as before")
# print "Marking partition 2 for cleanup"
common.set_partition_states(self._params, cleanup = [1])
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp3, map_view_result3) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag3 = map_resp3.getheader("ETag")
expected = self._params["ndocs"] / 2
self.assertEqual(len(map_view_result3["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(map_etag3, map_etag2, "Different Etag from before")
common.test_keys_sorted(map_view_result3)
all_keys = {}
for r in map_view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d from partition 2 is not in the result" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d from partition 3 is not in the result" % key)
# print "Triggering view compaction (to guarantee cleanup is complete)"
common.compact_set_view(self._params)
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp4, map_view_result4) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag4 = map_resp4.getheader("ETag")
# total_rows is different after cleanup
self.assertEquals(map_view_result4["rows"], map_view_result3["rows"], "Same result as before")
self.assertEquals(map_etag4, map_etag3, "Same Etag as before")
def test_compaction(self):
# print "Querying map view"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
map_etag = map_resp.getheader("ETag")
self.assertEqual(map_view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(map_view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(map_view_result)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying reduce view"
(red_resp, red_view_result) = common.query(self._params, "redview1")
red_etag = red_resp.getheader("ETag")
self.assertEqual(len(red_view_result["rows"]), 1, "Query returned 1 row")
self.assertEqual(red_view_result["rows"][0]["value"], self._params["ndocs"],
"Non-grouped reduce value is %d" % self._params["ndocs"])
# print "Triggering view compaction"
common.compact_set_view(self._params)
# print "Verifying set view group info"
info2 = common.get_set_view_info(self._params)
self.assertEqual(info2["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info2["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info2["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info2["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertTrue(info2["disk_size"] < info["disk_size"], "Smaller disk_size after compaction")
# print "Querying map view"
(map_resp2, map_view_result2) = common.query(self._params, "mapview1")
map_etag2 = map_resp2.getheader("ETag")
self.assertEqual(map_view_result2, map_view_result, "Got same map response after compaction")
self.assertEqual(map_etag2, map_etag, "Same map view etag after compaction")
# print "Querying reduce view"
(red_resp2, red_view_result2) = common.query(self._params, "redview1")
red_etag2 = red_resp2.getheader("ETag")
self.assertEqual(red_view_result2, red_view_result, "Got same reduce response after compaction")
self.assertEqual(red_etag2, red_etag, "Same reduce view etag after compaction")
# print "Adding 2 new documents to partition 4"
server = self._params["server"]
db4 = server[self._params["setname"] + "/3"]
new_doc1 = {"_id": "9999999999999", "integer": 999999999999999999, "string": "999999999"}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db4.save(new_doc1)
db4.save(new_doc2)
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertEqual(info["compact_running"], True, "Compaction is flagged as running in group info")
# print "Triggering set view group index update"
common.query(self._params, "redview1")
# print "Waiting for set view compaction to finish"
compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0)
self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
if i == 3:
seq = (self._params["ndocs"] / 4) + 2
else:
seq = (self._params["ndocs"] / 4)
self.assertEqual(info["update_seqs"][str(i)], seq, "right update seq for partition %d" % (i + 1))
# print "Querying map view"
(map_resp3, map_view_result3) = common.query(self._params, "mapview1")
map_etag3 = map_resp3.getheader("ETag")
self.assertEqual(map_view_result3["total_rows"], (self._params["ndocs"] + 2),
"Query returned %d total_rows" % (self._params["ndocs"] + 2))
self.assertEqual(len(map_view_result3["rows"]), (self._params["ndocs"] + 2),
"Query returned %d rows" % (self._params["ndocs"] + 2))
self.assertNotEqual(map_etag3, map_etag, "Different map view etag after recompaction")
common.test_keys_sorted(map_view_result3)
self.assertEqual(map_view_result3["rows"][0]["id"], new_doc2["_id"], "new_doc2 in map view")
self.assertEqual(map_view_result3["rows"][-1]["id"], new_doc1["_id"], "new_doc1 in map view")
# print "Querying reduce view"
(red_resp3, red_view_result3) = common.query(self._params, "redview1")
red_etag3 = red_resp3.getheader("ETag")
self.assertEqual(red_view_result3["rows"][0]["value"], (self._params["ndocs"] + 2),
"Non-grouped reduce value is %d" % (self._params["ndocs"] + 2))
self.assertNotEqual(red_etag3, red_etag, "Different reduce view etag after compaction")
# print "Triggering another view compaction"
common.compact_set_view(self._params, False)
# print "Triggering partition 4 cleanup while compaction is ongoing"
common.set_partition_states(self._params, cleanup = [3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [3], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "paritition 4 not in group info update_seqs")
self.assertEqual(info["compact_running"], True, "Compaction is flagged as running in group info")
self.assertEqual(info["cleanup_running"], True, "Cleanup is flagged as running in group info")
compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0)
self.assertTrue(compaction_was_running, "Compaction was running when the cleanup was triggered")
# print "Verifying set view group info"
info2 = common.get_set_view_info(self._params)
self.assertEqual(info2["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info2["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info2["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info2["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "paritition 4 not in group info update_seqs")
self.assertTrue(info2["disk_size"] < info["disk_size"], "Smaller disk_size after compaction+cleanup")
# print "Querying map view"
(map_resp4, map_view_result4) = common.query(self._params, "mapview1")
map_etag4 = map_resp4.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(map_view_result4["total_rows"], expected,
"Query returned %d total_rows after recompact+cleanup" % expected)
self.assertEqual(len(map_view_result4["rows"]), expected,
"Query returned %d rows after recompact+cleanup" % expected)
self.assertNotEqual(map_etag4, map_etag, "Different map view etag after compaction+cleanup")
common.test_keys_sorted(map_view_result4)
all_keys = {}
for r in map_view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 cleanup triggered" % key)
self.assertFalse(new_doc1["integer"] in all_keys, "new_doc1 not included in view query result")
self.assertFalse(new_doc2["integer"] in all_keys, "new_doc2 not included in view query result")
self.assertNotEqual(map_view_result4, map_view_result3, "Different map view result after recompact+cleanup")
# print "Querying reduce view"
(red_resp4, red_view_result4) = common.query(self._params, "redview1")
red_etag4 = red_resp4.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(red_view_result4["rows"][0]["value"], expected,
"Non-grouped reduce value is %d after recompact+cleanup" % (self._params["ndocs"] + 2))
self.assertNotEqual(red_etag4, red_etag3, "Different reduce view etag after compaction+cleanup")
def test_many_partitions(self):
total_doc_count = common.set_doc_count(self._params, range(self._params["nparts"]))
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], range(self._params["nparts"]), "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
# print "Adding 1 new document to each partition"
new_docs = []
for i in xrange(self._params["nparts"]):
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(self._params, range(self._params["nparts"]))
self.assertEqual(new_total_doc_count, (total_doc_count + len(new_docs)), "N documents were added")
total_doc_count = new_total_doc_count
# print "Querying view again"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], range(self._params["nparts"]), "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
# print "Marking half of the partitions as passive"
passive = range(self._params["nparts"] / 2, self._params["nparts"])
active = range(self._params["nparts"] / 2)
common.set_partition_states(
self._params,
passive = range(self._params["nparts"] / 2, self._params["nparts"])
)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], active, "right active partitions list")
self.assertEqual(info["passive_partitions"], passive, "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
# print "Querying view again"
(resp, view_result) = common.query(self._params, "mapview")
expected_row_count = common.set_doc_count(self._params, active)
self.assertEqual(len(view_result["rows"]), expected_row_count, "number of received rows is %d" % expected_row_count)
common.test_keys_sorted(view_result)
for row in view_result['rows']:
if row["key"] >= 2001:
key_part = ((row["key"] - 2000) % self._params["nparts"]) - 1
else:
key_part = (row["key"] % self._params["nparts"]) - 1
self.assertTrue(key_part in active, "Key %d from passive partition not in result set" % row["key"])
# print "Marking half of the partitions for cleanup"
common.set_partition_states(
self._params,
cleanup = passive
)
cleanup = passive
common.compact_set_view(self._params)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], active, "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
self.assertEqual(info["stats"]["compactions"], 1, "1 compaction")
self.assertEqual(info["stats"]["cleanups"], 1, "1 full cleanup")
for i in active:
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
