def do_test_with_updates(self, state_transitions):
# start from a clean state always
common.set_partition_states(
params = self._params,
cleanup = range(self._params["nparts"])
)
# print "Cleaning set view via compaction"
common.compact_set_view(self._params)
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(view_result["total_rows"], 0, "total_rows is 0 after compaction")
self.assertEqual(len(view_result["rows"]), 0, "0 rows returned after compaction")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
self.assertEqual(info["update_seqs"], {}, "right update seqs list")
# print "Triggering state changes with queries (?stale=updater_after) in between"
for state in state_transitions:
common.set_partition_states(
params = self._params,
active = state["active"],
passive = state["passive"],
cleanup = state["cleanup"]
)
time.sleep(0.5)
common.query(self._params, "mapview", {"stale": "update_after", "limit": "100"})
group_stats = self.verify_final_index_state(state_transitions)
self.assertTrue(group_stats["updater_interruptions"] > 0, "Got at least 1 updater interruption")
self.assertEqual(group_stats["updates"], 2, "Got 2 full updates")
self.assertEqual(group_stats["compactions"], 1, "Got 1 compaction")
self.assertTrue(group_stats["cleanups"] >= 1, "Got at least 1 full cleanup")
def do_test_set_active_during_compaction(self):
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Marking partitions 4, 5 and 6 as active while compaction is running"
common.set_partition_states(self._params, active = [3, 4, 5])
# print "Adding new partitions 7 and 8 with active state while compaction is running"
common.set_partition_states(self._params, active = [6, 7])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying map view while compaction is running"
(map_resp, map_view_result) = common.query(self._params, "mapview1", {"limit": "10"})
self.assertEqual(len(map_view_result["rows"]), 10, "Query returned 10 rows")
common.test_keys_sorted(map_view_result)
# print "Waiting for compaction to finish"
compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0)
self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying map view again"
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
(map_resp, map_view_result) = common.query(self._params, "mapview1")
self.assertEqual(map_view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(map_view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(map_view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5, 6, 7]:
self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i)
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"info.update_seqs[%d] is %d" % (i, expected_seq))
def verify_final_index_state(self, state_transitions):
# print "Verifying final view state"
doc_count = common.set_doc_count(self._params)
(resp, view_result) = common.query(self._params, "mapview")
# print "Final view result (%d rows): %s" % \
# (len(view_result["rows"]), json.dumps(view_result, sort_keys = True, indent = 4))
self.assertEqual(view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
self.assertEqual(view_result["rows"][0]["key"], 1, "First key is 1")
self.assertEqual(view_result["rows"][-1]["key"], doc_count, "Last key is %d" % doc_count)
common.test_keys_sorted(view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
# print "Final group info: %s" % json.dumps(info, sort_keys = True, indent = 4)
self.assertEqual(
info["active_partitions"],
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
"right active partitions list"
)
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]:
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
return info["stats"]
def __init__(self, table, odIDFields, places, placeIDField, location, **kwargs):
TableLinker.__init__(self, table, odIDFields, places, placeIDField, location, **kwargs)
self.placeMapper.setIDTransfer(True)
self.placeIDQPart = common.query(self.places, '[%s] = ', self.placesIDField)
self.placeFC = self.places
if not common.isLayer(self.places):
common.progress('creating place selection layer')
self.places = arcpy.MakeFeatureLayer_management(self.placeFC, self.PLACES_LAY).getOutput(0)
def do_test_keys_param(self):
# print "Running test do_test_keys_param"
keys = [1111, 4, self._params["ndocs"] * 5, 7]
# print "Querying map view with ?keys=%s" % json.dumps(keys)
(map_resp2,
map_view_result2) = common.query(self._params, "mapview1", {
"include_docs": "true",
"keys": json.dumps(keys)
})
self.assertEqual(len(map_view_result2["rows"]), 3,
"Query returned 3 rows")
all_keys = {}
for row in map_view_result2["rows"]:
all_keys[row["key"]] = True
for key in keys:
if key > self._params["ndocs"]:
self.assertFalse(key in all_keys, "Key %d not in result" % key)
else:
self.assertTrue(key in all_keys, "Key %d in result" % key)
# print "Querying reduce view with ?keys=%s" % json.dumps(keys)
(red_resp, red_view_result) = common.query(self._params, "redview1", {
"keys": json.dumps(keys),
"group": "true"
})
self.assertEqual(len(red_view_result["rows"]), 3,
"Query returned 3 rows")
all_keys = {}
for row in red_view_result["rows"]:
all_keys[row["key"]] = row["value"]
for key in keys:
if key > self._params["ndocs"]:
self.assertFalse(key in all_keys, "Key %d not in result" % key)
else:
self.assertTrue(key in all_keys, "Key %d in result" % key)
self.assertEqual(all_keys[key], 1, "Key %d has value 1" % key)
def do_test_with_updates(self, state_transitions):
# start from a clean state always
common.set_partition_states(params=self._params,
cleanup=range(self._params["nparts"]))
# print "Cleaning set view via compaction"
common.compact_set_view(self._params)
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(view_result["total_rows"], 0,
"total_rows is 0 after compaction")
self.assertEqual(len(view_result["rows"]), 0,
"0 rows returned after compaction")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
self.assertEqual(info["update_seqs"], {}, "right update seqs list")
# print "Triggering state changes with queries (?stale=updater_after) in between"
for state in state_transitions:
common.set_partition_states(params=self._params,
active=state["active"],
passive=state["passive"],
cleanup=state["cleanup"])
time.sleep(0.5)
common.query(self._params, "mapview", {
"stale": "update_after",
"limit": "100"
})
group_stats = self.verify_final_index_state(state_transitions)
self.assertTrue(group_stats["stopped_updates"] > 0,
"Got at least 1 updater interruption")
self.assertEqual(group_stats["full_updates"], 2, "Got 2 full updates")
self.assertEqual(group_stats["compactions"], 1, "Got 1 compaction")
self.assertTrue(group_stats["cleanups"] >= 1,
"Got at least 1 full cleanup")
def do_test_keys_param(self):
# print "Running test do_test_keys_param"
keys = [1111, 4, self._params["ndocs"] * 5, 7]
# print "Querying map view with ?keys=%s" % json.dumps(keys)
(map_resp2, map_view_result2) = common.query(
self._params, "mapview1", {"include_docs": "true", "keys": json.dumps(keys)})
self.assertEqual(len(map_view_result2["rows"]), 3, "Query returned 3 rows")
all_keys = {}
for row in map_view_result2["rows"]:
all_keys[row["key"]] = True
for key in keys:
if key > self._params["ndocs"]:
self.assertFalse(key in all_keys, "Key %d not in result" % key)
else:
self.assertTrue(key in all_keys, "Key %d in result" % key)
# print "Querying reduce view with ?keys=%s" % json.dumps(keys)
(red_resp, red_view_result) = common.query(
self._params, "redview1", {"keys": json.dumps(keys), "group": "true"})
self.assertEqual(len(red_view_result["rows"]), 3, "Query returned 3 rows")
all_keys = {}
for row in red_view_result["rows"]:
all_keys[row["key"]] = row["value"]
for key in keys:
if key > self._params["ndocs"]:
self.assertFalse(key in all_keys, "Key %d not in result" % key)
else:
self.assertTrue(key in all_keys, "Key %d in result" % key)
self.assertEqual(all_keys[key], 1, "Key %d has value 1" % key)
def test_include_docs(self):
# print "Querying map view"
(map_resp, map_view_result) = common.query(self._params, "mapview1", {"include_docs": "true"})
map_etag = map_resp.getheader("ETag")
self.assertEqual(map_view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(map_view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(map_view_result)
for row in map_view_result["rows"]:
self.assertTrue("doc" in row, 'row has a "doc" property')
doc = row["doc"]
self.assertEqual(doc["integer"], row["key"], "doc.integer same as row.key")
self.assertEqual(doc["string"], str(row["key"]), "doc.string same as String(row.key)")
def verify_final_index_state(self, state_transitions):
# print "Verifying final view state"
doc_count = common.set_doc_count(self._params)
(resp, view_result) = common.query(self._params, "mapview")
# print "Final view result (%d rows): %s" % \
# (len(view_result["rows"]), json.dumps(view_result, sort_keys = True, indent = 4))
self.assertEqual(view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
self.assertEqual(view_result["rows"][0]["key"], 1, "First key is 1")
self.assertEqual(view_result["rows"][-1]["key"], doc_count,
"Last key is %d" % doc_count)
common.test_keys_sorted(view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
# print "Final group info: %s" % json.dumps(info, sort_keys = True, indent = 4)
self.assertEqual(
info["active_partitions"],
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]:
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
return info["stats"]
def do_test_change_partition_states_while_cleanup_running(self):
# print "Marking all partitions as active"
common.set_partition_states(self._params,
active=range(self._params["nparts"]))
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
# print "Updating view"
(resp, view_result) = common.query(self._params, "mapview1",
{"limit": "100"})
self.assertEqual(view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), 100,
"Query returned 100 rows")
common.test_keys_sorted(view_result)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
# print "Marking partitions 1 and 2 for cleanup"
common.set_partition_states(self._params, cleanup=[0, 1])
info = common.get_set_view_info(self._params)
self.assertEqual(info["cleanup_running"], True, "cleanup is running")
self.assertEqual(info["active_partitions"], [2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [0, 1],
"right cleanup partitions list")
# print "Marking partitions 1 and 2 as active while cleanup is ongoing"
common.set_partition_states(self._params, active=[0, 1])
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
info = common.get_set_view_info(self._params)
self.assertEqual(view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d in result after partition 1 activated" % key)
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d in result after partition 2 activated" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 3) in result set" % key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 4) in result set" % key)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
def do_test_view_updates(self):
# print "Setting partition 2 state to passive"
common.set_partition_states(self._params, passive=[1])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [1],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Adding 2 new documents to partition 2"
server = self._params["server"]
db2 = server[self._params["setname"] + "/1"]
new_doc1 = {
"_id": "999999999",
"integer": 999999999,
"string": "999999999"
}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db2.save(new_doc1)
db2.save(new_doc2)
# print "Querying map view"
(resp, view_result) = common.query(self._params, "mapview1")
etag = resp.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(len(view_result["rows"]), expected,
"Query returned %d rows" % expected)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 2 was made passive" %
key)
self.assertFalse(new_doc1["integer"] in all_keys,
"new_doc1 not reflected in view")
self.assertFalse(new_doc2["integer"] in all_keys,
"new_doc2 not reflected in view")
# print "Setting partition 2 state to active"
common.set_partition_states(self._params, active=[1])
# print "Querying map view again"
(resp2, view_result2) = common.query(self._params, "mapview1")
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] + 2
self.assertEqual(view_result2["total_rows"], expected,
"Query returned %d total_rows" % expected)
self.assertEqual(len(view_result2["rows"]), expected,
"Query returned %d rows" % expected)
common.test_keys_sorted(view_result2)
all_keys = {}
for r in view_result2["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d in result after partition 2 was re-enabled" % key)
self.assertTrue(new_doc1["integer"] in all_keys,
"new_doc1 reflected in view")
self.assertTrue(new_doc2["integer"] in all_keys,
"new_doc2 reflected in view")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
if i == 1:
seq = (self._params["ndocs"] / 4) + 2
else:
seq = (self._params["ndocs"] / 4)
self.assertEqual(info["update_seqs"][str(i)], seq,
"right update seq for partition %d" % (i + 1))
def do_test_change_partition_states_while_cleanup_running(self):
# print "Marking all partitions as active"
common.set_partition_states(self._params, active = range(self._params["nparts"]))
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
# print "Updating view"
(resp, view_result) = common.query(self._params, "mapview1", {"limit": "100"})
self.assertEqual(view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), 100, "Query returned 100 rows")
common.test_keys_sorted(view_result)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
# print "Marking partitions 1 and 2 for cleanup"
common.set_partition_states(self._params, cleanup = [0, 1])
info = common.get_set_view_info(self._params)
self.assertEqual(info["cleanup_running"], True, "cleanup is running")
self.assertEqual(info["active_partitions"], [2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [0, 1], "right cleanup partitions list")
# print "Marking partitions 1 and 2 as active while cleanup is ongoing"
common.set_partition_states(self._params, active = [0, 1])
info = common.get_set_view_info(self._params)
self.assertEqual(type(info["pending_transition"]), dict, "pending_transition is an object")
self.assertEqual(sorted(info["pending_transition"]["active"]),
[0, 1],
"pending_transition active list is [0, 1]")
self.assertEqual(info["pending_transition"]["passive"],
[],
"pending_transition passive list is []")
self.assertEqual(info["pending_transition"]["cleanup"],
[],
"pending_transition cleanup list is []")
# print "Waiting for pending transition to be applied"
iterations = 0
while True:
if iterations > 600:
raise(Exception("timeout waiting for pending transition to be applied"))
info = common.get_set_view_info(self._params)
if info["pending_transition"] is None:
break
else:
time.sleep(1)
iterations += 1
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
info = common.get_set_view_info(self._params)
self.assertEqual(view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), doc_count, "Query returned %d rows" % doc_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d in result after partition 1 activated" % key)
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d in result after partition 2 activated" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 3) in result set" % key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 4) in result set" % key)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
def do_test_set_cleanup_partitions_when_updater_is_running(self):
# print "Marking all partitions for cleanup"
common.set_partition_states(self._params,
cleanup=range(self._params["nparts"]))
# print "Compacting the set view group"
common.compact_set_view(self._params)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
self.assertEqual(view_result["total_rows"], 0, "Empty view result")
self.assertEqual(len(view_result["rows"]), 0, "Empty view result")
# print "Marking all partitions as active"
common.set_partition_states(self._params,
active=range(self._params["nparts"]))
# print "Querying view with ?stale=update_after"
(resp, view_result) = common.query(self._params, "mapview1",
{"stale": "update_after"})
self.assertEqual(view_result["total_rows"], 0, "Empty view result")
self.assertEqual(len(view_result["rows"]), 0, "Empty view result")
# print "Marking partition 2 for cleanup while the updater is running"
common.set_partition_states(self._params, cleanup=[1])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [1],
"right cleanup partitions list")
self.assertFalse("1" in info["update_seqs"],
"partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"],
"partition 1 not in info.update_seqs")
# print "Waiting for the set view updater to finish"
iterations = 0
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
iterations += 1
else:
break
self.assertTrue(
iterations > 0,
"Updater was running when partition 2 was marked for cleanup")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertTrue(["cleanup_partitions"] == [1]
or info["cleanup_partitions"] == [],
"cleanup partitions list is not wrong")
self.assertFalse("1" in info["update_seqs"],
"partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"],
"partition 1 not in info.update_seqs")
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 2, 3])
self.assertEqual(len(view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 2 marked for cleanup" %
key)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
self.assertFalse("1" in info["update_seqs"],
"partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"],
"partition 1 not in info.update_seqs")
def test_update_cleanup(self):
# print "Triggering initial view update"
t0 = time.time()
(resp, view_result) = common.query(self._params, "mapview",
{"limit": "1"})
# print "Update took %.2f seconds" % (time.time() - t0)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
stats = info["stats"]
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
self.assertEqual(stats["full_updates"], 1, "1 full update done so far")
self.assertEqual(stats["stopped_updates"], 0,
"no updater interruptions so far")
self.assertEqual(stats["cleanups"], 0, "0 cleanups done")
self.assertEqual(stats["cleanup_interruptions"], 0,
"no cleanup interruptions so far")
# print "Adding new partitions 5, 6, 7 and 8 and marking partitions 1 and 4 for cleanup"
common.set_partition_states(self._params,
active=[4, 5, 6, 7],
cleanup=[0, 3])
# print "Querying view (should trigger update + cleanup)"
t0 = time.time()
(resp, view_result) = common.query(self._params, "mapview")
t1 = time.time()
# print "Verifying query results"
expected_total = common.set_doc_count(self._params, [1, 2, 4, 5, 6, 7])
self.assertEqual(view_result["total_rows"], expected_total,
"total rows is %d" % expected_total)
self.assertEqual(len(view_result["rows"]), expected_total,
"got %d tows" % expected_total)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d (partition 1) not in query result after update+cleanup"
% key)
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d (partition 2) in query result after update+cleanup" %
key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d (partition 3) in query result after update+cleanup" %
key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d (partition 4) not in query result after update+cleanup"
% key)
for key in xrange(5, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d (partition 5) in query result after update+cleanup" %
key)
for key in xrange(6, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d (partition 6) in query result after update+cleanup" %
key)
for key in xrange(7, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d (partition 7) in query result after update+cleanup" %
key)
for key in xrange(8, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d (partition 8) in query result after update+cleanup" %
key)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
stats = info["stats"]
self.assertEqual(info["active_partitions"], [1, 2, 4, 5, 6, 7],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
self.assertEqual(stats["full_updates"], 2,
"2 full updates done so far")
self.assertEqual(stats["stopped_updates"], 0,
"no updater interruptions so far")
self.assertEqual(stats["cleanups"], 1, "1 full cleanup done")
self.assertEqual(stats["cleanup_interruptions"], 1,
"1 cleanup interruption done")
for i in info["active_partitions"]:
expected_seq = common.partition_update_seq(self._params, i)
self.assertTrue(
str(i) in info["update_seqs"],
"partition %d in info.update_seqs" % (i + 1))
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right seq in info.update_seqs[%d]" % i)
def do_test_simple_cleanup(self):
# print "Querying view in steady state"
(resp, view_result) = common.query(self._params, "mapview1")
etag = resp.getheader("ETag")
self.assertEqual(
view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(view_result)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Triggering partition 4 cleanup"
common.set_partition_states(self._params, cleanup=[3])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [3],
"right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"],
"partition 3 not in info.update_seqs")
self.assertEqual(info["cleanup_running"], True,
"cleanup process is running")
# print "Querying view again"
(resp2, view_result2) = common.query(self._params, "mapview1")
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(
view_result2["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result2["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(etag2, etag,
"Different Etag after cleanup triggered")
common.test_keys_sorted(view_result2)
all_keys = {}
for r in view_result2["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 4 cleanup triggered" %
key)
# print "Triggering view compaction and querying view again"
common.compact_set_view(self._params)
(resp3, view_result3) = common.query(self._params, "mapview1")
etag3 = resp3.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result3["total_rows"], expected,
"Query returned %d total_rows" % expected)
self.assertEqual(len(view_result3["rows"]), expected,
"Query returned %d rows" % expected)
self.assertEqual(etag2, etag3, "Same Etag after cleanup finished")
common.test_keys_sorted(view_result3)
all_keys = {}
for r in view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 4 cleanup finished" %
key)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"],
"partition 3 not in info.update_seqs")
# print "Adding 2 new documents to partition 4"
server = self._params["server"]
db4 = server[self._params["setname"] + "/3"]
new_doc1 = {
"_id": "999999999",
"integer": 999999999,
"string": "999999999"
}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db4.save(new_doc1)
db4.save(new_doc2)
# print "Querying view again"
(resp4, view_result4) = common.query(self._params, "mapview1")
etag4 = resp4.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result4["total_rows"], expected,
"Query returned %d total_rows" % expected)
self.assertEqual(len(view_result4["rows"]), expected,
"Query returned %d rows" % expected)
self.assertEqual(
etag4, etag3,
"Same etag after adding new documents to cleaned partition")
common.test_keys_sorted(view_result4)
all_keys = {}
for r in view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 4 cleanup finished" %
key)
self.assertFalse(new_doc1["integer"] in all_keys,
"new_doc1 not in query result after cleanup")
self.assertFalse(new_doc2["integer"] in all_keys,
"new_doc2 not in query result after cleanup")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"],
"partition 3 not in info.update_seqs")
# print "Triggering compaction again and verifying it doesn't crash"
common.compact_set_view(self._params)
(resp5, view_result5) = common.query(self._params, "mapview1")
etag5 = resp5.getheader("ETag")
self.assertEqual(etag5, etag4, "Same etag after second compaction")
self.assertEqual(view_result5, view_result4,
"Same query results after second compaction")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"],
"partition 3 not in info.update_seqs")
def test_erlang_views(self):
total_doc_count = common.set_doc_count(self._params,
range(self._params["nparts"]))
# print "Querying map view in steady state"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(view_result["total_rows"], total_doc_count,
"Query returned %d total_rows" % total_doc_count)
self.assertEqual(len(view_result["rows"]), total_doc_count,
"Query returned %d rows" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Adding 1 new document to each partition"
new_docs = []
for i in xrange(self._params["nparts"]):
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(
self._params, range(self._params["nparts"]))
self.assertEqual(new_total_doc_count,
(total_doc_count + len(new_docs)),
"N documents were added")
total_doc_count = new_total_doc_count
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count,
"number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count,
"total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
i = len(new_docs) - 1
j = len(view_result["rows"]) - 1
while i >= 0:
self.assertEqual(
view_result["rows"][j]["key"], new_docs[i]["integer"],
"new document %s in view result set" % new_docs[i]["_id"])
self.assertEqual(
view_result["rows"][j]["id"], new_docs[i]["_id"],
"new document %s in view result set" % new_docs[i]["_id"])
i -= 1
j -= 1
# print "Deleting the documents that were added before"
i = 0
for doc in new_docs:
db = self._params["setname"] + "/" + str(i)
common.delete_doc(self._params, db, doc["_id"])
i = (i + 1) % self._params["nparts"]
new_total_doc_count = common.set_doc_count(
self._params, range(self._params["nparts"]))
self.assertEqual(new_total_doc_count,
(total_doc_count - len(new_docs)),
"documents deleted")
# print "Querying view again (steady state)"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), new_total_doc_count,
"number of received rows is %d" % new_total_doc_count)
self.assertEqual(view_result["total_rows"], new_total_doc_count,
"total_rows is %d" % new_total_doc_count)
common.test_keys_sorted(view_result)
all_ids = {}
all_keys = {}
for row in view_result["rows"]:
all_ids[row["id"]] = True
all_keys[row["key"]] = True
for doc in new_docs:
self.assertFalse(doc["_id"] in all_ids,
"deleted doc %s not in view results anymore")
self.assertFalse(doc["integer"] in all_keys,
"deleted doc %s not in view results anymore")
def test_staleness(self):
# print "Querying map view"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
map_etag = map_resp.getheader("ETag")
self.assertEqual(
map_view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(map_view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(map_view_result)
# To give the set view group some time to write and fsync the index header
time.sleep(5)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Adding 1 new document to each partition"
server = self._params["server"]
db1 = server[self._params["setname"] + "/0"]
db2 = server[self._params["setname"] + "/1"]
db3 = server[self._params["setname"] + "/2"]
db4 = server[self._params["setname"] + "/3"]
new_doc1 = {"_id": "new_doc_1", "integer": -1, "string": "1"}
new_doc2 = {"_id": "new_doc_2", "integer": -2, "string": "2"}
new_doc3 = {"_id": "new_doc_3", "integer": -3, "string": "3"}
new_doc4 = {"_id": "new_doc_4", "integer": -4, "string": "4"}
db1.save(new_doc1)
db2.save(new_doc2)
db3.save(new_doc3)
db4.save(new_doc4)
# print "Querying map view with ?stale=ok"
(map_resp2, map_view_result2) = common.query(self._params, "mapview1",
{"stale": "ok"})
map_etag2 = map_resp2.getheader("ETag")
self.assertEqual(map_view_result2, map_view_result,
"Same response as before with ?stale=ok")
self.assertEqual(map_etag2, map_etag,
"Same etag as before with ?stale=ok")
# print "Verifying set view group info"
info2 = common.get_set_view_info(self._params)
self.assertEqual(info2, info,
"Same set view group info after ?stale=ok query")
# print "Querying map view with ?stale=update_after"
(map_resp3, map_view_result3) = common.query(self._params, "mapview1",
{"stale": "update_after"})
map_etag3 = map_resp3.getheader("ETag")
self.assertEqual(map_view_result3, map_view_result,
"Same response as before with ?stale=update_after")
self.assertEqual(map_etag3, map_etag,
"Same etag as before with ?stale=updater_after")
time.sleep(5)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
((self._params["ndocs"] / 4) + 1),
"right update seq for partition %d" % (i + 1))
# print "Querying map view without stale option"
(map_resp4, map_view_result4) = common.query(self._params, "mapview1")
map_etag4 = map_resp4.getheader("ETag")
self.assertEqual(
map_view_result4["total_rows"], (self._params["ndocs"] + 4),
"Query returned %d total_rows" % (self._params["ndocs"] + 4))
self.assertEqual(
len(map_view_result4["rows"]), (self._params["ndocs"] + 4),
"Query returned %d rows" % (self._params["ndocs"] + 4))
self.assertNotEqual(map_etag4, map_etag3,
"New etag after ?stale=update_after")
common.test_keys_sorted(map_view_result4)
self.assertEqual(map_view_result4["rows"][0]["id"], new_doc4["_id"],
"new_doc4 in view result")
self.assertEqual(map_view_result4["rows"][1]["id"], new_doc3["_id"],
"new_doc3 in view result")
self.assertEqual(map_view_result4["rows"][2]["id"], new_doc2["_id"],
"new_doc2 in view result")
self.assertEqual(map_view_result4["rows"][3]["id"], new_doc1["_id"],
"new_doc1 in view result")
def do_test_set_passive_during_compaction(self):
# print "Querying map view"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
self.assertEqual(map_view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(map_view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(map_view_result)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
db = self._params["server"][self._params["setname"] + "/" + str(i)]
seq = db.info()["update_seq"]
self.assertEqual(info["update_seqs"][str(i)], seq,
"right update seq for partition %d" % (i + 1))
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Marking partition 4 as passive"
common.set_partition_states(self._params, passive = [3])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Waiting for compaction to finish"
count = 0
while info["stats"]["compactions"] < 1:
time.sleep(0.5)
count += 1
info = common.get_set_view_info(self._params)
self.assertTrue((count > 0), "Compaction was running when the partition states were updated")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying map view again"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
expected = common.set_doc_count(self._params, [0, 1, 2])
self.assertEqual(map_view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(map_view_result["rows"]), expected,
"Query returned %d rows" % expected)
common.test_keys_sorted(map_view_result)
all_keys = {}
for r in map_view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 set to passive" % key)
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Adding two new partitions, 5 and 6, as passive while compaction is running"
common.set_partition_states(self._params, passive = [4, 5])
# print "Waiting for compaction to finish"
info = common.get_set_view_info(self._params)
count = 0
while info["stats"]["compactions"] < 2:
time.sleep(0.5)
count += 1
info = common.get_set_view_info(self._params)
self.assertTrue((count > 0), "Compaction was running when the partition states were updated")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3, 4, 5], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5]:
self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i)
for i in [6, 7]:
self.assertFalse(str(i) in info["update_seqs"], "%d not in info.update_seqs" % i)
# print "Querying map view again"
(map_resp, map_view_result2) = common.query(self._params, "mapview1")
self.assertEqual(map_view_result2["rows"], map_view_result["rows"], "Same result set as before")
total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
# print "Adding 1 new document to each partition"
new_docs = []
for i in [0, 1, 2, 3, 4, 5, 6, 7]:
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
self.assertEqual(new_total_doc_count, (total_doc_count + 8), "8 documents added")
self.assertEqual(len(new_docs), 8, "8 documents added")
info = common.get_set_view_info(self._params)
if info["updater_running"]:
# print "Waiting for updater to finish"
self.assertEqual(info["updater_state"], "updating_passive",
"updater state is updating_passive")
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
self.assertEqual(info["updater_state"], "updating_passive",
"updater state is updating_passive")
time.sleep(3)
else:
break
expected_row_count = common.set_doc_count(self._params, [0, 1, 2])
expected_total_rows = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5])
# print "Querying map view again"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
self.assertEqual(len(map_view_result["rows"]), expected_row_count, "len(rows) is %d" % expected_row_count)
common.test_keys_sorted(map_view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3, 4, 5], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5]:
self.assertTrue(str(i) in info["update_seqs"], "%d in info.update_seqs" % i)
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"info.update_seqs[%d] is %d" % (i, expected_seq))
def do_test_set_passive_partitions_when_updater_is_running(self):
# print "Re-creating databases"
common.create_dbs(self._params)
common.populate(self._params)
# print "Configuring set view with all partitions active"
common.define_set_view(self._params, [0, 1, 2, 3], [])
# print "Querying map view in steady state with ?stale=update_after"
(resp, view_result) = common.query(self._params, "mapview", {"stale": "update_after"})
self.assertEqual(len(view_result["rows"]), 0, "Received empty row set")
self.assertEqual(view_result["total_rows"], 0, "Received empty row set")
# print "Marking partition 4 as passive"
common.set_partition_states(self._params, passive = [3])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Waiting for the set view updater to finish"
iterations = 0
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
iterations += 1
else:
break
self.assertTrue(iterations > 0, "Updater was running when partition 4 was set to passive")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying map view again"
(resp, view_result) = common.query(self._params, "mapview")
doc_count = common.set_doc_count(self._params)
expected_row_count = common.set_doc_count(self._params, [0, 1, 2])
self.assertEqual(view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), expected_row_count, "Query returned %d rows" % expected_row_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, doc_count, self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 was set to passive" % key)
# print "Adding 2 new documents to partition 4"
server = self._params["server"]
db4 = server[self._params["setname"] + "/3"]
new_doc1 = {"_id": "999999999", "integer": 999999999, "string": "999999999"}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db4.save(new_doc1)
db4.save(new_doc2)
# print "Querying map view again"
(resp, view_result2) = common.query(self._params, "mapview")
self.assertEqual(view_result2["rows"], view_result["rows"], "Same result set as before")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
total_doc_count = common.set_doc_count(self._params)
# print "Changing partition 4 from passive to active"
common.set_partition_states(self._params, active = [3])
# print "Querying map view again"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count)
self.assertEqual(len(view_result["rows"]), total_doc_count, "number of rows returned is %d" % total_doc_count)
common.test_keys_sorted(view_result)
self.assertEqual(view_result["rows"][0]["key"], new_doc2["integer"], "new_doc2 reflected in result set at first row")
self.assertEqual(view_result["rows"][-1]["key"], new_doc1["integer"], "new_doc1 reflected in result set at last row")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
def test_erlang_views(self):
total_doc_count = common.set_doc_count(self._params, range(self._params["nparts"]))
# print "Querying map view in steady state"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(view_result["total_rows"], total_doc_count,
"Query returned %d total_rows" % total_doc_count)
self.assertEqual(len(view_result["rows"]), total_doc_count,
"Query returned %d rows" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Adding 1 new document to each partition"
new_docs = []
for i in xrange(self._params["nparts"]):
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(self._params, range(self._params["nparts"]))
self.assertEqual(new_total_doc_count, (total_doc_count + len(new_docs)), "N documents were added")
total_doc_count = new_total_doc_count
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
i = len(new_docs) - 1
j = len(view_result["rows"]) - 1
while i >= 0:
self.assertEqual(view_result["rows"][j]["key"], new_docs[i]["integer"],
"new document %s in view result set" % new_docs[i]["_id"])
self.assertEqual(view_result["rows"][j]["id"], new_docs[i]["_id"],
"new document %s in view result set" % new_docs[i]["_id"])
i -= 1
j -= 1
# print "Deleting the documents that were added before"
i = 0
for doc in new_docs:
db = self._params["setname"] + "/" + str(i)
common.delete_doc(self._params, db, doc["_id"])
i = (i + 1) % self._params["nparts"]
new_total_doc_count = common.set_doc_count(self._params, range(self._params["nparts"]))
self.assertEqual(new_total_doc_count, (total_doc_count - len(new_docs)), "documents deleted")
# print "Querying view again (steady state)"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), new_total_doc_count,
"number of received rows is %d" % new_total_doc_count)
self.assertEqual(view_result["total_rows"], new_total_doc_count,
"total_rows is %d" % new_total_doc_count)
common.test_keys_sorted(view_result)
all_ids = {}
all_keys = {}
for row in view_result["rows"]:
all_ids[row["id"]] = True
all_keys[row["key"]] = True
for doc in new_docs:
self.assertFalse(doc["_id"] in all_ids, "deleted doc %s not in view results anymore")
self.assertFalse(doc["integer"] in all_keys, "deleted doc %s not in view results anymore")
def do_test_updates(self):
# print "Creating databases"
common.create_dbs(self._params)
common.populate(self._params)
# print "Databases created"
# print "Configuring set view with 1 active partition and 3 passive partitions"
common.define_set_view(self._params, [0], [1, 2, 3])
# print "Querying map view"
(resp, view_result) = common.query(self._params, "mapview", {"limit": "20"})
# print "Grabbing group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0], "right active partitions list")
self.assertEqual(info["passive_partitions"], [1, 2, 3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
self.assertEqual(info["update_seqs"]["0"], (self._params["ndocs"] / 4), "All data from partition 1 was indexed")
sum_passive_seqs = 0
for i in [1, 2, 3]:
sum_passive_seqs += info["update_seqs"][str(i)]
self.assertTrue(sum_passive_seqs < (self._params["ndocs"] - (self._params["ndocs"] / 4)),
"Passive partitions are still being indexed")
self.assertEqual(info["updater_running"], True, "View updater still running")
self.assertEqual(info["updater_state"], "updating_passive", "View updater in state 'updating_passive'")
# print "Verifying view query response"
self.assertEqual(len(view_result["rows"]), 20, "Query returned 20 rows")
common.test_keys_sorted(view_result)
self.assertEqual(view_result["rows"][0]["key"], 1, "First row has key 1")
last_key_expected = range(1, self._params["ndocs"], self._params["nparts"])[19]
self.assertEqual(view_result["rows"][-1]["key"], last_key_expected, "Last row has key %d" % last_key_expected)
# print "Waiting for view updater to finish"
while True:
info = common.get_set_view_info(self._params)
if not info["updater_running"]:
break
else:
time.sleep(3)
# print "Grabbing group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0], "right active partitions list")
self.assertEqual(info["passive_partitions"], [1, 2, 3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"Right update seq for partition %d" % (i + 1))
self.assertEqual(info["updater_running"], False, "View updater not running anymore")
self.assertEqual(info["updater_state"], "not_running", "View updater not running anymore")
# print "Querying view again and validating the response"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), (self._params["ndocs"] / 4), "Query returned %d rows" % (self._params["ndocs"] / 4))
common.test_keys_sorted(view_result)
self.assertEqual(view_result["rows"][0]["key"], 1, "First row has key 1")
last_key_expected = range(1, self._params["ndocs"], self._params["nparts"])[-1]
self.assertEqual(view_result["rows"][-1]["key"], last_key_expected, "Last row has key %d" % last_key_expected)
def test_many_partitions(self):
total_doc_count = common.set_doc_count(self._params,
range(self._params["nparts"]))
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count,
"number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count,
"total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"],
range(self._params["nparts"]),
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
# print "Adding 1 new document to each partition"
new_docs = []
for i in xrange(self._params["nparts"]):
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(
self._params, range(self._params["nparts"]))
self.assertEqual(new_total_doc_count,
(total_doc_count + len(new_docs)),
"N documents were added")
total_doc_count = new_total_doc_count
# print "Querying view again"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count,
"number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count,
"total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"],
range(self._params["nparts"]),
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
# print "Marking half of the partitions as passive"
passive = range(self._params["nparts"] / 2, self._params["nparts"])
active = range(self._params["nparts"] / 2)
common.set_partition_states(self._params,
passive=range(self._params["nparts"] / 2,
self._params["nparts"]))
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], active,
"right active partitions list")
self.assertEqual(info["passive_partitions"], passive,
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
# print "Querying view again"
(resp, view_result) = common.query(self._params, "mapview")
expected_row_count = common.set_doc_count(self._params, active)
self.assertEqual(len(view_result["rows"]), expected_row_count,
"number of received rows is %d" % expected_row_count)
common.test_keys_sorted(view_result)
for row in view_result['rows']:
if row["key"] >= 2001:
key_part = ((row["key"] - 2000) % self._params["nparts"]) - 1
else:
key_part = (row["key"] % self._params["nparts"]) - 1
self.assertTrue(
key_part in active,
"Key %d from passive partition not in result set" % row["key"])
# print "Marking half of the partitions for cleanup"
common.set_partition_states(self._params, cleanup=passive)
cleanup = passive
common.compact_set_view(self._params)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], active,
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
self.assertEqual(info["stats"]["compactions"], 1, "1 compaction")
self.assertEqual(info["stats"]["cleanups"], 1, "1 full cleanup")
for i in active:
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
def test_updates(self):
total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
# print "Querying view in steady state with ?stale=update_after"
(resp, view_result) = common.query(self._params, "mapview", {"stale": "update_after"})
self.assertEqual(len(view_result["rows"]), 0, "Received empty row set")
self.assertEqual(view_result["total_rows"], 0, "Received empty row set")
# print "Adding partitions 7 and 8 as active while updater is running"
common.set_partition_states(self._params, active = [6, 7])
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Waiting for updater to finish"
count = 0
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
count += 1
else:
break
self.assertTrue(count > 0, "Updater was running when partitions 7 and 8 were added as active")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5]:
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
# print "Querying view again (steady state)"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Adding 1 new document to each partition"
new_docs = []
for i in [0, 1, 2, 3, 4, 5, 6, 7]:
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
self.assertEqual(new_total_doc_count, (total_doc_count + len(new_docs)), "8 documents were added")
total_doc_count = new_total_doc_count
# print "Querying view again (steady state)"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
i = len(new_docs) - 1
j = len(view_result["rows"]) - 1
while i >= 0:
self.assertEqual(view_result["rows"][j]["key"], new_docs[i]["integer"],
"new document %s in view result set" % new_docs[i]["_id"])
self.assertEqual(view_result["rows"][j]["id"], new_docs[i]["_id"],
"new document %s in view result set" % new_docs[i]["_id"])
i -= 1
j -= 1
# print "Deleting the documents that were added before"
i = 0
for doc in new_docs:
db = self._params["setname"] + "/" + str(i)
common.delete_doc(self._params, db, doc["_id"])
i = (i + 1) % self._params["nparts"]
new_total_doc_count = common.set_doc_count(self._params, [0, 1, 2, 3, 4, 5, 6, 7])
self.assertEqual(new_total_doc_count, (total_doc_count - len(new_docs)), "8 documents were deleted")
# print "Querying view again (steady state)"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), new_total_doc_count,
"number of received rows is %d" % new_total_doc_count)
self.assertEqual(view_result["total_rows"], new_total_doc_count,
"total_rows is %d" % new_total_doc_count)
common.test_keys_sorted(view_result)
all_ids = {}
all_keys = {}
for row in view_result["rows"]:
all_ids[row["id"]] = True
all_keys[row["key"]] = True
for doc in new_docs:
self.assertFalse(doc["_id"] in all_ids, "deleted doc %s not in view results anymore")
self.assertFalse(doc["integer"] in all_keys, "deleted doc %s not in view results anymore")
'--duplicates'
],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
devices = e.output
except:
print "Error while scanning for bluetooth devices!"
raise
# Parse output and insert into iot-project.db
try:
for line in devices.split('\n'):
if (re.match(r'(\w+:\w+)+', line)):
(macaddress, deviceName) = line.split()
print "Found Device %s with mac address %s" % (deviceName,
macaddress)
dbQuery = "INSERT OR REPLACE INTO BluetoothDevices values ('%s','%s',datetime('now'))" % (
macaddress, deviceName)
common.query(dbQuery)
except:
print "Error recording bluetooth devices in DB!"
raise
print "Scan complete , sleeping for 60 seconds"
time.sleep(60)
except:
print "Error running Bluetooth Scanner!"
traceback.print_exc()
finally:
sys.exit()
def test_updates(self):
total_doc_count = common.set_doc_count(self._params,
[0, 1, 2, 3, 4, 5, 6, 7])
# print "Querying view in steady state with ?stale=update_after"
(resp, view_result) = common.query(self._params, "mapview",
{"stale": "update_after"})
self.assertEqual(len(view_result["rows"]), 0, "Received empty row set")
self.assertEqual(view_result["total_rows"], 0,
"Received empty row set")
# print "Adding partitions 7 and 8 as active while updater is running"
common.set_partition_states(self._params, active=[6, 7])
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
# print "Waiting for updater to finish"
count = 0
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
count += 1
else:
break
self.assertTrue(
count > 0,
"Updater was running when partitions 7 and 8 were added as active")
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3, 4, 5, 6, 7],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3, 4, 5]:
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(
info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" %
(expected_seq, i + 1))
# print "Querying view again (steady state)"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count,
"number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count,
"total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Adding 1 new document to each partition"
new_docs = []
for i in [0, 1, 2, 3, 4, 5, 6, 7]:
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(self._params,
[0, 1, 2, 3, 4, 5, 6, 7])
self.assertEqual(new_total_doc_count,
(total_doc_count + len(new_docs)),
"8 documents were added")
total_doc_count = new_total_doc_count
# print "Querying view again (steady state)"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count,
"number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count,
"total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
i = len(new_docs) - 1
j = len(view_result["rows"]) - 1
while i >= 0:
self.assertEqual(
view_result["rows"][j]["key"], new_docs[i]["integer"],
"new document %s in view result set" % new_docs[i]["_id"])
self.assertEqual(
view_result["rows"][j]["id"], new_docs[i]["_id"],
"new document %s in view result set" % new_docs[i]["_id"])
i -= 1
j -= 1
# print "Deleting the documents that were added before"
i = 0
for doc in new_docs:
db = self._params["setname"] + "/" + str(i)
common.delete_doc(self._params, db, doc["_id"])
i = (i + 1) % self._params["nparts"]
new_total_doc_count = common.set_doc_count(self._params,
[0, 1, 2, 3, 4, 5, 6, 7])
self.assertEqual(new_total_doc_count,
(total_doc_count - len(new_docs)),
"8 documents were deleted")
# print "Querying view again (steady state)"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), new_total_doc_count,
"number of received rows is %d" % new_total_doc_count)
self.assertEqual(view_result["total_rows"], new_total_doc_count,
"total_rows is %d" % new_total_doc_count)
common.test_keys_sorted(view_result)
all_ids = {}
all_keys = {}
for row in view_result["rows"]:
all_ids[row["id"]] = True
all_keys[row["key"]] = True
for doc in new_docs:
self.assertFalse(doc["_id"] in all_ids,
"deleted doc %s not in view results anymore")
self.assertFalse(doc["integer"] in all_keys,
"deleted doc %s not in view results anymore")
def do_test_reduce(self, viewname):
# print "Querying reduce view in steady state"
(resp, view_result) = common.query(self._params, viewname)
etag = resp.getheader("ETag")
self.assertEqual(len(view_result["rows"]), 1, "Query returned 1 row")
self.assertEqual(view_result["rows"][0]["value"], self._params["ndocs"],
"Non-grouped reduce value is %d" % self._params["ndocs"])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Setting partition 3 to passive state"
common.set_partition_states(self._params, passive = [2])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [2], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying reduce view again"
(resp2, view_result2) = common.query(self._params, viewname)
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(len(view_result2["rows"]), 1, "Query returned 1 row")
self.assertEqual(view_result2["rows"][0]["value"], expected,
"Non-grouped reduce value is %d" % expected)
self.assertNotEqual(etag2, etag, "Different ETags")
# print "Querying view with ?group=true"
(resp3, view_result3) = common.query(self._params, viewname, {"group": "true"})
etag3 = resp3.getheader("ETag")
self.assertEqual(len(view_result3["rows"]), expected, "Query returned % rows" % expected)
self.assertNotEqual(etag3, etag, "Different ETags")
self.assertEqual(etag3, etag2, "Equal ETags for responses 2 and 3")
common.test_keys_sorted(view_result3)
all_keys = {}
for r in view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 3 was made passive" % key)
# print "Querying view with ?group=true&descending=true"
(resp4, view_result4) = common.query(
self._params, viewname, {"group": "true", "descending": "true"})
etag4 = resp4.getheader("ETag")
self.assertEqual(len(view_result4["rows"]), expected, "Query returned % rows" % expected)
self.assertNotEqual(etag4, etag, "Different ETags")
self.assertEqual(etag4, etag3, "Equal ETags for responses 3 and 4")
common.test_keys_sorted(view_result4, lambda a, b: a > b)
all_keys = {}
for r in view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 3 was made passive" % key)
# print "Querying view with ?group=true&startkey=3333&endkey=44781"
(resp5, view_result5) = common.query(
self._params, viewname,
{"group": "true", "startkey": "3333", "endkey": "44781"})
etag5 = resp5.getheader("ETag")
self.assertNotEqual(etag5, etag, "Different ETags")
self.assertEqual(etag5, etag4, "Equal ETags for responses 4 and 5")
common.test_keys_sorted(view_result5)
self.assertTrue(view_result5["rows"][0]["key"] >= 3333, "First key is >= 3333")
self.assertTrue(view_result5["rows"][-1]["key"] <= 44781, "Last key is <= 44781")
all_keys = {}
for r in view_result5["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 3 was made passive" % key)
# print "Querying view with ?group=true&startkey=44781&endkey=3333&descending=true"
(resp6, view_result6) = common.query(
self._params, viewname,
{"group": "true", "startkey": "44781", "endkey": "3333", "descending": "true"})
etag6 = resp6.getheader("ETag")
self.assertNotEqual(etag6, etag, "Different ETags")
self.assertEqual(etag6, etag5, "Equal ETags for responses 5 and 6")
common.test_keys_sorted(view_result6, lambda a, b: a > b)
self.assertTrue(view_result6["rows"][0]["key"] <= 44781, "First key is <= 44781")
self.assertTrue(view_result6["rows"][-1]["key"] >= 3333, "Last key is >= 3333")
self.assertEqual(len(view_result6["rows"]), len(view_result5["rows"]),
"Same number of rows for responses 5 and 6")
all_keys = {}
for r in view_result6["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 3 was made passive" % key)
# print "Setting partition 3 to active state"
common.set_partition_states(self._params, active = [2])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view with ?group=true"
(resp7, view_result7) = common.query(self._params, viewname, {"group": "true"})
etag7 = resp7.getheader("ETag")
self.assertEqual(len(view_result7["rows"]), self._params["ndocs"],
"Query returned % rows" % self._params["ndocs"])
self.assertEqual(etag7, etag, "Same etags for responses 1 and 7")
common.test_keys_sorted(view_result7)
# print "Querying view with ?group=true&descending=true"
(resp8, view_result8) = common.query(
self._params, viewname, {"group": "true", "descending": "true"})
etag8 = resp8.getheader("ETag")
self.assertEqual(len(view_result8["rows"]), self._params["ndocs"],
"Query returned % rows" % self._params["ndocs"])
self.assertEqual(etag7, etag8, "Same etags for responses 7 and 8")
common.test_keys_sorted(view_result8, lambda a, b: a > b)
# print "Querying view with ?group=true&startkey=3333&endkey=44781"
(resp9, view_result9) = common.query(
self._params, viewname,
{"group": "true", "startkey": "3333", "endkey": "44781"})
etag9 = resp9.getheader("ETag")
self.assertEqual(etag9, etag8, "Equal ETags for responses 8 and 9")
common.test_keys_sorted(view_result9)
self.assertTrue(view_result9["rows"][0]["key"] >= 3333, "First key is >= 3333")
self.assertTrue(view_result9["rows"][-1]["key"] <= 44781, "Last key is <= 44781")
# print "Querying view with ?group=true&startkey=44781&endkey=3333&descending=true"
(resp10, view_result10) = common.query(
self._params, viewname,
{"group": "true", "startkey": "44781", "endkey": "3333", "descending": "true"})
etag10 = resp10.getheader("ETag")
self.assertEqual(etag10, etag9, "Equal ETags for responses 9 and 10")
common.test_keys_sorted(view_result10, lambda a, b: a > b)
self.assertTrue(view_result10["rows"][0]["key"] <= 44781, "First key is <= 44781")
self.assertTrue(view_result10["rows"][-1]["key"] >= 3333, "Last key is >= 3333")
self.assertEqual(len(view_result10["rows"]), len(view_result9["rows"]),
"Same number of rows for responses 9 and 10")
def do_test_change_partition_states_while_cleanup_running(self):
# print "Marking all partitions as active"
common.set_partition_states(self._params,
active=range(self._params["nparts"]))
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
# print "Updating view"
(resp, view_result) = common.query(self._params, "mapview1",
{"limit": "100"})
self.assertEqual(view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), 100,
"Query returned 100 rows")
common.test_keys_sorted(view_result)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
# print "Marking partitions 1 and 2 for cleanup"
common.set_partition_states(self._params, cleanup=[0, 1])
info = common.get_set_view_info(self._params)
self.assertEqual(info["cleanup_running"], True, "cleanup is running")
self.assertEqual(info["active_partitions"], [2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [0, 1],
"right cleanup partitions list")
# print "Marking partitions 1 and 2 as active while cleanup is ongoing"
common.set_partition_states(self._params, active=[0, 1])
info = common.get_set_view_info(self._params)
self.assertEqual(type(info["pending_transition"]), dict,
"pending_transition is an object")
self.assertEqual(sorted(info["pending_transition"]["active"]), [0, 1],
"pending_transition active list is [0, 1]")
self.assertEqual(info["pending_transition"]["passive"], [],
"pending_transition passive list is []")
self.assertEqual(info["pending_transition"]["cleanup"], [],
"pending_transition cleanup list is []")
# print "Waiting for pending transition to be applied"
iterations = 0
while True:
if iterations > 600:
raise (Exception(
"timeout waiting for pending transition to be applied"))
info = common.get_set_view_info(self._params)
if info["pending_transition"] is None:
break
else:
time.sleep(1)
iterations += 1
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
info = common.get_set_view_info(self._params)
self.assertEqual(view_result["total_rows"], doc_count,
"Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), doc_count,
"Query returned %d rows" % doc_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d in result after partition 1 activated" % key)
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(
key in all_keys,
"Key %d in result after partition 2 activated" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 3) in result set" % key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 4) in result set" % key)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
def do_test_maps(self):
# print "Querying map view in steady state"
(resp, view_result) = common.query(self._params, "mapview1")
etag = resp.getheader("ETag")
self.assertEqual(view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(view_result)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Setting partition 4 as passive"
common.set_partition_states(self._params, passive = [3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp2, view_result2) = common.query(self._params, "mapview1")
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result2["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result2["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(etag2, etag, "Different Etag after setting passive partition")
common.test_keys_sorted(view_result2)
all_keys = {}
for r in view_result2["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys, "Key %d not in result after partition 4 was made passive" % key)
# print "Setting partition 4 state to active"
common.set_partition_states(self._params, active = [3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp3, view_result3) = common.query(self._params, "mapview1")
etag3 = resp3.getheader("ETag")
self.assertEqual(view_result3["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result3["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
self.assertEqual(etag3, etag, "ETag is same as first query response")
common.test_keys_sorted(view_result3)
# print "Setting partitions 1 and 4 to passive state"
common.set_partition_states(self._params, passive = [0, 3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [0, 3], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp4, view_result4) = common.query(self._params, "mapview1")
etag4 = resp4.getheader("ETag")
expected = self._params["ndocs"] / 2
self.assertEqual(view_result4["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result4["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(etag4, etag, "ETag is different from all previous responses")
self.assertNotEqual(etag4, etag2, "ETag is different from all previous responses")
common.test_keys_sorted(view_result4)
all_keys = {}
for r in view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 1 was made passive" % key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 was made passive" % key)
# print "Marking all partitions as passive and querying view again"
common.set_partition_states(self._params, passive = range(self._params["nparts"]))
(resp5, view_result5) = common.query(self._params, "mapview1")
etag5 = resp5.getheader("ETag")
self.assertEqual(view_result5["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result5["rows"]), 0, "Query returned 0 rows")
self.assertNotEqual(etag5, etag, "ETag is different from all previous responses")
self.assertNotEqual(etag5, etag2, "ETag is different from all previous responses")
self.assertNotEqual(etag5, etag3, "ETag is different from all previous responses")
self.assertNotEqual(etag5, etag4, "ETag is different from all previous responses")
# print "Setting all partitions to active state"
common.set_partition_states(self._params, active = range(self._params["nparts"]))
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp6, view_result6) = common.query(self._params, "mapview1")
etag6 = resp6.getheader("ETag")
self.assertEqual(view_result6["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result6["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
self.assertEqual(etag6, etag, "ETag is the same from first view query response")
common.test_keys_sorted(view_result6)
def test_update_cleanup(self):
# print "Triggering initial view update"
t0 = time.time()
(resp, view_result) = common.query(self._params, "mapview", {"limit": "1"})
# print "Update took %.2f seconds" % (time.time() - t0)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
stats = info["stats"]
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
self.assertEqual(stats["full_updates"], 1, "1 full update done so far")
self.assertEqual(stats["stopped_updates"], 0, "no updater interruptions so far")
self.assertEqual(stats["cleanups"], 0, "0 cleanups done")
self.assertEqual(stats["cleanup_interruptions"], 0, "no cleanup interruptions so far")
# print "Adding new partitions 5, 6, 7 and 8 and marking partitions 1 and 4 for cleanup"
common.set_partition_states(self._params, active=[4, 5, 6, 7], cleanup=[0, 3])
# print "Querying view (should trigger update + cleanup)"
t0 = time.time()
(resp, view_result) = common.query(self._params, "mapview")
t1 = time.time()
# print "Verifying query results"
expected_total = common.set_doc_count(self._params, [1, 2, 4, 5, 6, 7])
self.assertEqual(view_result["total_rows"], expected_total, "total rows is %d" % expected_total)
self.assertEqual(len(view_result["rows"]), expected_total, "got %d tows" % expected_total)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys, "Key %d (partition 1) not in query result after update+cleanup" % key)
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys, "Key %d (partition 2) in query result after update+cleanup" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys, "Key %d (partition 3) in query result after update+cleanup" % key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys, "Key %d (partition 4) not in query result after update+cleanup" % key)
for key in xrange(5, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys, "Key %d (partition 5) in query result after update+cleanup" % key)
for key in xrange(6, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys, "Key %d (partition 6) in query result after update+cleanup" % key)
for key in xrange(7, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys, "Key %d (partition 7) in query result after update+cleanup" % key)
for key in xrange(8, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys, "Key %d (partition 8) in query result after update+cleanup" % key)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
stats = info["stats"]
self.assertEqual(info["active_partitions"], [1, 2, 4, 5, 6, 7], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
self.assertEqual(stats["full_updates"], 2, "2 full updates done so far")
self.assertEqual(stats["stopped_updates"], 0, "no updater interruptions so far")
self.assertEqual(stats["cleanups"], 1, "1 full cleanup done")
self.assertEqual(stats["cleanup_interruptions"], 1, "1 cleanup interruption done")
for i in info["active_partitions"]:
expected_seq = common.partition_update_seq(self._params, i)
self.assertTrue(str(i) in info["update_seqs"], "partition %d in info.update_seqs" % (i + 1))
self.assertEqual(info["update_seqs"][str(i)], expected_seq, "right seq in info.update_seqs[%d]" % i)
def test_filter_partitions(self):
# print "Querying map view with ?partitions=[0,1,3]"
(map_resp,
map_view_result) = common.query(self._params, "mapview1",
{"partitions": json.dumps([0, 1, 3])})
map_etag = map_resp.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(
map_view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(map_view_result["rows"]), expected,
"Query returned %d rows" % expected)
common.test_keys_sorted(map_view_result)
all_keys = {}
for r in map_view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d from partition 3 is not in the result" % key)
# print "Disabling (making it passive) partition 2"
common.set_partition_states(self._params, passive=[1])
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp2, map_view_result2) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag2 = map_resp2.getheader("ETag")
self.assertEqual(map_view_result2, map_view_result,
"Same result as before")
self.assertEqual(map_etag2, map_etag, "Same Etag as before")
# print "Marking partition 2 for cleanup"
common.set_partition_states(self._params, cleanup=[1])
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp3, map_view_result3) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag3 = map_resp3.getheader("ETag")
expected = self._params["ndocs"] / 2
self.assertEqual(len(map_view_result3["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(map_etag3, map_etag2, "Different Etag from before")
common.test_keys_sorted(map_view_result3)
all_keys = {}
for r in map_view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d from partition 2 is not in the result" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d from partition 3 is not in the result" % key)
# print "Triggering view compaction (to guarantee cleanup is complete)"
common.compact_set_view(self._params)
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp4, map_view_result4) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag4 = map_resp4.getheader("ETag")
# total_rows is different after cleanup
self.assertEquals(map_view_result4["rows"], map_view_result3["rows"],
"Same result as before")
self.assertEquals(map_etag4, map_etag3, "Same Etag as before")
def createSelectQuery(where, fld, vals, quote=False):
return common.query(where, '[%s] IN (', fld) + ', '.join(
("'" + unicode(val) + "'" if quote else unicode(val)) for val in vals
) + ')'
import arcpy, common
strLayer = 'tmp_i095'
relLayer = 'tmp_i043'
with common.runtool(7) as parameters:
interLayer, strengthFld, lengthFld, minStrengthStr, minRelStrengthStr, maxLengthStr, output = parameters
if minStrengthStr or maxLengthStr:
queries = []
if minStrengthStr:
common.progress('assembling absolute strength exclusion')
minStrength = common.toFloat(minStrengthStr, 'minimum absolute interaction strength')
queries.append(common.query(interLayer, '[%s] >= %g', strengthFld, minStrength))
if maxLengthStr:
common.progress('assembling absolute length exclusion')
maxLength = common.toFloat(maxLengthStr, 'maximum absolute interaction length')
queries.append(common.query(interLayer, '[%s] <= %g', lengthFld, maxLength))
common.selection(interLayer, strLayer, ' OR '.join(queries))
else:
strLayer = interLayer
if minRelStrengthStr:
common.progress('performing relative strength exclusion')
minRelStrength = common.toFloat(minRelStrengthStr, 'minimum relative interaction strength')
relQuery = common.query(interLayer, '[%s] > 0 AND ([%s] / [%s] * 1000) >= %g', lengthFld, strengthFld,
lengthFld, minRelStrength)
common.select(strLayer, relLayer, relQuery)
else:
relLayer = strLayer
common.progress('counting selected interactions')
common.message('%i interactions selected.' % common.count(relLayer))
common.progress('writing output')
def do_test_set_cleanup_partitions_when_updater_is_running(self):
# print "Marking all partitions for cleanup"
common.set_partition_states(self._params, cleanup = range(self._params["nparts"]))
# print "Compacting the set view group"
common.compact_set_view(self._params)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
self.assertEqual(view_result["total_rows"], 0, "Empty view result")
self.assertEqual(len(view_result["rows"]), 0, "Empty view result")
# print "Marking all partitions as active"
common.set_partition_states(self._params, active = range(self._params["nparts"]))
# print "Querying view with ?stale=update_after"
(resp, view_result) = common.query(self._params, "mapview1", {"stale": "update_after"})
self.assertEqual(view_result["total_rows"], 0, "Empty view result")
self.assertEqual(len(view_result["rows"]), 0, "Empty view result")
# print "Marking partition 2 for cleanup while the updater is running"
common.set_partition_states(self._params, cleanup = [1])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [1], "right cleanup partitions list")
self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs")
# print "Waiting for the set view updater to finish"
iterations = 0
while True:
info = common.get_set_view_info(self._params)
if info["updater_running"]:
iterations += 1
else:
break
self.assertTrue(iterations > 0, "Updater was running when partition 2 was marked for cleanup")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertTrue(["cleanup_partitions"] == [1] or info["cleanup_partitions"] == [],
"cleanup partitions list is not wrong")
self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs")
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 2, 3])
self.assertEqual(len(view_result["rows"]), doc_count, "Query returned %d rows" % doc_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 2 marked for cleanup" % key)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
self.assertFalse("1" in info["update_seqs"], "partition 1 not in info.update_seqs")
self.assertFalse("1" in info["purge_seqs"], "partition 1 not in info.update_seqs")
def do_test_change_partition_states_while_cleanup_running(self):
# print "Marking all partitions as active"
common.set_partition_states(self._params, active = range(self._params["nparts"]))
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
# print "Updating view"
(resp, view_result) = common.query(self._params, "mapview1", {"limit": "100"})
self.assertEqual(view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), 100, "Query returned 100 rows")
common.test_keys_sorted(view_result)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
# print "Marking partitions 1 and 2 for cleanup"
common.set_partition_states(self._params, cleanup = [0, 1])
info = common.get_set_view_info(self._params)
self.assertEqual(info["cleanup_running"], True, "cleanup is running")
self.assertEqual(info["active_partitions"], [2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [0, 1], "right cleanup partitions list")
# print "Marking partitions 1 and 2 as active while cleanup is ongoing"
common.set_partition_states(self._params, active = [0, 1])
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview1")
doc_count = common.set_doc_count(self._params, [0, 1, 2, 3])
info = common.get_set_view_info(self._params)
self.assertEqual(view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count)
self.assertEqual(len(view_result["rows"]), doc_count, "Query returned %d rows" % doc_count)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d in result after partition 1 activated" % key)
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d in result after partition 2 activated" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 3) in result set" % key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d (partition 4) in result set" % key)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
expected = common.set_doc_count(self._params, [i])
self.assertEqual(info["update_seqs"][str(i)], expected,
"right update seq for partition %d" % (i + 1))
def do_test_view_updates(self):
# print "Setting partition 2 state to passive"
common.set_partition_states(self._params, passive = [1])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [1], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Adding 2 new documents to partition 2"
server = self._params["server"]
db2 = server[self._params["setname"] + "/1"]
new_doc1 = {"_id": "999999999", "integer": 999999999, "string": "999999999"}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db2.save(new_doc1)
db2.save(new_doc2)
# print "Querying map view"
(resp, view_result) = common.query(self._params, "mapview1")
etag = resp.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(len(view_result["rows"]), expected, "Query returned %d rows" % expected)
common.test_keys_sorted(view_result)
all_keys = {}
for r in view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 2 was made passive" % key)
self.assertFalse(new_doc1["integer"] in all_keys, "new_doc1 not reflected in view")
self.assertFalse(new_doc2["integer"] in all_keys, "new_doc2 not reflected in view")
# print "Setting partition 2 state to active"
common.set_partition_states(self._params, active = [1])
# print "Querying map view again"
(resp2, view_result2) = common.query(self._params, "mapview1")
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] + 2
self.assertEqual(view_result2["total_rows"], expected, "Query returned %d total_rows" % expected)
self.assertEqual(len(view_result2["rows"]), expected, "Query returned %d rows" % expected)
common.test_keys_sorted(view_result2)
all_keys = {}
for r in view_result2["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertTrue(key in all_keys,
"Key %d in result after partition 2 was re-enabled" % key)
self.assertTrue(new_doc1["integer"] in all_keys, "new_doc1 reflected in view")
self.assertTrue(new_doc2["integer"] in all_keys, "new_doc2 reflected in view")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
if i == 1:
seq = (self._params["ndocs"] / 4) + 2
else:
seq = (self._params["ndocs"] / 4)
self.assertEqual(info["update_seqs"][str(i)], seq, "right update seq for partition %d" % (i + 1))
def do_test_simple_cleanup(self):
# print "Querying view in steady state"
(resp, view_result) = common.query(self._params, "mapview1")
etag = resp.getheader("ETag")
self.assertEqual(view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(view_result)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Triggering partition 4 cleanup"
common.set_partition_states(self._params, cleanup = [3])
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [3], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs")
self.assertEqual(info["cleanup_running"], True, "cleanup process is running")
# print "Querying view again"
(resp2, view_result2) = common.query(self._params, "mapview1")
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result2["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result2["rows"]), expected, "Query returned %d rows" % expected)
self.assertNotEqual(etag2, etag, "Different Etag after cleanup triggered")
common.test_keys_sorted(view_result2)
all_keys = {}
for r in view_result2["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 cleanup triggered" % key)
# print "Triggering view compaction and querying view again"
common.compact_set_view(self._params)
(resp3, view_result3) = common.query(self._params, "mapview1")
etag3 = resp3.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result3["total_rows"], expected, "Query returned %d total_rows" % expected)
self.assertEqual(len(view_result3["rows"]), expected, "Query returned %d rows" % expected)
self.assertEqual(etag2, etag3, "Same Etag after cleanup finished")
common.test_keys_sorted(view_result3)
all_keys = {}
for r in view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 cleanup finished" % key)
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs")
# print "Adding 2 new documents to partition 4"
server = self._params["server"]
db4 = server[self._params["setname"] + "/3"]
new_doc1 = {"_id": "999999999", "integer": 999999999, "string": "999999999"}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db4.save(new_doc1)
db4.save(new_doc2)
# print "Querying view again"
(resp4, view_result4) = common.query(self._params, "mapview1")
etag4 = resp4.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(view_result4["total_rows"], expected, "Query returned %d total_rows" % expected)
self.assertEqual(len(view_result4["rows"]), expected, "Query returned %d rows" % expected)
self.assertEqual(etag4, etag3, "Same etag after adding new documents to cleaned partition")
common.test_keys_sorted(view_result4)
all_keys = {}
for r in view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 cleanup finished" % key)
self.assertFalse(new_doc1["integer"] in all_keys, "new_doc1 not in query result after cleanup")
self.assertFalse(new_doc2["integer"] in all_keys, "new_doc2 not in query result after cleanup")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs")
# print "Triggering compaction again and verifying it doesn't crash"
common.compact_set_view(self._params)
(resp5, view_result5) = common.query(self._params, "mapview1")
etag5 = resp5.getheader("ETag")
self.assertEqual(etag5, etag4, "Same etag after second compaction")
self.assertEqual(view_result5, view_result4, "Same query results after second compaction")
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "partition 3 not in info.update_seqs")
def do_test_reduce(self, viewname):
# print "Querying reduce view in steady state"
(resp, view_result) = common.query(self._params, viewname)
etag = resp.getheader("ETag")
self.assertEqual(len(view_result["rows"]), 1, "Query returned 1 row")
self.assertEqual(
view_result["rows"][0]["value"], self._params["ndocs"],
"Non-grouped reduce value is %d" % self._params["ndocs"])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Setting partition 3 to passive state"
common.set_partition_states(self._params, passive=[2])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [2],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying reduce view again"
(resp2, view_result2) = common.query(self._params, viewname)
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(len(view_result2["rows"]), 1, "Query returned 1 row")
self.assertEqual(view_result2["rows"][0]["value"], expected,
"Non-grouped reduce value is %d" % expected)
self.assertNotEqual(etag2, etag, "Different ETags")
# print "Querying view with ?group=true"
(resp3, view_result3) = common.query(self._params, viewname,
{"group": "true"})
etag3 = resp3.getheader("ETag")
self.assertEqual(len(view_result3["rows"]), expected,
"Query returned % rows" % expected)
self.assertNotEqual(etag3, etag, "Different ETags")
self.assertEqual(etag3, etag2, "Equal ETags for responses 2 and 3")
common.test_keys_sorted(view_result3)
all_keys = {}
for r in view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 3 was made passive" %
key)
# print "Querying view with ?group=true&descending=true"
(resp4, view_result4) = common.query(self._params, viewname, {
"group": "true",
"descending": "true"
})
etag4 = resp4.getheader("ETag")
self.assertEqual(len(view_result4["rows"]), expected,
"Query returned % rows" % expected)
self.assertNotEqual(etag4, etag, "Different ETags")
self.assertEqual(etag4, etag3, "Equal ETags for responses 3 and 4")
common.test_keys_sorted(view_result4, lambda a, b: a > b)
all_keys = {}
for r in view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 3 was made passive" %
key)
# print "Querying view with ?group=true&startkey=3333&endkey=44781"
(resp5, view_result5) = common.query(self._params, viewname, {
"group": "true",
"startkey": "3333",
"endkey": "44781"
})
etag5 = resp5.getheader("ETag")
self.assertNotEqual(etag5, etag, "Different ETags")
self.assertEqual(etag5, etag4, "Equal ETags for responses 4 and 5")
common.test_keys_sorted(view_result5)
self.assertTrue(view_result5["rows"][0]["key"] >= 3333,
"First key is >= 3333")
self.assertTrue(view_result5["rows"][-1]["key"] <= 44781,
"Last key is <= 44781")
all_keys = {}
for r in view_result5["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 3 was made passive" %
key)
# print "Querying view with ?group=true&startkey=44781&endkey=3333&descending=true"
(resp6, view_result6) = common.query(
self._params, viewname, {
"group": "true",
"startkey": "44781",
"endkey": "3333",
"descending": "true"
})
etag6 = resp6.getheader("ETag")
self.assertNotEqual(etag6, etag, "Different ETags")
self.assertEqual(etag6, etag5, "Equal ETags for responses 5 and 6")
common.test_keys_sorted(view_result6, lambda a, b: a > b)
self.assertTrue(view_result6["rows"][0]["key"] <= 44781,
"First key is <= 44781")
self.assertTrue(view_result6["rows"][-1]["key"] >= 3333,
"Last key is >= 3333")
self.assertEqual(len(view_result6["rows"]), len(view_result5["rows"]),
"Same number of rows for responses 5 and 6")
all_keys = {}
for r in view_result6["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 3 was made passive" %
key)
# print "Setting partition 3 to active state"
common.set_partition_states(self._params, active=[2])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view with ?group=true"
(resp7, view_result7) = common.query(self._params, viewname,
{"group": "true"})
etag7 = resp7.getheader("ETag")
self.assertEqual(len(view_result7["rows"]), self._params["ndocs"],
"Query returned % rows" % self._params["ndocs"])
self.assertEqual(etag7, etag, "Same etags for responses 1 and 7")
common.test_keys_sorted(view_result7)
# print "Querying view with ?group=true&descending=true"
(resp8, view_result8) = common.query(self._params, viewname, {
"group": "true",
"descending": "true"
})
etag8 = resp8.getheader("ETag")
self.assertEqual(len(view_result8["rows"]), self._params["ndocs"],
"Query returned % rows" % self._params["ndocs"])
self.assertEqual(etag7, etag8, "Same etags for responses 7 and 8")
common.test_keys_sorted(view_result8, lambda a, b: a > b)
# print "Querying view with ?group=true&startkey=3333&endkey=44781"
(resp9, view_result9) = common.query(self._params, viewname, {
"group": "true",
"startkey": "3333",
"endkey": "44781"
})
etag9 = resp9.getheader("ETag")
self.assertEqual(etag9, etag8, "Equal ETags for responses 8 and 9")
common.test_keys_sorted(view_result9)
self.assertTrue(view_result9["rows"][0]["key"] >= 3333,
"First key is >= 3333")
self.assertTrue(view_result9["rows"][-1]["key"] <= 44781,
"Last key is <= 44781")
# print "Querying view with ?group=true&startkey=44781&endkey=3333&descending=true"
(resp10, view_result10) = common.query(
self._params, viewname, {
"group": "true",
"startkey": "44781",
"endkey": "3333",
"descending": "true"
})
etag10 = resp10.getheader("ETag")
self.assertEqual(etag10, etag9, "Equal ETags for responses 9 and 10")
common.test_keys_sorted(view_result10, lambda a, b: a > b)
self.assertTrue(view_result10["rows"][0]["key"] <= 44781,
"First key is <= 44781")
self.assertTrue(view_result10["rows"][-1]["key"] >= 3333,
"Last key is >= 3333")
self.assertEqual(len(view_result10["rows"]), len(view_result9["rows"]),
"Same number of rows for responses 9 and 10")
def test_staleness(self):
# print "Querying map view"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
map_etag = map_resp.getheader("ETag")
self.assertEqual(map_view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(map_view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(map_view_result)
# To give the set view group some time to write and fsync the index header
time.sleep(5)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Adding 1 new document to each partition"
server = self._params["server"]
db1 = server[self._params["setname"] + "/0"]
db2 = server[self._params["setname"] + "/1"]
db3 = server[self._params["setname"] + "/2"]
db4 = server[self._params["setname"] + "/3"]
new_doc1 = {"_id": "new_doc_1", "integer": -1, "string": "1"}
new_doc2 = {"_id": "new_doc_2", "integer": -2, "string": "2"}
new_doc3 = {"_id": "new_doc_3", "integer": -3, "string": "3"}
new_doc4 = {"_id": "new_doc_4", "integer": -4, "string": "4"}
db1.save(new_doc1)
db2.save(new_doc2)
db3.save(new_doc3)
db4.save(new_doc4)
# print "Querying map view with ?stale=ok"
(map_resp2, map_view_result2) = common.query(self._params, "mapview1", {"stale": "ok"})
map_etag2 = map_resp2.getheader("ETag")
self.assertEqual(map_view_result2, map_view_result, "Same response as before with ?stale=ok")
self.assertEqual(map_etag2, map_etag, "Same etag as before with ?stale=ok")
# print "Verifying set view group info"
info2 = common.get_set_view_info(self._params)
self.assertEqual(info2, info, "Same set view group info after ?stale=ok query")
# print "Querying map view with ?stale=update_after"
(map_resp3, map_view_result3) = common.query(self._params, "mapview1", {"stale": "update_after"})
map_etag3 = map_resp3.getheader("ETag")
self.assertEqual(map_view_result3, map_view_result, "Same response as before with ?stale=update_after")
self.assertEqual(map_etag3, map_etag, "Same etag as before with ?stale=updater_after")
time.sleep(5)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], ((self._params["ndocs"] / 4) + 1),
"right update seq for partition %d" % (i + 1))
# print "Querying map view without stale option"
(map_resp4, map_view_result4) = common.query(self._params, "mapview1")
map_etag4 = map_resp4.getheader("ETag")
self.assertEqual(map_view_result4["total_rows"], (self._params["ndocs"] + 4),
"Query returned %d total_rows" % (self._params["ndocs"] + 4))
self.assertEqual(len(map_view_result4["rows"]), (self._params["ndocs"] + 4),
"Query returned %d rows" % (self._params["ndocs"] + 4))
self.assertNotEqual(map_etag4, map_etag3, "New etag after ?stale=update_after")
common.test_keys_sorted(map_view_result4)
self.assertEqual(map_view_result4["rows"][0]["id"], new_doc4["_id"], "new_doc4 in view result")
self.assertEqual(map_view_result4["rows"][1]["id"], new_doc3["_id"], "new_doc3 in view result")
self.assertEqual(map_view_result4["rows"][2]["id"], new_doc2["_id"], "new_doc2 in view result")
self.assertEqual(map_view_result4["rows"][3]["id"], new_doc1["_id"], "new_doc1 in view result")
def do_test_maps(self):
# print "Querying map view in steady state"
(resp, view_result) = common.query(self._params, "mapview1")
etag = resp.getheader("ETag")
self.assertEqual(
view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(view_result)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Setting partition 4 as passive"
common.set_partition_states(self._params, passive=[3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [3],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp2, view_result2) = common.query(self._params, "mapview1")
etag2 = resp2.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(
view_result2["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result2["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(etag2, etag,
"Different Etag after setting passive partition")
common.test_keys_sorted(view_result2)
all_keys = {}
for r in view_result2["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 4 was made passive" %
key)
# print "Setting partition 4 state to active"
common.set_partition_states(self._params, active=[3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp3, view_result3) = common.query(self._params, "mapview1")
etag3 = resp3.getheader("ETag")
self.assertEqual(
view_result3["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result3["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
self.assertEqual(etag3, etag, "ETag is same as first query response")
common.test_keys_sorted(view_result3)
# print "Setting partitions 1 and 4 to passive state"
common.set_partition_states(self._params, passive=[0, 3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [1, 2],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [0, 3],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp4, view_result4) = common.query(self._params, "mapview1")
etag4 = resp4.getheader("ETag")
expected = self._params["ndocs"] / 2
self.assertEqual(
view_result4["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result4["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(etag4, etag,
"ETag is different from all previous responses")
self.assertNotEqual(etag4, etag2,
"ETag is different from all previous responses")
common.test_keys_sorted(view_result4)
all_keys = {}
for r in view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(1, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 1 was made passive" %
key)
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(
key in all_keys,
"Key %d not in result after partition 4 was made passive" %
key)
# print "Marking all partitions as passive and querying view again"
common.set_partition_states(self._params,
passive=range(self._params["nparts"]))
(resp5, view_result5) = common.query(self._params, "mapview1")
etag5 = resp5.getheader("ETag")
self.assertEqual(
view_result5["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result5["rows"]), 0, "Query returned 0 rows")
self.assertNotEqual(etag5, etag,
"ETag is different from all previous responses")
self.assertNotEqual(etag5, etag2,
"ETag is different from all previous responses")
self.assertNotEqual(etag5, etag3,
"ETag is different from all previous responses")
self.assertNotEqual(etag5, etag4,
"ETag is different from all previous responses")
# print "Setting all partitions to active state"
common.set_partition_states(self._params,
active=range(self._params["nparts"]))
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3],
"right active partitions list")
self.assertEqual(info["passive_partitions"], [],
"right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [],
"right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)],
(self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying view again"
(resp6, view_result6) = common.query(self._params, "mapview1")
etag6 = resp6.getheader("ETag")
self.assertEqual(
view_result6["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(view_result6["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
self.assertEqual(etag6, etag,
"ETag is the same from first view query response")
common.test_keys_sorted(view_result6)
def test_filter_partitions(self):
# print "Querying map view with ?partitions=[0,1,3]"
(map_resp, map_view_result) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag = map_resp.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(map_view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(map_view_result["rows"]), expected, "Query returned %d rows" % expected)
common.test_keys_sorted(map_view_result)
all_keys = {}
for r in map_view_result["rows"]:
all_keys[r["key"]] = True
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys, "Key %d from partition 3 is not in the result" % key)
# print "Disabling (making it passive) partition 2"
common.set_partition_states(self._params, passive = [1])
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp2, map_view_result2) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag2 = map_resp2.getheader("ETag")
self.assertEqual(map_view_result2, map_view_result, "Same result as before")
self.assertEqual(map_etag2, map_etag, "Same Etag as before")
# print "Marking partition 2 for cleanup"
common.set_partition_states(self._params, cleanup = [1])
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp3, map_view_result3) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag3 = map_resp3.getheader("ETag")
expected = self._params["ndocs"] / 2
self.assertEqual(len(map_view_result3["rows"]), expected,
"Query returned %d rows" % expected)
self.assertNotEqual(map_etag3, map_etag2, "Different Etag from before")
common.test_keys_sorted(map_view_result3)
all_keys = {}
for r in map_view_result3["rows"]:
all_keys[r["key"]] = True
for key in xrange(2, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d from partition 2 is not in the result" % key)
for key in xrange(3, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d from partition 3 is not in the result" % key)
# print "Triggering view compaction (to guarantee cleanup is complete)"
common.compact_set_view(self._params)
# print "Querying map view again with ?partitions=[0,1,3]"
(map_resp4, map_view_result4) = common.query(
self._params, "mapview1", {"partitions": json.dumps([0, 1, 3])})
map_etag4 = map_resp4.getheader("ETag")
# total_rows is different after cleanup
self.assertEquals(map_view_result4["rows"], map_view_result3["rows"], "Same result as before")
self.assertEquals(map_etag4, map_etag3, "Same Etag as before")
def test_compaction(self):
# print "Querying map view"
(map_resp, map_view_result) = common.query(self._params, "mapview1")
map_etag = map_resp.getheader("ETag")
self.assertEqual(map_view_result["total_rows"], self._params["ndocs"],
"Query returned %d total_rows" % self._params["ndocs"])
self.assertEqual(len(map_view_result["rows"]), self._params["ndocs"],
"Query returned %d rows" % self._params["ndocs"])
common.test_keys_sorted(map_view_result)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
# print "Querying reduce view"
(red_resp, red_view_result) = common.query(self._params, "redview1")
red_etag = red_resp.getheader("ETag")
self.assertEqual(len(red_view_result["rows"]), 1, "Query returned 1 row")
self.assertEqual(red_view_result["rows"][0]["value"], self._params["ndocs"],
"Non-grouped reduce value is %d" % self._params["ndocs"])
# print "Triggering view compaction"
common.compact_set_view(self._params)
# print "Verifying set view group info"
info2 = common.get_set_view_info(self._params)
self.assertEqual(info2["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info2["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info2["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info2["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertTrue(info2["disk_size"] < info["disk_size"], "Smaller disk_size after compaction")
# print "Querying map view"
(map_resp2, map_view_result2) = common.query(self._params, "mapview1")
map_etag2 = map_resp2.getheader("ETag")
self.assertEqual(map_view_result2, map_view_result, "Got same map response after compaction")
self.assertEqual(map_etag2, map_etag, "Same map view etag after compaction")
# print "Querying reduce view"
(red_resp2, red_view_result2) = common.query(self._params, "redview1")
red_etag2 = red_resp2.getheader("ETag")
self.assertEqual(red_view_result2, red_view_result, "Got same reduce response after compaction")
self.assertEqual(red_etag2, red_etag, "Same reduce view etag after compaction")
# print "Adding 2 new documents to partition 4"
server = self._params["server"]
db4 = server[self._params["setname"] + "/3"]
new_doc1 = {"_id": "9999999999999", "integer": 999999999999999999, "string": "999999999"}
new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
db4.save(new_doc1)
db4.save(new_doc2)
# print "Triggering view compaction"
common.compact_set_view(self._params, False)
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertEqual(info["compact_running"], True, "Compaction is flagged as running in group info")
# print "Triggering set view group index update"
common.query(self._params, "redview1")
# print "Waiting for set view compaction to finish"
compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0)
self.assertTrue(compaction_was_running, "Compaction was running when the view update was triggered")
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2, 3]:
if i == 3:
seq = (self._params["ndocs"] / 4) + 2
else:
seq = (self._params["ndocs"] / 4)
self.assertEqual(info["update_seqs"][str(i)], seq, "right update seq for partition %d" % (i + 1))
# print "Querying map view"
(map_resp3, map_view_result3) = common.query(self._params, "mapview1")
map_etag3 = map_resp3.getheader("ETag")
self.assertEqual(map_view_result3["total_rows"], (self._params["ndocs"] + 2),
"Query returned %d total_rows" % (self._params["ndocs"] + 2))
self.assertEqual(len(map_view_result3["rows"]), (self._params["ndocs"] + 2),
"Query returned %d rows" % (self._params["ndocs"] + 2))
self.assertNotEqual(map_etag3, map_etag, "Different map view etag after recompaction")
common.test_keys_sorted(map_view_result3)
self.assertEqual(map_view_result3["rows"][0]["id"], new_doc2["_id"], "new_doc2 in map view")
self.assertEqual(map_view_result3["rows"][-1]["id"], new_doc1["_id"], "new_doc1 in map view")
# print "Querying reduce view"
(red_resp3, red_view_result3) = common.query(self._params, "redview1")
red_etag3 = red_resp3.getheader("ETag")
self.assertEqual(red_view_result3["rows"][0]["value"], (self._params["ndocs"] + 2),
"Non-grouped reduce value is %d" % (self._params["ndocs"] + 2))
self.assertNotEqual(red_etag3, red_etag, "Different reduce view etag after compaction")
# print "Triggering another view compaction"
common.compact_set_view(self._params, False)
# print "Triggering partition 4 cleanup while compaction is ongoing"
common.set_partition_states(self._params, cleanup = [3])
# print "Verifying set view group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [3], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "paritition 4 not in group info update_seqs")
self.assertEqual(info["compact_running"], True, "Compaction is flagged as running in group info")
self.assertEqual(info["cleanup_running"], True, "Cleanup is flagged as running in group info")
compaction_was_running = (common.wait_set_view_compaction_complete(self._params) > 0)
self.assertTrue(compaction_was_running, "Compaction was running when the cleanup was triggered")
# print "Verifying set view group info"
info2 = common.get_set_view_info(self._params)
self.assertEqual(info2["active_partitions"], [0, 1, 2], "right active partitions list")
self.assertEqual(info2["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info2["cleanup_partitions"], [], "right cleanup partitions list")
for i in [0, 1, 2]:
self.assertEqual(info2["update_seqs"][str(i)], (self._params["ndocs"] / 4),
"right update seq for partition %d" % (i + 1))
self.assertFalse("3" in info["update_seqs"], "paritition 4 not in group info update_seqs")
self.assertTrue(info2["disk_size"] < info["disk_size"], "Smaller disk_size after compaction+cleanup")
# print "Querying map view"
(map_resp4, map_view_result4) = common.query(self._params, "mapview1")
map_etag4 = map_resp4.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(map_view_result4["total_rows"], expected,
"Query returned %d total_rows after recompact+cleanup" % expected)
self.assertEqual(len(map_view_result4["rows"]), expected,
"Query returned %d rows after recompact+cleanup" % expected)
self.assertNotEqual(map_etag4, map_etag, "Different map view etag after compaction+cleanup")
common.test_keys_sorted(map_view_result4)
all_keys = {}
for r in map_view_result4["rows"]:
all_keys[r["key"]] = True
for key in xrange(4, self._params["ndocs"], self._params["nparts"]):
self.assertFalse(key in all_keys,
"Key %d not in result after partition 4 cleanup triggered" % key)
self.assertFalse(new_doc1["integer"] in all_keys, "new_doc1 not included in view query result")
self.assertFalse(new_doc2["integer"] in all_keys, "new_doc2 not included in view query result")
self.assertNotEqual(map_view_result4, map_view_result3, "Different map view result after recompact+cleanup")
# print "Querying reduce view"
(red_resp4, red_view_result4) = common.query(self._params, "redview1")
red_etag4 = red_resp4.getheader("ETag")
expected = self._params["ndocs"] - (self._params["ndocs"] / 4)
self.assertEqual(red_view_result4["rows"][0]["value"], expected,
"Non-grouped reduce value is %d after recompact+cleanup" % (self._params["ndocs"] + 2))
self.assertNotEqual(red_etag4, red_etag3, "Different reduce view etag after compaction+cleanup")
def test_many_partitions(self):
total_doc_count = common.set_doc_count(self._params, range(self._params["nparts"]))
# print "Querying view"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], range(self._params["nparts"]), "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
# print "Adding 1 new document to each partition"
new_docs = []
for i in xrange(self._params["nparts"]):
server = self._params["server"]
db = self._params["server"][self._params["setname"] + "/" + str(i)]
value = total_doc_count + i + 1
new_doc = {
"_id": str(value),
"integer": value,
"string": str(value)
}
new_docs.append(new_doc)
db.save(new_doc)
new_total_doc_count = common.set_doc_count(self._params, range(self._params["nparts"]))
self.assertEqual(new_total_doc_count, (total_doc_count + len(new_docs)), "N documents were added")
total_doc_count = new_total_doc_count
# print "Querying view again"
(resp, view_result) = common.query(self._params, "mapview")
self.assertEqual(len(view_result["rows"]), total_doc_count, "number of received rows is %d" % total_doc_count)
self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count)
common.test_keys_sorted(view_result)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], range(self._params["nparts"]), "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
# print "Marking half of the partitions as passive"
passive = range(self._params["nparts"] / 2, self._params["nparts"])
active = range(self._params["nparts"] / 2)
common.set_partition_states(
self._params,
passive = range(self._params["nparts"] / 2, self._params["nparts"])
)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], active, "right active partitions list")
self.assertEqual(info["passive_partitions"], passive, "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
for i in xrange(self._params["nparts"]):
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))
# print "Querying view again"
(resp, view_result) = common.query(self._params, "mapview")
expected_row_count = common.set_doc_count(self._params, active)
self.assertEqual(len(view_result["rows"]), expected_row_count, "number of received rows is %d" % expected_row_count)
common.test_keys_sorted(view_result)
for row in view_result['rows']:
if row["key"] >= 2001:
key_part = ((row["key"] - 2000) % self._params["nparts"]) - 1
else:
key_part = (row["key"] % self._params["nparts"]) - 1
self.assertTrue(key_part in active, "Key %d from passive partition not in result set" % row["key"])
# print "Marking half of the partitions for cleanup"
common.set_partition_states(
self._params,
cleanup = passive
)
cleanup = passive
common.compact_set_view(self._params)
# print "Verifying group info"
info = common.get_set_view_info(self._params)
self.assertEqual(info["active_partitions"], active, "right active partitions list")
self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
self.assertEqual(info["stats"]["compactions"], 1, "1 compaction")
self.assertEqual(info["stats"]["cleanups"], 1, "1 full cleanup")
for i in active:
expected_seq = common.partition_update_seq(self._params, i)
self.assertEqual(info["update_seqs"][str(i)], expected_seq,
"right update seq number (%d) for partition %d" % (expected_seq, i + 1))