Ejemplo n.º 1
0
    def setUp(self):
        server = couchdb.Server(url="http://" + HOST)
        self._params_local = {
            "host": HOST,
            "ddoc": deepcopy(DDOC),
            "nparts": NUM_PARTS,
            "ndocs": NUM_DOCS,
            "setname": SET_NAME_LOCAL,
            "server": server
        }
        self._params_remote = {
            "host": HOST,
            "ddoc": deepcopy(DDOC),
            "nparts": NUM_PARTS,
            "start_id": NUM_DOCS,
            "ndocs": NUM_DOCS,
            "setname": SET_NAME_REMOTE,
            "server": server
        }
        # print "Creating databases"
        common.create_dbs(self._params_local)
        common.create_dbs(self._params_remote)
        common.populate(self._params_local)
        common.populate(self._params_remote)

        common.define_set_view(self._params_local, range(NUM_PARTS), [])
        common.define_set_view(self._params_remote, range(NUM_PARTS), [])
Ejemplo n.º 2
0
    def setUp(self):
        server = couchdb.Server(url = "http://" + HOST)
        self._params_local = {
            "host": HOST,
            "ddoc": deepcopy(DDOC),
            "nparts": NUM_PARTS,
            "ndocs": NUM_DOCS,
            "setname": SET_NAME_LOCAL,
            "server": server
            }
        self._params_remote = {
            "host": HOST,
            "ddoc": deepcopy(DDOC),
            "nparts": NUM_PARTS,
            "start_id": NUM_DOCS,
            "ndocs": NUM_DOCS,
            "setname": SET_NAME_REMOTE,
            "server": server
            }
        # print "Creating databases"
        common.create_dbs(self._params_local)
        common.create_dbs(self._params_remote)
        common.populate(self._params_local)
        common.populate(self._params_remote)

        common.define_set_view(self._params_local, range(NUM_PARTS), [])
        common.define_set_view(self._params_remote, range(NUM_PARTS), [])
Ejemplo n.º 3
0
 def setUp(self):
     self._params = {
         "host": HOST,
         "ddoc": DDOC,
         "nparts": NUM_PARTS,
         "ndocs": NUM_DOCS,
         "setname": SET_NAME,
         "server": couchdb.Server(url = "http://" + HOST)
         }
     # print "Creating databases"
     common.create_dbs(self._params)
     common.populate(self._params)
     common.define_set_view(self._params, range(NUM_PARTS), [])
Ejemplo n.º 4
0
 def setUp(self):
     self._params = {
         "host": HOST,
         "ddoc": DDOC,
         "nparts": NUM_PARTS,
         "ndocs": NUM_DOCS,
         "setname": SET_NAME,
         "server": couchdb.Server(url="http://" + HOST)
     }
     # print "Creating databases"
     common.create_dbs(self._params)
     common.populate(self._params)
     common.define_set_view(self._params, range(NUM_PARTS), [])
Ejemplo n.º 5
0
 def setUp(self):
     self._params = {
         "host": HOST,
         "ddoc": DDOC,
         "nparts": NUM_PARTS,
         "ndocs": NUM_DOCS,
         "setname": SET_NAME,
         "server": couchdb.Server(url = "http://" + HOST)
         }
     # print "Creating databases"
     common.create_dbs(self._params)
     common.populate(self._params)
     # print "Configuring set view with:"
     # print "\tmaximum of 8 partitions"
     # print "\tactive partitions = [0, 1, 2, 3, 4]"
     # print "\tpassive partitions = []"
     common.define_set_view(self._params, [0, 1, 2, 3], [])
Ejemplo n.º 6
0
 def setUp(self):
     self._params = {
         "host": HOST,
         "ddoc": DDOC,
         "nparts": NUM_PARTS,
         "ndocs": NUM_DOCS,
         "setname": SET_NAME,
         "server": couchdb.Server(url="http://" + HOST)
     }
     # print "Creating databases"
     common.create_dbs(self._params)
     common.populate(self._params)
     # print "Databases created"
     active_parts = range(self._params["nparts"])
     common.define_set_view(self._params, active_parts, [])
     common.set_config_parameter(self._params, "native_query_servers",
                                 "erlang",
                                 "{couch_native_process, start_link, []}")
Ejemplo n.º 7
0
 def setUp(self):
     self._params = {
         "host": HOST,
         "ddoc": DDOC,
         "nparts": NUM_PARTS,
         "ndocs": NUM_DOCS,
         "setname": SET_NAME,
         "server": couchdb.Server(url = "http://" + HOST)
         }
     # print "Creating databases"
     common.create_dbs(self._params)
     common.populate(self._params)
     # print "Databases created"
     active_parts = range(self._params["nparts"])
     common.define_set_view(self._params, active_parts, [])
     common.set_config_parameter(
         self._params,
         "native_query_servers",
         "erlang",
         "{couch_native_process, start_link, []}"
         )
Ejemplo n.º 8
0
 def tearDown(self):
     # print "Deleting test data"
     common.create_dbs(self._params, True)
Ejemplo n.º 9
0
 def tearDown(self):
     # print "Deleting test data"
     common.create_dbs(self._params, True)
Ejemplo n.º 10
0
 def tearDown(self):
     # print "Deleting test data"
     common.create_dbs(self._params_local, True)
     common.create_dbs(self._params_remote, True)
Ejemplo n.º 11
0
 def tearDown(self):
     # print "Deleting test data"
     common.create_dbs(self._params_local, True)
     common.create_dbs(self._params_remote, True)
Ejemplo n.º 12
0
    def do_test_replica_support(self):
        server = couchdb.Server(url = "http://" + HOST)
        local_params = {
            "host": HOST,
            "ddoc": deepcopy(DDOC),
            "nparts": NUM_PARTS,
            "ndocs": 50000,
            "setname": SET_NAME_LOCAL + "_2",
            "server": server
            }
        remote_params = {
            "host": HOST,
            "ddoc": deepcopy(DDOC),
            "nparts": NUM_PARTS,
            "start_id": 50000,
            "ndocs": 50000,
            "setname": SET_NAME_REMOTE + "_2",
            "server": server
            }
        # print "Creating databases"
        common.create_dbs(local_params)
        common.create_dbs(remote_params)
        common.populate(local_params)
        common.populate(remote_params)
        # print "Databases created"

        common.define_set_view(local_params, [0, 1], [], True)
        common.define_set_view(remote_params, [2, 3], [], True)
        common.add_replica_partitions(local_params, [2, 3])
        common.add_replica_partitions(remote_params, [0, 1])

        local_spec = self.set_spec(local_params["setname"], "mapview", [0, 1])
        remote_spec = self.set_spec(remote_params["setname"], "mapview", [2, 3])
        remote_merge = self.merge_spec(remote_params["host"], [], [remote_spec])

        # print "Triggering view update"
        full_spec = self.views_spec([remote_merge], [local_spec])
        _, result = self.query(local_params["host"], full_spec, {"limit": "1", "stale": "false"})

        info = common.get_set_view_info(local_params)
        seconds = 0
        while info["replica_group_info"]["stats"]["full_updates"] < 1:
            time.sleep(3)
            seconds += 3
            if seconds > 900:
                raise(Exception("timeout waiting for replica group full update"))
            info = common.get_set_view_info(local_params)

        info = common.get_set_view_info(remote_params)
        seconds = 0
        while info["replica_group_info"]["stats"]["full_updates"] < 1:
            time.sleep(3)
            seconds += 3
            if seconds > 900:
                raise(Exception("timeout waiting for replica group full update"))
            info = common.get_set_view_info(remote_params)

        local_spec = self.set_spec(local_params["setname"], "mapview", [0, 1])
        remote_spec = self.set_spec(remote_params["setname"], "mapview", [2, 3])
        remote_merge = self.merge_spec(remote_params["host"], [], [remote_spec])
        full_spec = self.views_spec([remote_merge], [local_spec])
        _, result = self.query(local_params["host"], full_spec, {"stale": "false"})
        self.assertEqual(len(result["rows"]), 50000, "response has 50000 rows")

        # print "Activating replica partitions"
        common.set_partition_states(local_params, active = [2, 3])
        common.set_partition_states(remote_params, active = [0, 1])

        # print "Querying with ?debug=true"
        local_spec = self.set_spec(local_params["setname"], "mapview", [0, 1, 2, 3])
        remote_spec = self.set_spec(remote_params["setname"], "mapview", [0, 1, 2, 3])
        remote_merge = self.merge_spec(remote_params["host"], [], [remote_spec])
        full_spec = self.views_spec([remote_merge], [local_spec])
        _, result = self.query(local_params["host"], full_spec, {"stale": "ok", "debug": "true"})

        self.assertEqual(len(result["rows"]), 100000, "response has 100000 rows")
        self.assertTrue("debug_info" in result, "result has debug info")
        self.assertTrue("local" in result["debug_info"], "result have debug info for local node")
        remote_node = "http://%s/_view_merge/" % remote_params["host"]
        self.assertTrue(remote_node in result["debug_info"], "result have debug info for remote node")

        next_key = 1
        next_part = 0
        i = 0
        while next_key < 100000:
            row = result["rows"][i]
            self.assertEqual(row["key"], next_key, "correct row key at iteration %d" % (i + 1))
            self.assertEqual(row["value"], str(next_key), "correct row value at iteration %d" % (i + 1))
            self.assertEqual(row["partition"], next_part, "correct row partition at iteration %d" % (i + 1))
            node = "local"
            if next_key > 50000:
                node = "http://%s/_view_merge/" % remote_params["host"]
            self.assertEqual(row["node"], node, "correct row node at iteration %d" % (i + 1))
            i += 1
            next_key += 1
            next_part = (next_part + 1) % 4

        # print "Deleting databases"
        common.create_dbs(local_params, True)
        common.create_dbs(remote_params, True)
Ejemplo n.º 13
0
    def do_test_updates(self):
        # print "Creating databases"
        common.create_dbs(self._params)
        common.populate(self._params)
        # print "Databases created"
        # print "Configuring set view with 1 active partition and 3 passive partitions"
        common.define_set_view(self._params, [0], [1, 2, 3])

        # print "Querying map view"
        (resp, view_result) = common.query(self._params, "mapview", {"limit": "20"})

        # print "Grabbing group info"
        info = common.get_set_view_info(self._params)
        self.assertEqual(info["active_partitions"], [0], "right active partitions list")
        self.assertEqual(info["passive_partitions"], [1, 2, 3], "right passive partitions list")
        self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
        self.assertEqual(info["update_seqs"]["0"], (self._params["ndocs"] / 4), "All data from partition 1 was indexed")

        sum_passive_seqs = 0
        for i in [1, 2, 3]:
            sum_passive_seqs += info["update_seqs"][str(i)]

        self.assertTrue(sum_passive_seqs < (self._params["ndocs"] - (self._params["ndocs"] / 4)),
                        "Passive partitions are still being indexed")
        self.assertEqual(info["updater_running"], True, "View updater still running")
        self.assertEqual(info["updater_state"], "updating_passive", "View updater in state 'updating_passive'")

        # print "Verifying view query response"
        self.assertEqual(len(view_result["rows"]), 20, "Query returned 20 rows")

        common.test_keys_sorted(view_result)
        self.assertEqual(view_result["rows"][0]["key"], 1, "First row has key 1")
        last_key_expected = range(1, self._params["ndocs"], self._params["nparts"])[19]
        self.assertEqual(view_result["rows"][-1]["key"], last_key_expected, "Last row has key %d" % last_key_expected)

        # print "Waiting for view updater to finish"
        while True:
            info = common.get_set_view_info(self._params)
            if not info["updater_running"]:
                break
            else:
                time.sleep(3)

        # print "Grabbing group info"
        info = common.get_set_view_info(self._params)
        self.assertEqual(info["active_partitions"], [0], "right active partitions list")
        self.assertEqual(info["passive_partitions"], [1, 2, 3], "right passive partitions list")
        self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")
        for i in [0, 1, 2, 3]:
            self.assertEqual(info["update_seqs"][str(i)], (self._params["ndocs"] / 4),
                             "Right update seq for partition %d" % (i + 1))
        self.assertEqual(info["updater_running"], False, "View updater not running anymore")
        self.assertEqual(info["updater_state"], "not_running", "View updater not running anymore")

        # print "Querying view again and validating the response"
        (resp, view_result) = common.query(self._params, "mapview")

        self.assertEqual(len(view_result["rows"]), (self._params["ndocs"] / 4), "Query returned %d rows" % (self._params["ndocs"] / 4))

        common.test_keys_sorted(view_result)
        self.assertEqual(view_result["rows"][0]["key"], 1, "First row has key 1")
        last_key_expected = range(1, self._params["ndocs"], self._params["nparts"])[-1]
        self.assertEqual(view_result["rows"][-1]["key"], last_key_expected, "Last row has key %d" % last_key_expected)
Ejemplo n.º 14
0
    def do_test_set_passive_partitions_when_updater_is_running(self):
        # print "Re-creating databases"
        common.create_dbs(self._params)
        common.populate(self._params)
        # print "Configuring set view with all partitions active"
        common.define_set_view(self._params, [0, 1, 2, 3], [])

        # print "Querying map view in steady state with ?stale=update_after"
        (resp, view_result) = common.query(self._params, "mapview", {"stale": "update_after"})

        self.assertEqual(len(view_result["rows"]), 0, "Received empty row set")
        self.assertEqual(view_result["total_rows"], 0, "Received empty row set")

        # print "Marking partition 4 as passive"
        common.set_partition_states(self._params, passive = [3])

        info = common.get_set_view_info(self._params)
        self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
        self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
        self.assertEqual(info["cleanup_partitions"], [], "right cleanup partitions list")

        # print "Waiting for the set view updater to finish"
        iterations = 0
        while True:
            info = common.get_set_view_info(self._params)
            if info["updater_running"]:
                iterations += 1
            else:
                break

        self.assertTrue(iterations > 0, "Updater was running when partition 4 was set to passive")
        # print "Verifying set view group info"
        info = common.get_set_view_info(self._params)
        self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
        self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
        self.assertEqual(info["cleanup_partitions"], [],  "right cleanup partitions list")

        # print "Querying map view again"
        (resp, view_result) = common.query(self._params, "mapview")

        doc_count = common.set_doc_count(self._params)
        expected_row_count = common.set_doc_count(self._params, [0, 1, 2])
        self.assertEqual(view_result["total_rows"], doc_count, "Query returned %d total_rows" % doc_count)
        self.assertEqual(len(view_result["rows"]), expected_row_count, "Query returned %d rows" % expected_row_count)

        common.test_keys_sorted(view_result)

        all_keys = {}
        for r in view_result["rows"]:
            all_keys[r["key"]] = True

        for key in xrange(4, doc_count, self._params["nparts"]):
            self.assertFalse(key in all_keys,
                             "Key %d not in result after partition 4 was set to passive" % key)

        # print "Adding 2 new documents to partition 4"
        server = self._params["server"]
        db4 = server[self._params["setname"] + "/3"]
        new_doc1 = {"_id": "999999999", "integer": 999999999, "string": "999999999"}
        new_doc2 = {"_id": "000", "integer": -1111, "string": "000"}
        db4.save(new_doc1)
        db4.save(new_doc2)

        # print "Querying map view again"
        (resp, view_result2) = common.query(self._params, "mapview")
        self.assertEqual(view_result2["rows"], view_result["rows"], "Same result set as before")

        # print "Verifying set view group info"
        info = common.get_set_view_info(self._params)
        self.assertEqual(info["active_partitions"], [0, 1, 2], "right active partitions list")
        self.assertEqual(info["passive_partitions"], [3], "right passive partitions list")
        self.assertEqual(info["cleanup_partitions"], [],  "right cleanup partitions list")

        total_doc_count = common.set_doc_count(self._params)
        # print "Changing partition 4 from passive to active"
        common.set_partition_states(self._params, active = [3])

        # print "Querying map view again"
        (resp, view_result) = common.query(self._params, "mapview")

        self.assertEqual(view_result["total_rows"], total_doc_count, "total_rows is %d" % total_doc_count)
        self.assertEqual(len(view_result["rows"]), total_doc_count, "number of rows returned is %d" % total_doc_count)
        common.test_keys_sorted(view_result)

        self.assertEqual(view_result["rows"][0]["key"], new_doc2["integer"], "new_doc2 reflected in result set at first row")
        self.assertEqual(view_result["rows"][-1]["key"], new_doc1["integer"], "new_doc1 reflected in result set at last row")

        # print "Verifying set view group info"
        info = common.get_set_view_info(self._params)
        self.assertEqual(info["active_partitions"], [0, 1, 2, 3], "right active partitions list")
        self.assertEqual(info["passive_partitions"], [], "right passive partitions list")
        self.assertEqual(info["cleanup_partitions"], [],  "right cleanup partitions list")
        for i in [0, 1, 2, 3]:
            expected_seq = common.partition_update_seq(self._params, i)
            self.assertEqual(info["update_seqs"][str(i)], expected_seq,
                             "right update seq number (%d) for partition %d" % (expected_seq, i + 1))