def test_peer_2_peer_sanity_push_one_shot_continuous(setup_p2p_test):
"""
1. Create ls_db1 database on LiteServ One
2. Create ls_db2 database on LiteServ Two
3. Add 10000 docs to LiteServ 1 ls_db1
4. Create one shot push replication LiteServ 1 ls_db1 -> LiteServ 2 ls_db2
5. sleep for 5 seconds
6. Create continuous push replication LiteServ 1 ls_db1 -> LiteServ 2 ls_db2
7. Add 10000 docs to LiteServ 1 ls_db1
8. Verify all docs replicate to LiteServ 2 ls_db2
9. Verify all docs show up in changes for LiteServ 2 ls_db2
"""
ls_url_one = setup_p2p_test["ls_url_one"]
ls_url_two = setup_p2p_test["ls_url_two"]
num_docs_per_db = 10000
log_info("ls_url_one: {}".format(ls_url_one))
log_info("ls_url_two: {}".format(ls_url_two))
client = MobileRestClient()
log_info("Creating databases")
ls_db1 = client.create_database(url=ls_url_one, name="ls_db1")
ls_db2 = client.create_database(url=ls_url_two, name="ls_db2")
bulk_docs = create_docs("test_ls_db1_oneshot", num_docs_per_db)
ls_db1_docs_oneshot = client.add_bulk_docs(ls_url_one, ls_db1, bulk_docs)
assert len(ls_db1_docs_oneshot) == num_docs_per_db
# Setup one shot push replication from LiteServ 1 ls_db1 to LiteServ 2 ls_db2
log_info("Setting up a one-shot push replication from ls_db1 to ls_db2")
push_repl = client.start_replication(
url=ls_url_one,
continuous=False,
from_db=ls_db1,
to_url=ls_url_two, to_db=ls_db2,
)
log_info("Replication ID: {}".format(push_repl))
client.verify_docs_present(url=ls_url_two, db=ls_db2, expected_docs=ls_db1_docs_oneshot)
client.verify_docs_in_changes(url=ls_url_two, db=ls_db2, expected_docs=ls_db1_docs_oneshot)
# Setup continuous push replication from LiteServ 1 ls_db1 to LiteServ 2 ls_db2
log_info("Setting up a continuous push replication from ls_db1 to ls_db2")
push_repl = client.start_replication(
url=ls_url_one,
continuous=True,
from_db=ls_db1,
to_url=ls_url_two, to_db=ls_db2,
)
log_info("Replication ID: {}".format(push_repl))
ls_db1_docs = client.add_docs(url=ls_url_one, db=ls_db1, number=num_docs_per_db, id_prefix="test_ls_db1")
assert len(ls_db1_docs) == num_docs_per_db
client.verify_docs_present(url=ls_url_two, db=ls_db2, expected_docs=ls_db1_docs_oneshot + ls_db1_docs)
client.verify_docs_in_changes(url=ls_url_two, db=ls_db2, expected_docs=ls_db1_docs_oneshot + ls_db1_docs)
def test_peer_2_peer_sanity_pull(setup_p2p_test, num_docs_per_db, seeded_db, attachments_generator):
"""
1. Create ls_db1 database on LiteServ One
2. Create ls_db2 database on LiteServ Two
3. Create continuous pull replication LiteServ 1 ls_db1 <- LiteServ 2 ls_db2
4. Add 5000 docs to LiteServ 2 ls_db2
5. Verify all docs replicate to LiteServ 1 ls_db1
6. Verify all docs show up in changes for LiteServ 1 ls_db1
"""
ls_url_one = setup_p2p_test["ls_url_one"]
ls_url_two = setup_p2p_test["ls_url_two"]
log_info("ls_url_one: {}".format(ls_url_one))
log_info("ls_url_two: {}".format(ls_url_two))
attachments = False
if attachments_generator:
log_info("Running test_peer_2_peer_sanity_pull with attachment {}".format(attachments_generator))
attachments = True
client = MobileRestClient()
log_info("Creating databases")
ls_db1 = client.create_database(url=ls_url_one, name="ls_db1")
ls_db2 = client.create_database(url=ls_url_two, name="ls_db2")
if seeded_db:
bulk_docs = create_docs("test_ls_db2_seed", num_docs_per_db)
ls_db2_docs_seed = client.add_bulk_docs(url=ls_url_two, db=ls_db2, docs=bulk_docs)
assert len(ls_db2_docs_seed) == num_docs_per_db
# Setup continuous pull replication from LiteServ 2 ls_db2 to LiteServ 1 ls_db1
pull_repl = client.start_replication(
url=ls_url_one,
continuous=True,
from_url=ls_url_two, from_db=ls_db2,
to_db=ls_db1
)
client.wait_for_replication_status_idle(url=ls_url_one, replication_id=pull_repl)
ls_db2_docs = client.add_docs(url=ls_url_two, db=ls_db2, number=num_docs_per_db, id_prefix="test_ls_db2", attachments_generator=attachments_generator)
assert len(ls_db2_docs) == num_docs_per_db
client.verify_docs_present(url=ls_url_one, db=ls_db1, expected_docs=ls_db2_docs, attachments=attachments)
client.verify_docs_in_changes(url=ls_url_one, db=ls_db1, expected_docs=ls_db2_docs)
total_ls_db2_docs = ls_db2_docs
if seeded_db:
total_ls_db2_docs += ls_db2_docs_seed
client.verify_docs_present(url=ls_url_one, db=ls_db1, expected_docs=total_ls_db2_docs, attachments=attachments)
client.verify_docs_in_changes(url=ls_url_one, db=ls_db1, expected_docs=total_ls_db2_docs)
def test_auto_prune_with_pull(setup_client_syncgateway_test):
"""Sanity test for autopruning with replication
1. Create a database on LiteServ (ls_db)
2. Add doc to sync gateway
3. Update doc 50 times on sync_gateway
4. Set up pull replication from sync_gateway db to LiteServ db
5. Verify number of revisions on client is default (20)
"""
cluster_config = setup_client_syncgateway_test["cluster_config"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_url = setup_client_syncgateway_test["sg_url"]
sg_admin_url = setup_client_syncgateway_test["sg_admin_url"]
client = MobileRestClient()
sg_helper = SyncGateway()
sg_helper.start_sync_gateway(
cluster_config=cluster_config, url=sg_url, config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS)
)
log_info("Running 'test_auto_prune_listener_sanity' ...")
log_info("ls_url: {}".format(ls_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_admin_url: {}".format(sg_admin_url))
num_docs = 1
num_revs = 50
sg_user_channels = ["NBC"]
sg_db = "db"
sg_user_name = "sg_user"
client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels)
sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name)
ls_db = client.create_database(url=ls_url, name="ls_db")
sg_db_docs = client.add_docs(
url=sg_url, db=sg_db, number=num_docs, id_prefix=sg_db, channels=sg_user_channels, auth=sg_session
)
assert len(sg_db_docs) == num_docs
sg_docs_update = client.update_docs(url=sg_url, db=sg_db, docs=sg_db_docs, number_updates=num_revs, auth=sg_session)
# Start continuous replication ls_db <- sg_db
repl_one = client.start_replication(url=ls_url, continuous=True, from_url=sg_admin_url, from_db=sg_db, to_db=ls_db)
client.wait_for_replication_status_idle(url=ls_url, replication_id=repl_one)
client.verify_docs_present(url=ls_url, db=ls_db, expected_docs=sg_docs_update)
client.verify_revs_num_for_docs(url=ls_url, db=ls_db, docs=sg_docs_update, expected_revs_per_doc=20)
def test_peer_2_peer_sanity_pull_one_shot(setup_p2p_test):
"""
1. Create ls_db1 database on LiteServ One
2. Create ls_db2 database on LiteServ Two
3. Add 10000 docs to LiteServ 2 ls_db2
4. Create one shot pull replication LiteServ 1 ls_db1 <- LiteServ 2 ls_db2
5. Verify all docs replicate to LiteServ 1 ls_db1
6. Verify all docs show up in changes for LiteServ 1 ls_db1
"""
ls_url_one = setup_p2p_test["ls_url_one"]
ls_url_two = setup_p2p_test["ls_url_two"]
num_docs_per_db = 10000
log_info("ls_url_one: {}".format(ls_url_one))
log_info("ls_url_two: {}".format(ls_url_two))
client = MobileRestClient()
log_info("Creating databases")
ls_db1 = client.create_database(url=ls_url_one, name="ls_db1")
ls_db2 = client.create_database(url=ls_url_two, name="ls_db2")
bulk_docs = create_docs("test_ls_db2", num_docs_per_db)
ls_db2_docs = client.add_bulk_docs(url=ls_url_two, db=ls_db2, docs=bulk_docs)
assert len(ls_db2_docs) == num_docs_per_db
# Setup one shot pull replication from LiteServ 2 ls_db2 to LiteServ 1 ls_db1
log_info("Setting up a one-shot pull replication from ls_db2 to ls_db1")
pull_repl = client.start_replication(
url=ls_url_one,
continuous=False,
from_url=ls_url_two, from_db=ls_db2,
to_db=ls_db1
)
log_info("Replication ID: {}".format(pull_repl))
client.verify_docs_present(url=ls_url_one, db=ls_db1, expected_docs=ls_db2_docs)
client.verify_docs_in_changes(url=ls_url_one, db=ls_db1, expected_docs=ls_db2_docs)
def test_peer_2_peer_sanity_pull(setup_p2p_test):
"""
1. Create ls_db1 database on LiteServ One
2. Create ls_db2 database on LiteServ Two
3. Create continuous pull replication LiteServ 1 ls_db1 <- LiteServ 2 ls_db2
4. Add 5000 docs to LiteServ 2 ls_db2
5. Verify all docs replicate to LiteServ 1 ls_db1
6. Verify all docs show up in changes for LiteServ 1 ls_db1
"""
ls_url_one = setup_p2p_test["ls_url_one"]
ls_url_two = setup_p2p_test["ls_url_two"]
num_docs_per_db = 5000
log_info("ls_url_one: {}".format(ls_url_one))
log_info("ls_url_two: {}".format(ls_url_two))
client = MobileRestClient()
log_info("Creating databases")
ls_db1 = client.create_database(url=ls_url_one, name="ls_db1")
ls_db2 = client.create_database(url=ls_url_two, name="ls_db2")
# Setup continuous pull replication from LiteServ 2 ls_db2 to LiteServ 1 ls_db1
pull_repl = client.start_replication(
url=ls_url_one,
continuous=True,
from_url=ls_url_two, from_db=ls_db2,
to_db=ls_db1
)
client.wait_for_replication_status_idle(url=ls_url_one, replication_id=pull_repl)
ls_db2_docs = client.add_docs(url=ls_url_two, db=ls_db2, number=num_docs_per_db, id_prefix="test_ls_db2")
assert len(ls_db2_docs) == num_docs_per_db
client.verify_docs_present(url=ls_url_one, db=ls_db1, expected_docs=ls_db2_docs)
client.verify_docs_in_changes(url=ls_url_one, db=ls_db1, expected_docs=ls_db2_docs)
def test_inline_large_attachments(setup_client_syncgateway_test):
"""
1. Start LiteServ and Sync Gateway
2. Create 2 databases on LiteServ (ls_db1, ls_db2)
3. Start continuous push replication from ls_db1 to sg_db
4. Start continuous pull replication from sg_db to ls_db2
5. PUT 5 large inline attachments to ls_db1
6. DELETE the docs on ls_db1
7. PUT same 5 large inline attachments to ls_db1
8. Verify docs replicate to ls_db2
9. Purge ls_db1
10. Verify docs removed
"""
log_info("Running 'test_inline_large_attachments' ...")
cluster_config = setup_client_syncgateway_test["cluster_config"]
sg_mode = setup_client_syncgateway_test["sg_mode"]
sg_url = setup_client_syncgateway_test["sg_url"]
sg_url_admin = setup_client_syncgateway_test["sg_admin_url"]
ls_url = setup_client_syncgateway_test["ls_url"]
log_info("ls_url: {}".format(ls_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
ls_db1 = "ls_db1"
ls_db2 = "ls_db2"
sg_db = "db"
# Reset cluster to ensure no data in system
sg_config = sync_gateway_config_path_for_mode(
"listener_tests/listener_tests", sg_mode)
c = cluster.Cluster(config=cluster_config)
c.reset(sg_config_path=sg_config)
client = MobileRestClient()
client.create_database(ls_url, ls_db1)
client.create_database(ls_url, ls_db2)
# Start continuous push replication from ls_db1 -> sg_db
client.start_replication(url=ls_url,
continuous=True,
from_db=ls_db1,
to_url=sg_url_admin,
to_db=sg_db)
# Start continuous push replication from sg_db -> ls_db2
client.start_replication(url=ls_url,
continuous=True,
from_url=sg_url_admin,
from_db=sg_db,
to_db=ls_db2)
# doc with 2.36 PNG attachment
attachment_docs = []
for i in range(5):
atts = attachment.load_from_data_dir(["golden_gate_large.jpg"])
doc = document.create_doc(doc_id="large_attach_{}".format(i),
attachments=atts,
channels=["ABC"])
attachment_docs.append(doc)
# add large attachments to ls_db1
docs = []
for doc in attachment_docs:
docs.append(client.add_doc(ls_url, ls_db1, doc, use_post=False))
# Delete docs
client.delete_docs(ls_url, ls_db1, docs)
client.verify_docs_deleted(ls_url, ls_db1, docs)
# Recreated docs
recreated_docs = []
for doc in attachment_docs:
recreated_docs.append(
client.add_doc(ls_url, ls_db1, doc, use_post=False))
client.verify_docs_present(ls_url, ls_db1, recreated_docs)
client.verify_docs_present(sg_url_admin, sg_db, recreated_docs)
client.verify_docs_present(ls_url, ls_db2, recreated_docs)
purged_docs = client.purge_docs(ls_url, ls_db1, recreated_docs)
log_info(purged_docs)
client.verify_docs_deleted(ls_url,
ls_db1,
recreated_docs,
reason="missing")
def test_stale_revision_should_not_be_in_the_index(setup_client_syncgateway_test):
"""original ticket: https://github.com/couchbase/couchbase-lite-android/issues/855
scenario:
1. Running sync_gateway
2. Create database and starts both push and pull replicators through client REST API
3. Create two or more views through client REST API
4. Add doc, and verify doc is index with current revision through client REST API
5. Make sure document is pushed to sync gateway through sync gateway REST API
6. Update doc with sync gateway (not client side) through sync gateway REST API
7. Make sure updated document is pull replicated to client through client REST API
8. Make sure updated document is indexed through client REST API
9. Make sure stale revision is deleted from index. through client REST API
10. Pass criteria
"""
cluster_config = setup_client_syncgateway_test["cluster_config"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_url = setup_client_syncgateway_test["sg_url"]
sg_admin_url = setup_client_syncgateway_test["sg_admin_url"]
num_docs = 10
num_revs = 100
d_doc_name = "dd"
sg_db = "db"
sg_user_name = "sg_user"
sg_helper = SyncGateway()
sg_helper.start_sync_gateway(
cluster_config=cluster_config,
url=sg_url,
config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS)
)
log_info("Running 'test_stale_revision_should_not_be_in_the_index'")
log_info("ls_url: {}".format(ls_url))
log_info("sg_admin_url: {}".format(sg_admin_url))
log_info("sg_url: {}".format(sg_url))
log_info("num_docs: {}".format(num_docs))
log_info("num_revs: {}".format(num_revs))
client = MobileRestClient()
sg_user_channels = ["NBC"]
client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels)
sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name)
view = """{
"language" : "javascript",
"views" : {
"content_view" : {
"map" : "function(doc, meta) { if (doc.content) { emit(doc._id, doc._rev); } }"
},
"update_view" : {
"map" : "function(doc, meta) { emit(doc.updates, null); }"
}
}
}"""
ls_db = client.create_database(url=ls_url, name="ls_db")
# Setup continuous push / pull replication from ls_db1 to sg_db
client.start_replication(
url=ls_url,
continuous=True,
from_db=ls_db,
to_url=sg_admin_url, to_db=sg_db
)
client.start_replication(
url=ls_url,
continuous=True,
from_url=sg_admin_url, from_db=sg_db,
to_db=ls_db
)
design_doc_id = client.add_design_doc(url=ls_url, db=ls_db, name=d_doc_name, doc=view)
client.get_doc(url=ls_url, db=ls_db, doc_id=design_doc_id)
doc_body = document.create_doc(doc_id="doc_1", content={"hi": "I should be in the view"}, channels=sg_user_channels)
log_info(doc_body)
doc_body_2 = document.create_doc(doc_id="doc_2", channels=sg_user_channels)
doc = client.add_doc(url=ls_url, db=ls_db, doc=doc_body)
doc_2 = client.add_doc(url=ls_url, db=ls_db, doc=doc_body_2)
content_view_rows = client.get_view(url=ls_url, db=ls_db, design_doc_id=design_doc_id, view_name="content_view")
client.verify_view_row_num(view_response=content_view_rows, expected_num_rows=1)
update_view_rows = client.get_view(url=ls_url, db=ls_db, design_doc_id=design_doc_id, view_name="update_view")
client.verify_view_row_num(view_response=update_view_rows, expected_num_rows=2)
expected_docs_list = [doc, doc_2]
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=expected_docs_list, auth=sg_session)
updated_doc = client.update_doc(url=sg_url, db=sg_db, doc_id=doc["id"], number_updates=10, auth=sg_session)
client.verify_docs_present(url=ls_url, db=ls_db, expected_docs=updated_doc)
content_view_rows_2 = client.get_view(url=ls_url, db=ls_db, design_doc_id=design_doc_id, view_name="content_view")
client.verify_view_row_num(view_response=content_view_rows_2, expected_num_rows=1)
client.verify_view_contains_keys(view_response=content_view_rows_2, keys=doc["id"])
client.verify_view_contains_values(view_response=content_view_rows_2, values=updated_doc["rev"])
def test_peer_2_peer_sanity(setup_p2p_test):
"""
1. Sanity P2P Scenario
2. Launch LiteServ 1 and LiteServ 2
3. Create a database on each LiteServ
4. Start continuous push pull replication from each db to the other
5. Add docs to each db
6. Verify the docs show up at each db
7. Verify the docs show up in the database's changes feed.
"""
ls_url_one = setup_p2p_test["ls_url_one"]
ls_url_two = setup_p2p_test["ls_url_two"]
num_docs_per_db = 1000
log_info("ls_url_one: {}".format(ls_url_one))
log_info("ls_url_two: {}".format(ls_url_two))
client = MobileRestClient()
log_info("Creating databases")
ls_db1 = client.create_database(url=ls_url_one, name="ls_db1")
ls_db2 = client.create_database(url=ls_url_two, name="ls_db2")
# Setup continuous push / pull replication from LiteServ 1 ls_db1 to LiteServ 2 ls_db2
repl_one = client.start_replication(
url=ls_url_one,
continuous=True,
from_db=ls_db1,
to_url=ls_url_two, to_db=ls_db2
)
repl_two = client.start_replication(
url=ls_url_one,
continuous=True,
from_url=ls_url_two, from_db=ls_db2,
to_db=ls_db1
)
# Setup continuous push / pull replication from LiteServ 2 ls_db2 to LiteServ 1 ls_db1
repl_three = client.start_replication(
url=ls_url_two,
continuous=True,
from_db=ls_db2,
to_url=ls_url_one, to_db=ls_db1
)
repl_four = client.start_replication(
url=ls_url_two,
continuous=True,
from_url=ls_url_one, from_db=ls_db1,
to_db=ls_db2
)
client.wait_for_replication_status_idle(url=ls_url_one, replication_id=repl_one)
client.wait_for_replication_status_idle(url=ls_url_one, replication_id=repl_two)
client.wait_for_replication_status_idle(url=ls_url_two, replication_id=repl_three)
client.wait_for_replication_status_idle(url=ls_url_two, replication_id=repl_four)
ls_url_one_replications = client.get_replications(ls_url_one)
assert len(ls_url_one_replications) == 2
ls_url_two_replications = client.get_replications(ls_url_two)
assert len(ls_url_two_replications) == 2
ls_db1_docs = client.add_docs(url=ls_url_one, db=ls_db1, number=num_docs_per_db, id_prefix="test_ls_db1")
assert len(ls_db1_docs) == num_docs_per_db
ls_db2_docs = client.add_docs(url=ls_url_two, db=ls_db2, number=num_docs_per_db, id_prefix="test_ls_db2")
assert len(ls_db2_docs) == num_docs_per_db
all_docs = client.merge(ls_db1_docs, ls_db2_docs)
assert len(all_docs) == 2000
client.verify_docs_present(url=ls_url_one, db=ls_db1, expected_docs=all_docs)
client.verify_docs_present(url=ls_url_two, db=ls_db2, expected_docs=all_docs)
client.verify_docs_in_changes(url=ls_url_one, db=ls_db1, expected_docs=all_docs)
client.verify_docs_in_changes(url=ls_url_two, db=ls_db2, expected_docs=all_docs)
def test_take_all_sgaccels_down(params_from_base_test_setup, sg_conf):
"""
Scenario that takes all sync_gateway accel nodes offline during doc load.
After bring the nodes back online during load, the reshard of the DCP feed is verified.
The changes feed is verified that all docs show up.
1. Start doc load (1000 doc)
2. Take all sg_accel nodes down in parallel
3. Verify node are down
4. Wait for doc adds to complete, store "doc_push_result_1"
5. Verify "doc_push_result_1" docs added
6. Start doc load (1000 docs)
7. Wait for 5. to complete, store "doc_push_result_2"
8. Verify "doc_push_result_2" docs added
9. Start another doc load (1000 docs)
10. Bring up nodes in parallel
11. poll on p-index reshard
12. Wait for 9. to complete, store "doc_push_result_3"
13. Verify "doc_push_result_3" docs added
14. Verify "doc_push_result_1" + "doc_push_result_2" + "doc_push_result_3" show up in _changes feed
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_dcp_reshard_single_sg_accel_goes_down_and_up'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_util = ClusterKeywords()
topology = cluster_util.get_cluster_topology(cluster_conf)
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
num_docs = 1000
client = MobileRestClient()
doc_pusher_user_info = userinfo.UserInfo("doc_pusher", "pass", channels=["A"], roles=[])
doc_pusher_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=doc_pusher_user_info.name,
password=doc_pusher_user_info.password,
channels=doc_pusher_user_info.channels
)
a_user_info = userinfo.UserInfo("a_user", "pass", channels=["A"], roles=[])
client.create_user(
url=sg_admin_url,
db=sg_db,
name=a_user_info.name,
password=a_user_info.password,
channels=a_user_info.channels
)
a_user_session = client.create_session(
url=sg_admin_url,
db=sg_db,
name=a_user_info.name,
password=a_user_info.password
)
# Shutdown all accel nodes in parallel
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as ex:
# Start adding docs
docs_1 = document.create_docs(None, num_docs, channels=doc_pusher_user_info.channels)
docs_1_task = ex.submit(client.add_bulk_docs, url=sg_url, db=sg_db, docs=docs_1, auth=doc_pusher_auth)
# Take down all access nodes
log_info("Shutting down sg_accels: [{}, {}, {}] ...".format(
cluster.sg_accels[0],
cluster.sg_accels[1],
cluster.sg_accels[2]
))
sg_accel_down_task_1 = ex.submit(cluster.sg_accels[0].stop)
sg_accel_down_task_2 = ex.submit(cluster.sg_accels[1].stop)
sg_accel_down_task_3 = ex.submit(cluster.sg_accels[2].stop)
assert sg_accel_down_task_1.result() == 0
assert sg_accel_down_task_2.result() == 0
assert sg_accel_down_task_3.result() == 0
# Block until bulk_docs is complete
doc_push_result_1 = docs_1_task.result()
assert len(doc_push_result_1) == num_docs
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=doc_push_result_1, auth=doc_pusher_auth)
# Load sync_gateway with another batch of docs while the sg_accel nodes are offline
docs_2_bodies = document.create_docs(None, num_docs, channels=doc_pusher_user_info.channels)
docs_push_result_2 = client.add_bulk_docs(url=sg_url, db=sg_db, docs=docs_2_bodies, auth=doc_pusher_auth)
assert len(docs_push_result_2) == num_docs
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=docs_push_result_2, auth=doc_pusher_auth)
# Start loading Sync Gateway with another set of docs while bringing the sg_accel nodes online
docs_3 = document.create_docs(None, num_docs, channels=doc_pusher_user_info.channels)
docs_3_task = ex.submit(client.add_bulk_docs, url=sg_url, db=sg_db, docs=docs_3, auth=doc_pusher_auth)
# Bring all the sg_accel nodes back up
# Take down all access nodes
log_info("Starting sg_accels: [{}, {}, {}] ...".format(
cluster.sg_accels[0],
cluster.sg_accels[1],
cluster.sg_accels[2]
))
sg_accel_up_task_1 = ex.submit(cluster.sg_accels[0].start, sg_conf)
sg_accel_up_task_2 = ex.submit(cluster.sg_accels[1].start, sg_conf)
sg_accel_up_task_3 = ex.submit(cluster.sg_accels[2].start, sg_conf)
assert sg_accel_up_task_1.result() == 0
assert sg_accel_up_task_2.result() == 0
assert sg_accel_up_task_3.result() == 0
# Wait for pindex to reshard correctly
assert cluster.validate_cbgt_pindex_distribution_retry(3)
# Block until second bulk_docs is complete
doc_push_result_3 = docs_3_task.result()
assert len(doc_push_result_3) == num_docs
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=doc_push_result_3, auth=doc_pusher_auth)
# Combine the 3 push results and make sure the changes propagate to a_user
# a_user has access to the doc's channel.
log_info("Verifying all the changes show up for 'a_user' ...")
all_docs = doc_push_result_1 + docs_push_result_2 + doc_push_result_3
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=all_docs, auth=a_user_session, polling_interval=2)
def test_server_goes_down_sanity(params_from_base_test_setup):
"""
1. Start with a two node couchbase server cluster
2. Starting adding docs
3. Kill one of the server nodes and signal completion
4. Stop adding docs
5. Verify that that the expected docs are present and in the changes feed.
6. Start server again and add to cluster
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
cluster_helper = ClusterKeywords()
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf_path)
topology = cluster_helper.get_cluster_topology(cluster_config)
admin_sg = topology["sync_gateways"][0]["admin"]
sg_url = topology["sync_gateways"][0]["public"]
coucbase_servers = topology["couchbase_servers"]
cbs_one_url = coucbase_servers[0]
cbs_two_url = coucbase_servers[1]
log_info("Running: 'test_server_goes_down_sanity'")
log_info("cluster_config: {}".format(cluster_config))
log_info("admin_sg: {}".format(admin_sg))
log_info("sg_url: {}".format(sg_url))
log_info("cbs_one_url: {}".format(cbs_one_url))
log_info("cbs_two_url: {}".format(cbs_two_url))
sg_db = "db"
num_docs = 100
sg_user_name = "seth"
sg_user_password = "******"
channels = ["ABC", "CBS"]
client = MobileRestClient()
main_server = CouchbaseServer(cbs_one_url)
flakey_server = CouchbaseServer(cbs_two_url)
client.create_user(admin_sg, sg_db, sg_user_name, sg_user_password, channels=channels)
session = client.create_session(admin_sg, sg_db, sg_user_name)
# Stop second server
flakey_server.stop()
# Try to add 100 docs in a loop until all succeed, if the never do, fail with timeout
errors = num_docs
# Wait 30 seconds for auto failover
# (Minimum value suggested - http://docs.couchbase.com/admin/admin/Tasks/tasks-nodeFailover.html)
# + 15 seconds to add docs
timeout = 45
start = time.time()
successful_add = False
while not successful_add:
# Fail tests if all docs do not succeed before timeout
if (time.time() - start) > timeout:
# Bring server back up before failing the test
flakey_server.start()
main_server.rebalance_in(coucbase_servers, flakey_server)
raise TimeoutError("Failed to successfully put docs before timeout")
try:
docs = client.add_docs(url=sg_url, db=sg_db, number=num_docs, id_prefix=None, auth=session, channels=channels)
# If the above add doc does not throw, it was a successfull add.
successful_add = True
except requests.exceptions.HTTPError as he:
log_info("Failed to add docs: {}".format(he))
log_info("Seeing: {} errors".format(errors))
time.sleep(1)
assert len(docs) == 100
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=docs, auth=session)
try:
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=docs, auth=session, polling_interval=5)
except keywords.exceptions.TimeoutException:
# timeout verifying docs. Bring server back in to restore topology, then fail
# Failing due to https://github.com/couchbase/sync_gateway/issues/2197
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
raise keywords.exceptions.TimeoutException("Failed to get all changes")
# Test succeeded without timeout, bring server back into topology
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
# Make sure all docs were not added before server was
log_info("test_server_goes_down_sanity complete!")
def test_replication_with_multiple_client_dbs_and_single_sync_gateway_db(setup_client_syncgateway_test):
"""Test replication from multiple client dbs to one sync_gateway db"""
cluster_config = setup_client_syncgateway_test["cluster_config"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_url = setup_client_syncgateway_test["sg_url"]
sg_admin_url = setup_client_syncgateway_test["sg_admin_url"]
num_docs = 1000
sg_helper = SyncGateway()
sg_helper.start_sync_gateway(
cluster_config=cluster_config,
url=sg_url,
config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS)
)
log_info("Running 'test_replication_with_multiple_client_dbs_and_single_sync_gateway_db'")
log_info("ls_url: {}".format(ls_url))
log_info("sg_admin_url: {}".format(sg_admin_url))
log_info("sg_url: {}".format(sg_url))
client = MobileRestClient()
ls_db1 = client.create_database(url=ls_url, name="ls_db1")
ls_db2 = client.create_database(url=ls_url, name="ls_db2")
sg_db = client.create_database(url=sg_admin_url, name="sg_db", server="walrus:")
# Setup continuous push / pull replication from ls_db1 to sg_db
client.start_replication(
url=ls_url,
continuous=True,
from_db=ls_db1,
to_url=sg_admin_url, to_db=sg_db
)
client.start_replication(
url=ls_url,
continuous=True,
from_url=sg_admin_url, from_db=sg_db,
to_db=ls_db1
)
# Setup continuous push / pull replication from ls_db2 to sg_db
client.start_replication(
url=ls_url,
continuous=True,
from_db=ls_db2,
to_url=sg_admin_url, to_db=sg_db
)
client.start_replication(
url=ls_url,
continuous=True,
from_url=sg_admin_url, from_db=sg_db,
to_db=ls_db2
)
ls_db_one_docs = client.add_docs(url=ls_url, db=ls_db1, number=num_docs, id_prefix=ls_db1)
assert len(ls_db_one_docs) == 1000
ls_db_two_docs = client.add_docs(url=ls_url, db=ls_db2, number=num_docs, id_prefix=ls_db2)
assert len(ls_db_two_docs) == 1000
ls_db1_db2_docs = ls_db_one_docs + ls_db_two_docs
client.verify_docs_present(url=ls_url, db=ls_db1, expected_docs=ls_db1_db2_docs)
client.verify_docs_present(url=ls_url, db=ls_db2, expected_docs=ls_db1_db2_docs)
client.verify_docs_present(url=sg_admin_url, db=sg_db, expected_docs=ls_db1_db2_docs)
client.verify_docs_in_changes(url=sg_admin_url, db=sg_db, expected_docs=ls_db1_db2_docs)
client.verify_docs_in_changes(url=ls_url, db=ls_db1, expected_docs=ls_db1_db2_docs)
client.verify_docs_in_changes(url=ls_url, db=ls_db2, expected_docs=ls_db1_db2_docs)
def test_replication_with_session_cookie(setup_client_syncgateway_test):
"""Regression test for https://github.com/couchbase/couchbase-lite-android/issues/817
1. SyncGateway Config with guest disabled = true and One user added (e.g. user1 / 1234)
2. Create a new session on SGW for the user1 by using POST /_session.
Capture the SyncGatewaySession cookie from the set-cookie in the response header.
3. Start continuous push and pull replicator on the LiteServ with SyncGatewaySession cookie.
Make sure that both replicators start correctly
4. Delete the session from SGW by sending DELETE /_sessions/ to SGW
5. Cancel both push and pull replicator on the LiteServ
6. Repeat step 1 and 2
"""
ls_db = "ls_db"
sg_db = "db"
cluster_config = setup_client_syncgateway_test["cluster_config"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_url = setup_client_syncgateway_test["sg_url"]
sg_admin_url = setup_client_syncgateway_test["sg_admin_url"]
sg_helper = SyncGateway()
sg_helper.start_sync_gateway(
cluster_config=cluster_config,
url=sg_url,
config="{}/walrus-user.json".format(SYNC_GATEWAY_CONFIGS)
)
log_info("Running 'test_replication_with_session_cookie'")
log_info("ls_url: {}".format(ls_url))
log_info("sg_admin_url: {}".format(sg_admin_url))
log_info("sg_url: {}".format(sg_url))
client = MobileRestClient()
client.create_database(url=ls_url, name=ls_db)
# Get session header for user_1
session_header = client.create_session_header(url=sg_url, db=sg_db, name="user_1", password="******")
# Get session id from header
session_parts = re.split("=|;", session_header)
session_id = session_parts[1]
log_info("{}: {}".format(session_parts[0], session_id))
session = (session_parts[0], session_id)
# Start authenticated push replication
repl_one = client.start_replication(
url=ls_url,
continuous=True,
from_db=ls_db,
to_url=sg_url,
to_db=sg_db,
to_auth=session_header
)
# Start authenticated pull replication
repl_two = client.start_replication(
url=ls_url,
continuous=True,
from_url=sg_url,
from_db=sg_db,
from_auth=session_header,
to_db=ls_db,
)
# Wait for 2 replications to be 'Idle', On .NET they may not be immediately available via _active_tasks
client.wait_for_replication_status_idle(ls_url, repl_one)
client.wait_for_replication_status_idle(ls_url, repl_two)
replications = client.get_replications(ls_url)
assert len(replications) == 2, "2 replications (push / pull should be running)"
num_docs_pushed = 100
# Sanity test docs
ls_docs = client.add_docs(url=ls_url, db=ls_db, number=num_docs_pushed, id_prefix="ls_doc", channels=["ABC"])
assert len(ls_docs) == num_docs_pushed
sg_docs = client.add_docs(url=sg_url, db=sg_db, number=num_docs_pushed, id_prefix="sg_doc", auth=session, channels=["ABC"])
assert len(sg_docs) == num_docs_pushed
all_docs = client.merge(ls_docs, sg_docs)
log_info(all_docs)
client.verify_docs_present(url=sg_admin_url, db=sg_db, expected_docs=all_docs)
client.verify_docs_present(url=ls_url, db=ls_db, expected_docs=all_docs)
# GET from session endpoint /{db}/_session/{session-id}
session = client.get_session(url=sg_admin_url, db=sg_db, session_id=session_id)
assert len(session["userCtx"]["channels"]) == 2, "There should be only 2 channels for the user"
assert "ABC" in session["userCtx"]["channels"], "The channel info should contain 'ABC'"
assert session["userCtx"]["name"] == "user_1", "The user should have the name 'user_1'"
assert len(session["authentication_handlers"]) == 2, "There should be 2 authentication_handlers"
assert "default" in session["authentication_handlers"], "Did not find 'default' in authentication_headers"
assert "cookie" in session["authentication_handlers"], "Did not find 'cookie' in authentication_headers"
log_info("SESSIONs: {}".format(session))
# Delete session via sg admin port and _user rest endpoint
client.delete_session(url=sg_admin_url, db=sg_db, user_name="user_1", session_id=session_id)
# Make sure session is deleted
try:
session = client.get_session(url=sg_admin_url, db=sg_db, session_id=session_id)
except HTTPError as he:
expected_error_code = he.response.status_code
log_info(expected_error_code)
assert expected_error_code == 404, "Expected 404 status, actual {}".format(expected_error_code)
# Cancel the replications
# Stop repl_one
client.stop_replication(
url=ls_url,
continuous=True,
from_db=ls_db,
to_url=sg_url,
to_db=sg_db,
to_auth=session_header
)
# Stop repl_two
client.stop_replication(
url=ls_url,
continuous=True,
from_url=sg_url,
from_db=sg_db,
from_auth=session_header,
to_db=ls_db,
)
client.wait_for_no_replications(ls_url)
replications = client.get_replications(ls_url)
assert len(replications) == 0, "All replications should be stopped"
# Create new session and new push / pull replications
session_header = client.create_session_header(url=sg_url, db=sg_db, name="user_1", password="******")
# Get session id from header
session_parts = re.split("=|;", session_header)
session_id = session_parts[1]
log_info("{}: {}".format(session_parts[0], session_id))
# Start authenticated push replication
repl_one = client.start_replication(
url=ls_url,
continuous=True,
from_db=ls_db,
to_url=sg_url,
to_db=sg_db,
to_auth=session_header
)
# Start authenticated pull replication
repl_two = client.start_replication(
url=ls_url,
continuous=True,
from_url=sg_url,
from_db=sg_db,
from_auth=session_header,
to_db=ls_db,
)
replications = client.get_replications(ls_url)
assert len(replications) == 2, "2 replications (push / pull should be running), found: {}".format(2)
session = client.get_session(url=sg_admin_url, db=sg_db, session_id=session_id)
assert len(session["userCtx"]["channels"]) == 2, "There should be only 2 channels for the user"
assert "ABC" in session["userCtx"]["channels"], "The channel info should contain 'ABC'"
assert session["userCtx"]["name"] == "user_1", "The user should have the name 'user_1'"
assert len(session["authentication_handlers"]) == 2, "There should be 2 authentication_handlers"
assert "default" in session["authentication_handlers"], "Did not find 'default' in authentication_headers"
assert "cookie" in session["authentication_handlers"], "Did not find 'cookie' in authentication_headers"
log_info("SESSIONs: {}".format(session))
# Delete session via sg admin port and db rest endpoint
client.delete_session(url=sg_admin_url, db=sg_db, session_id=session_id)
# Make sure session is deleted
try:
session = client.get_session(url=sg_admin_url, db=sg_db, session_id=session_id)
except HTTPError as he:
expected_error_code = he.response.status_code
log_info(expected_error_code)
assert expected_error_code == 404, "Expected 404 status, actual {}".format(expected_error_code)
def test_peer_2_peer_sanity(setup_p2p_test):
"""
1. Sanity P2P Scenario
2. Launch LiteServ 1 and LiteServ 2
3. Create a database on each LiteServ
4. Start continuous push pull replication from each db to the other
5. Add docs to each db
6. Verify the docs show up at each db
7. Verify the docs show up in the database's changes feed.
"""
ls_url_one = setup_p2p_test["ls_url_one"]
ls_url_two = setup_p2p_test["ls_url_two"]
num_docs_per_db = 1000
log_info("ls_url_one: {}".format(ls_url_one))
log_info("ls_url_two: {}".format(ls_url_two))
client = MobileRestClient()
log_info("Creating databases")
ls_db1 = client.create_database(url=ls_url_one, name="ls_db1")
ls_db2 = client.create_database(url=ls_url_two, name="ls_db2")
# Setup continuous push / pull replication from LiteServ 1 ls_db1 to LiteServ 2 ls_db2
repl_one = client.start_replication(
url=ls_url_one,
continuous=True,
from_db=ls_db1,
to_url=ls_url_two, to_db=ls_db2
)
repl_two = client.start_replication(
url=ls_url_one,
continuous=True,
from_url=ls_url_two, from_db=ls_db2,
to_db=ls_db1
)
# Setup continuous push / pull replication from LiteServ 2 ls_db2 to LiteServ 1 ls_db1
repl_three = client.start_replication(
url=ls_url_two,
continuous=True,
from_db=ls_db2,
to_url=ls_url_one, to_db=ls_db1
)
repl_four = client.start_replication(
url=ls_url_two,
continuous=True,
from_url=ls_url_one, from_db=ls_db1,
to_db=ls_db2
)
client.wait_for_replication_status_idle(url=ls_url_one, replication_id=repl_one)
client.wait_for_replication_status_idle(url=ls_url_one, replication_id=repl_two)
client.wait_for_replication_status_idle(url=ls_url_two, replication_id=repl_three)
client.wait_for_replication_status_idle(url=ls_url_two, replication_id=repl_four)
ls_url_one_replications = client.get_replications(ls_url_one)
assert len(ls_url_one_replications) == 2
ls_url_two_replications = client.get_replications(ls_url_two)
assert len(ls_url_two_replications) == 2
ls_db1_docs = client.add_docs(url=ls_url_one, db=ls_db1, number=num_docs_per_db, id_prefix="test_ls_db1")
assert len(ls_db1_docs) == num_docs_per_db
ls_db2_docs = client.add_docs(url=ls_url_two, db=ls_db2, number=num_docs_per_db, id_prefix="test_ls_db2")
assert len(ls_db2_docs) == num_docs_per_db
all_docs = client.merge(ls_db1_docs, ls_db2_docs)
assert len(all_docs) == 2000
client.verify_docs_present(url=ls_url_one, db=ls_db1, expected_docs=all_docs)
client.verify_docs_present(url=ls_url_two, db=ls_db2, expected_docs=all_docs)
client.verify_docs_in_changes(url=ls_url_one, db=ls_db1, expected_docs=all_docs)
client.verify_docs_in_changes(url=ls_url_two, db=ls_db2, expected_docs=all_docs)
def test_stale_revision_should_not_be_in_the_index(setup_client_syncgateway_test):
"""original ticket: https://github.com/couchbase/couchbase-lite-android/issues/855
scenario:
1. Running sync_gateway
2. Create database and starts both push and pull replicators through client REST API
3. Create two or more views through client REST API
4. Add doc, and verify doc is index with current revision through client REST API
5. Make sure document is pushed to sync gateway through sync gateway REST API
6. Update doc with sync gateway (not client side) through sync gateway REST API
7. Make sure updated document is pull replicated to client through client REST API
8. Make sure updated document is indexed through client REST API
9. Make sure stale revision is deleted from index. through client REST API
10. Pass criteria
"""
cluster_config = setup_client_syncgateway_test["cluster_config"]
sg_mode = setup_client_syncgateway_test["sg_mode"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_url = setup_client_syncgateway_test["sg_url"]
sg_admin_url = setup_client_syncgateway_test["sg_admin_url"]
num_docs = 10
num_revs = 100
d_doc_name = "dd"
sg_db = "db"
sg_user_name = "sg_user"
sg_config = sync_gateway_config_path_for_mode("listener_tests/listener_tests", sg_mode)
c = cluster.Cluster(config=cluster_config)
c.reset(sg_config_path=sg_config)
log_info("Running 'test_stale_revision_should_not_be_in_the_index'")
log_info("ls_url: {}".format(ls_url))
log_info("sg_admin_url: {}".format(sg_admin_url))
log_info("sg_url: {}".format(sg_url))
log_info("num_docs: {}".format(num_docs))
log_info("num_revs: {}".format(num_revs))
client = MobileRestClient()
sg_user_channels = ["NBC"]
client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels)
sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name)
view = """{
"language" : "javascript",
"views" : {
"content_view" : {
"map" : "function(doc, meta) { if (doc.content) { emit(doc._id, doc._rev); } }"
},
"update_view" : {
"map" : "function(doc, meta) { emit(doc.updates, null); }"
}
}
}"""
ls_db = client.create_database(url=ls_url, name="ls_db")
# Setup continuous push / pull replication from ls_db1 to sg_db
client.start_replication(
url=ls_url,
continuous=True,
from_db=ls_db,
to_url=sg_admin_url, to_db=sg_db
)
client.start_replication(
url=ls_url,
continuous=True,
from_url=sg_admin_url, from_db=sg_db,
to_db=ls_db
)
design_doc_id = client.add_design_doc(url=ls_url, db=ls_db, name=d_doc_name, doc=view)
client.get_doc(url=ls_url, db=ls_db, doc_id=design_doc_id)
doc_body = document.create_doc(doc_id="doc_1", content={"hi": "I should be in the view"}, channels=sg_user_channels)
log_info(doc_body)
doc_body_2 = document.create_doc(doc_id="doc_2", channels=sg_user_channels)
doc = client.add_doc(url=ls_url, db=ls_db, doc=doc_body)
doc_2 = client.add_doc(url=ls_url, db=ls_db, doc=doc_body_2)
content_view_rows = client.get_view(url=ls_url, db=ls_db, design_doc_name=d_doc_name, view_name="content_view")
client.verify_view_row_num(view_response=content_view_rows, expected_num_rows=1)
update_view_rows = client.get_view(url=ls_url, db=ls_db, design_doc_name=d_doc_name, view_name="update_view")
client.verify_view_row_num(view_response=update_view_rows, expected_num_rows=2)
expected_docs_list = [doc, doc_2]
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=expected_docs_list, auth=sg_session)
updated_doc = client.update_doc(url=sg_url, db=sg_db, doc_id=doc["id"], number_updates=10, auth=sg_session)
client.verify_docs_present(url=ls_url, db=ls_db, expected_docs=updated_doc)
content_view_rows_2 = client.get_view(url=ls_url, db=ls_db, design_doc_name=d_doc_name, view_name="content_view")
client.verify_view_row_num(view_response=content_view_rows_2, expected_num_rows=1)
client.verify_view_contains_keys(view_response=content_view_rows_2, keys=doc["id"])
client.verify_view_contains_values(view_response=content_view_rows_2, values=updated_doc["rev"])
def test_upgrade_cbl(setup_client_syncgateway_test):
"""
1. install previous version of couchbase lite.
2. Create docs in the lite.
3. upgrade to latest version of couchbase lite.
4. Verfiy docs still exits and accessible
"""
sg_db = "db"
ls_db = "ls_db"
channels = ["auto"]
num_docs = 30000
cluster_config = setup_client_syncgateway_test["cluster_config"]
sg_mode = setup_client_syncgateway_test["sg_mode"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_one_admin = setup_client_syncgateway_test["sg_admin_url"]
sg_one_public = setup_client_syncgateway_test["sg_url"]
liteserv = setup_client_syncgateway_test["liteserv"]
device_enabled = setup_client_syncgateway_test["device_enabled"]
liteserv_platform = setup_client_syncgateway_test["liteserv_platform"]
liteserv_version = setup_client_syncgateway_test["liteserv_version"]
if liteserv_platform.lower() == "android" or liteserv_platform.lower() == "net-msft" or device_enabled:
# There is a signature match issue on Android and older version ios does not have app for devices
pytest.skip('upgrade lite serv app does not work on Android and there is no app for ios device created' +
' for older version, so skipping the test')
sg_config = sync_gateway_config_path_for_mode("listener_tests/listener_tests", sg_mode)
c = cluster.Cluster(config=cluster_config)
c.reset(sg_config_path=sg_config)
log_info("ls_url: {}".format(ls_url))
log_info("sg_one_admin: {}".format(sg_one_admin))
log_info("sg_one_public: {}".format(sg_one_public))
test_name = "test_upgrade"
# Downloading 1.4.0-3, we are not paramatrizing here for older version as this is not going to work
# so hardcodig now rather than adding new parameter
log_info("Downloading older version of LiteServ ...")
liteserv.download(version_build="1.4.0-3")
# Install LiteServ
if device_enabled and liteserv_platform == "ios":
liteserv.stop()
liteserv.install_device()
ls_url = liteserv.start_device("{}/logs/{}-{}-{}.txt".format(RESULTS_DIR, type(liteserv).__name__,
test_name, datetime.datetime.now()))
else:
liteserv.stop()
liteserv.install()
log_info("Liteserv starts now ....")
ls_url = liteserv.start("{}/logs/{}-{}-{}.txt".format(RESULTS_DIR, type(liteserv).__name__,
test_name, datetime.datetime.now()))
client = MobileRestClient()
client.create_user(sg_one_admin, sg_db, "test", password="******", channels=channels)
session = client.create_session(sg_one_admin, sg_db, "test")
client.create_database(url=ls_url, name=ls_db)
# Create 'num_docs' docs on LiteServ
docs = client.add_docs(url=ls_url, db=ls_db, number=num_docs, id_prefix="seeded_doc",
generator="four_k", channels=channels)
assert len(docs) == num_docs
client.verify_docs_present(url=ls_url, db=ls_db, expected_docs=docs, timeout=240)
with ThreadPoolExecutor(max_workers=2) as tpe:
download_cbl_tpe = tpe.submit(download_install, liteserv, liteserv_version)
client.start_replication(url=ls_url, continuous=True, from_db=ls_db, to_url=sg_one_admin,
to_db=sg_db)
download_cbl_tpe.result()
log_info("download is done")
liteserv.stop()
ls_url = liteserv.start("{}/logs/{}-{}-{}.txt".format(RESULTS_DIR, type(liteserv).__name__,
test_name, datetime.datetime.now()))
client.start_replication(url=ls_url, continuous=True, from_db=ls_db, to_url=sg_one_admin,
to_db=sg_db)
client.verify_docs_present(url=ls_url, db=ls_db, expected_docs=docs, timeout=240)
client.verify_docs_present(url=sg_one_public, db=sg_db, expected_docs=docs, auth=session, timeout=600)
def test_rebalance_sanity(params_from_base_test_setup):
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_version = get_sg_version(cluster_config)
if compare_versions(sg_version, '1.5') < 0:
pytest.skip("This test needs multiple URLs in the SG config, not supported by SG < 1.5")
cluster_helper = ClusterKeywords()
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf_path)
topology = cluster_helper.get_cluster_topology(cluster_config)
admin_sg_one = topology["sync_gateways"][0]["admin"]
sg_one_url = topology["sync_gateways"][0]["public"]
cluster_servers = topology["couchbase_servers"]
cbs_one_url = cluster_servers[0]
cbs_two_url = cluster_servers[1]
log_info("Running: 'test_distributed_index_rebalance_sanity'")
log_info("cluster_config: {}".format(cluster_config))
log_info("admin_sg: {}".format(admin_sg_one))
log_info("sg_url: {}".format(sg_one_url))
log_info("cbs_one_url: {}".format(cbs_one_url))
log_info("cbs_two_url: {}".format(cbs_two_url))
sg_db = "db"
num_docs = 100
num_updates = 100
sg_user_name = "seth"
sg_user_password = "******"
channels = ["ABC", "CBS"]
client = MobileRestClient()
cb_server = couchbaseserver.CouchbaseServer(cbs_one_url)
server_to_remove = couchbaseserver.CouchbaseServer(cbs_two_url)
client.create_user(admin_sg_one, sg_db, sg_user_name, sg_user_password, channels=channels)
session = client.create_session(admin_sg_one, sg_db, sg_user_name)
with concurrent.futures.ThreadPoolExecutor(5) as executor:
# Add docs to sg
log_info("Adding docs to sync_gateway")
docs = client.add_docs(sg_one_url, sg_db, num_docs, "test_doc", channels=channels, auth=session)
assert len(docs) == num_docs
# Start updating docs and rebalance out one CBS node
log_info("Updating docs on sync_gateway")
update_docs_task = executor.submit(client.update_docs, sg_one_url, sg_db, docs, num_updates, auth=session)
# Run rebalance in background
cb_server.rebalance_out(cluster_servers, server_to_remove)
updated_docs = update_docs_task.result()
log_info(updated_docs)
# Verify docs / revisions present
client.verify_docs_present(sg_one_url, sg_db, updated_docs, auth=session)
# Verify docs revisions in changes feed
client.verify_docs_in_changes(sg_one_url, sg_db, updated_docs, auth=session)
# Rebalance Server back in to the pool
cb_server.add_node(server_to_remove)
cb_server.rebalance_in(cluster_servers, server_to_remove)
def test_server_goes_down_sanity(params_from_base_test_setup):
"""
1. Start with a two node couchbase server cluster
2. Starting adding docs
3. Kill one of the server nodes and signal completion
4. Stop adding docs
5. Verify that that the expected docs are present and in the changes feed.
6. Start server again and add to cluster
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_version = get_sg_version(cluster_config)
if compare_versions(sg_version, '1.5') < 0:
pytest.skip("This test needs multiple URLs in the SG config, not supported by SG < 1.5")
cluster_helper = ClusterKeywords()
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf_path)
topology = cluster_helper.get_cluster_topology(cluster_config)
admin_sg = topology["sync_gateways"][0]["admin"]
sg_url = topology["sync_gateways"][0]["public"]
coucbase_servers = topology["couchbase_servers"]
cbs_one_url = coucbase_servers[0]
cbs_two_url = coucbase_servers[1]
log_info("Running: 'test_server_goes_down_sanity'")
log_info("cluster_config: {}".format(cluster_config))
log_info("admin_sg: {}".format(admin_sg))
log_info("sg_url: {}".format(sg_url))
log_info("cbs_one_url: {}".format(cbs_one_url))
log_info("cbs_two_url: {}".format(cbs_two_url))
sg_db = "db"
num_docs = 100
sg_user_name = "seth"
sg_user_password = "******"
channels = ["ABC", "CBS"]
client = MobileRestClient()
main_server = couchbaseserver.CouchbaseServer(cbs_one_url)
flakey_server = couchbaseserver.CouchbaseServer(cbs_two_url)
client.create_user(admin_sg, sg_db, sg_user_name, sg_user_password, channels=channels)
session = client.create_session(admin_sg, sg_db, sg_user_name)
# Stop second server
flakey_server.stop()
# Try to add 100 docs in a loop until all succeed, if the never do, fail with timeout
errors = num_docs
# Wait 30 seconds for auto failover
# (Minimum value suggested - http://docs.couchbase.com/admin/admin/Tasks/tasks-nodeFailover.html)
# + 15 seconds to add docs
timeout = 45
start = time.time()
successful_add = False
while not successful_add:
# Fail tests if all docs do not succeed before timeout
if (time.time() - start) > timeout:
# Bring server back up before failing the test
flakey_server.start()
main_server.rebalance_in(coucbase_servers, flakey_server)
raise TimeoutError("Failed to successfully put docs before timeout")
try:
docs = client.add_docs(url=sg_url, db=sg_db, number=num_docs, id_prefix=None, auth=session, channels=channels)
# If the above add doc does not throw, it was a successfull add.
successful_add = True
except requests.exceptions.HTTPError as he:
log_info("Failed to add docs: {}".format(he))
log_info("Seeing: {} errors".format(errors))
time.sleep(1)
assert len(docs) == 100
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=docs, auth=session)
try:
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=docs, auth=session, polling_interval=5)
except keywords.exceptions.TimeoutException:
# timeout verifying docs. Bring server back in to restore topology, then fail
# Failing due to https://github.com/couchbase/sync_gateway/issues/2197
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
raise keywords.exceptions.TimeoutException("Failed to get all changes")
# Test succeeded without timeout, bring server back into topology
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
# Make sure all docs were not added before server was
log_info("test_server_goes_down_sanity complete!")
def test_initial_push_replication(setup_client_syncgateway_test, continuous):
"""
1. Prepare LiteServ to have 10000 documents.
2. Create a single shot push / continuous replicator and to push the docs into a sync_gateway database.
3. Verify if all of the docs get pushed.
"""
sg_db = "db"
ls_db = "ls_db"
seth_channels = ["ABC", "NBC"]
num_docs = 10000
cluster_config = setup_client_syncgateway_test["cluster_config"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_one_admin = setup_client_syncgateway_test["sg_admin_url"]
sg_one_public = setup_client_syncgateway_test["sg_url"]
sg_helper = SyncGateway()
sg_helper.start_sync_gateway(
cluster_config=cluster_config,
url=sg_one_public,
config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS)
)
log_info("Running 'test_initial_push_replication', continuous: {}".format(continuous))
log_info("ls_url: {}".format(ls_url))
log_info("sg_one_admin: {}".format(sg_one_admin))
log_info("sg_one_public: {}".format(sg_one_public))
client = MobileRestClient()
client.create_user(sg_one_admin, sg_db, "seth", password="******", channels=seth_channels)
session = client.create_session(sg_one_admin, sg_db, "seth")
client.create_database(url=ls_url, name=ls_db)
# Create 'num_docs' docs on LiteServ
docs = client.add_docs(
url=ls_url,
db=ls_db,
number=num_docs,
id_prefix="seeded_doc",
generator="four_k",
channels=seth_channels
)
assert len(docs) == num_docs
# Start push replication
repl_id = client.start_replication(
url=ls_url,
continuous=continuous,
from_db=ls_db,
to_url=sg_one_admin,
to_db=sg_db
)
if continuous:
log_info("Waiting for replication status 'Idle' for: {}".format(repl_id))
client.wait_for_replication_status_idle(ls_url, repl_id)
else:
log_info("Waiting for no replications: {}".format(repl_id))
client.wait_for_no_replications(ls_url)
# Verify docs replicated to sync_gateway
client.verify_docs_present(url=sg_one_public, db=sg_db, expected_docs=docs, auth=session)
# Verify docs show up in sync_gateway's changes feed
client.verify_docs_in_changes(url=sg_one_public, db=sg_db, expected_docs=docs, auth=session)
replications = client.get_replications(url=ls_url)
if continuous:
assert len(replications) == 1, "There should only be one replication running"
assert replications[0]["status"] == "Idle", "Replication Status should be 'Idle'"
assert replications[0]["continuous"], "Running replication should be continuous"
# Only .NET has an 'error' property
if "error" in replications[0]:
assert len(replications[0]["error"]) == 0
else:
assert len(replications) == 0, "No replications should be running"
def test_take_all_sgaccels_down(params_from_base_test_setup, sg_conf):
"""
Scenario that takes all sync_gateway accel nodes offline during doc load.
After bring the nodes back online during load, the reshard of the DCP feed is verified.
The changes feed is verified that all docs show up.
1. Start doc load (1000 doc)
2. Take all sg_accel nodes down in parallel
3. Verify node are down
4. Wait for doc adds to complete, store "doc_push_result_1"
5. Verify "doc_push_result_1" docs added
6. Start doc load (1000 docs)
7. Wait for 5. to complete, store "doc_push_result_2"
8. Verify "doc_push_result_2" docs added
9. Start another doc load (1000 docs)
10. Bring up nodes in parallel
11. poll on p-index reshard
12. Wait for 9. to complete, store "doc_push_result_3"
13. Verify "doc_push_result_3" docs added
14. Verify "doc_push_result_1" + "doc_push_result_2" + "doc_push_result_3" show up in _changes feed
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_dcp_reshard_single_sg_accel_goes_down_and_up'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_util = ClusterKeywords()
topology = cluster_util.get_cluster_topology(cluster_conf)
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
num_docs = 1000
client = MobileRestClient()
doc_pusher_user_info = userinfo.UserInfo("doc_pusher",
"pass",
channels=["A"],
roles=[])
doc_pusher_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=doc_pusher_user_info.name,
password=doc_pusher_user_info.password,
channels=doc_pusher_user_info.channels)
a_user_info = userinfo.UserInfo("a_user", "pass", channels=["A"], roles=[])
client.create_user(url=sg_admin_url,
db=sg_db,
name=a_user_info.name,
password=a_user_info.password,
channels=a_user_info.channels)
a_user_session = client.create_session(url=sg_admin_url,
db=sg_db,
name=a_user_info.name,
password=a_user_info.password)
# Shutdown all accel nodes in parallel
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as ex:
# Start adding docs
docs_1 = document.create_docs(None,
num_docs,
channels=doc_pusher_user_info.channels)
docs_1_task = ex.submit(client.add_bulk_docs,
url=sg_url,
db=sg_db,
docs=docs_1,
auth=doc_pusher_auth)
# Take down all access nodes
log_info("Shutting down sg_accels: [{}, {}, {}] ...".format(
cluster.sg_accels[0], cluster.sg_accels[1], cluster.sg_accels[2]))
sg_accel_down_task_1 = ex.submit(cluster.sg_accels[0].stop)
sg_accel_down_task_2 = ex.submit(cluster.sg_accels[1].stop)
sg_accel_down_task_3 = ex.submit(cluster.sg_accels[2].stop)
assert sg_accel_down_task_1.result() == 0
assert sg_accel_down_task_2.result() == 0
assert sg_accel_down_task_3.result() == 0
# Block until bulk_docs is complete
doc_push_result_1 = docs_1_task.result()
assert len(doc_push_result_1) == num_docs
client.verify_docs_present(url=sg_url,
db=sg_db,
expected_docs=doc_push_result_1,
auth=doc_pusher_auth)
# Load sync_gateway with another batch of docs while the sg_accel nodes are offline
docs_2_bodies = document.create_docs(
None, num_docs, channels=doc_pusher_user_info.channels)
docs_push_result_2 = client.add_bulk_docs(url=sg_url,
db=sg_db,
docs=docs_2_bodies,
auth=doc_pusher_auth)
assert len(docs_push_result_2) == num_docs
client.verify_docs_present(url=sg_url,
db=sg_db,
expected_docs=docs_push_result_2,
auth=doc_pusher_auth)
# Start loading Sync Gateway with another set of docs while bringing the sg_accel nodes online
docs_3 = document.create_docs(None,
num_docs,
channels=doc_pusher_user_info.channels)
docs_3_task = ex.submit(client.add_bulk_docs,
url=sg_url,
db=sg_db,
docs=docs_3,
auth=doc_pusher_auth)
# Bring all the sg_accel nodes back up
# Take down all access nodes
log_info("Starting sg_accels: [{}, {}, {}] ...".format(
cluster.sg_accels[0], cluster.sg_accels[1], cluster.sg_accels[2]))
sg_accel_up_task_1 = ex.submit(cluster.sg_accels[0].start, sg_conf)
sg_accel_up_task_2 = ex.submit(cluster.sg_accels[1].start, sg_conf)
sg_accel_up_task_3 = ex.submit(cluster.sg_accels[2].start, sg_conf)
assert sg_accel_up_task_1.result() == 0
assert sg_accel_up_task_2.result() == 0
assert sg_accel_up_task_3.result() == 0
# Wait for pindex to reshard correctly
assert cluster.validate_cbgt_pindex_distribution_retry(3)
# Block until second bulk_docs is complete
doc_push_result_3 = docs_3_task.result()
assert len(doc_push_result_3) == num_docs
client.verify_docs_present(url=sg_url,
db=sg_db,
expected_docs=doc_push_result_3,
auth=doc_pusher_auth)
# Combine the 3 push results and make sure the changes propagate to a_user
# a_user has access to the doc's channel.
log_info("Verifying all the changes show up for 'a_user' ...")
all_docs = doc_push_result_1 + docs_push_result_2 + doc_push_result_3
client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=all_docs,
auth=a_user_session,
polling_interval=2)
def test_initial_pull_replication(setup_client_syncgateway_test, continuous):
"""
1. Prepare sync-gateway to have 10000 documents.
2. Create a single shot / continuous pull replicator and to pull the docs into a database.
3. Verify if all of the docs get pulled.
Referenced issue: couchbase/couchbase-lite-android#955.
"""
sg_db = "db"
ls_db = "ls_db"
num_docs = 10000
cluster_config = setup_client_syncgateway_test["cluster_config"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_one_admin = setup_client_syncgateway_test["sg_admin_url"]
sg_one_public = setup_client_syncgateway_test["sg_url"]
sg_helper = SyncGateway()
sg_helper.start_sync_gateway(
cluster_config=cluster_config,
url=sg_one_public,
config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS)
)
log_info("Running 'test_initial_pull_replication', continuous: {}".format(continuous))
log_info("ls_url: {}".format(ls_url))
log_info("sg_one_admin: {}".format(sg_one_admin))
log_info("sg_one_public: {}".format(sg_one_public))
client = MobileRestClient()
client.create_user(sg_one_admin, sg_db, "seth", password="******", channels=["ABC", "NBC"])
session = client.create_session(sg_one_admin, sg_db, "seth")
# Create 'num_docs' docs on sync_gateway
docs = client.add_docs(
url=sg_one_public,
db=sg_db,
number=num_docs,
id_prefix="seeded_doc",
generator="four_k",
auth=session
)
assert len(docs) == num_docs
client.create_database(url=ls_url, name=ls_db)
# Start oneshot pull replication
repl_id = client.start_replication(
url=ls_url,
continuous=continuous,
from_url=sg_one_admin,
from_db=sg_db,
to_db=ls_db
)
start = time.time()
if continuous:
log_info("Waiting for replication status 'Idle' for: {}".format(repl_id))
# Android will report IDLE status, and drop into the 'verify_docs_present' below
# due to https://github.com/couchbase/couchbase-lite-java-core/issues/1409
client.wait_for_replication_status_idle(ls_url, repl_id)
else:
log_info("Waiting for no replications: {}".format(repl_id))
client.wait_for_no_replications(ls_url)
# Verify docs replicated to client
client.verify_docs_present(url=ls_url, db=ls_db, expected_docs=docs, timeout=240)
all_docs_replicated_time = time.time() - start
log_info("Replication took: {}s".format(all_docs_replicated_time))
# Verify docs show up in client's changes feed
client.verify_docs_in_changes(url=ls_url, db=ls_db, expected_docs=docs)
replications = client.get_replications(url=ls_url)
if continuous:
assert len(replications) == 1, "There should only be one replication running"
assert replications[0]["status"] == "Idle", "Replication Status should be 'Idle'"
assert replications[0]["continuous"], "Running replication should be continuous"
# Only .NET has an 'error' property
if "error" in replications[0]:
assert len(replications[0]["error"]) == 0
else:
assert len(replications) == 0, "No replications should be running"
def test_auto_prune_with_pull(setup_client_syncgateway_test):
"""Sanity test for autopruning with replication
1. Create a database on LiteServ (ls_db)
2. Add doc to sync gateway
3. Update doc 50 times on sync_gateway
4. Set up pull replication from sync_gateway db to LiteServ db
5. Verify number of revisions on client is default (20)
"""
cluster_config = setup_client_syncgateway_test["cluster_config"]
sg_mode = setup_client_syncgateway_test["sg_mode"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_url = setup_client_syncgateway_test["sg_url"]
sg_admin_url = setup_client_syncgateway_test["sg_admin_url"]
client = MobileRestClient()
sg_config = sync_gateway_config_path_for_mode(
"listener_tests/listener_tests", sg_mode)
c = cluster.Cluster(config=cluster_config)
c.reset(sg_config_path=sg_config)
log_info("Running 'test_auto_prune_listener_sanity' ...")
log_info("ls_url: {}".format(ls_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_admin_url: {}".format(sg_admin_url))
num_docs = 1
num_revs = 50
sg_user_channels = ["NBC"]
sg_db = "db"
sg_user_name = "sg_user"
client.create_user(url=sg_admin_url,
db=sg_db,
name=sg_user_name,
password="******",
channels=sg_user_channels)
sg_session = client.create_session(url=sg_admin_url,
db=sg_db,
name=sg_user_name)
ls_db = client.create_database(url=ls_url, name="ls_db")
sg_db_docs = client.add_docs(url=sg_url,
db=sg_db,
number=num_docs,
id_prefix=sg_db,
channels=sg_user_channels,
auth=sg_session)
assert len(sg_db_docs) == num_docs
sg_docs_update = client.update_docs(url=sg_url,
db=sg_db,
docs=sg_db_docs,
number_updates=num_revs,
auth=sg_session)
# Start continuous replication ls_db <- sg_db
repl_one = client.start_replication(url=ls_url,
continuous=True,
from_url=sg_admin_url,
from_db=sg_db,
to_db=ls_db)
client.wait_for_replication_status_idle(url=ls_url,
replication_id=repl_one)
client.verify_docs_present(url=ls_url,
db=ls_db,
expected_docs=sg_docs_update)
client.verify_revs_num_for_docs(url=ls_url,
db=ls_db,
docs=sg_docs_update,
expected_revs_per_doc=20)
def test_client_to_sync_gateway_complex_replication_with_revs_limit(setup_client_syncgateway_test):
""" Ported from sync_gateway tests repo
... 1. Clear server buckets
... 2. Restart liteserv with _session
... 3. Restart sync_gateway wil that config
... 4. Create db on LiteServ
... 5. Add numDocs to LiteServ db
... 6. Setup push replication from LiteServ db to sync_gateway
... 7. Verify doc present on sync_gateway (number of docs)
... 8. Update sg docs numRevs * 4 = 480
... 9. Update docs on LiteServ db numRevs * 4 = 480
... 10. Setup pull replication from sg -> liteserv db
... 11. Verify all docs are replicated
... 12. compact LiteServ db (POST _compact)
... 13. Verify number of revs in LiteServ db (?revs_info=true) check rev status == available fail if revs available > revs limit
... 14. Delete LiteServ db conflicts (?conflicts=true) DELETE _conflicts
... 15. Create numDoc number of docs in LiteServ db
... 16. Update LiteServ db docs numRevs * 5 (600)
... 17. Verify LiteServ db revs is < 602
... 18. Verify LiteServ db docs revs prefix (9 * numRevs + 3)
... 19. Compact LiteServ db
... 20. Verify number of revs <= 10
... 21. Delete LiteServ docs
... 22. Delete Server bucket
... 23. Delete LiteServ db
"""
ls_db_name = "ls_db"
sg_db = "db"
sg_user_name = "sg_user"
num_docs = 10
num_revs = 100
cluster_config = setup_client_syncgateway_test["cluster_config"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_url = setup_client_syncgateway_test["sg_url"]
sg_admin_url = setup_client_syncgateway_test["sg_admin_url"]
sg_helper = SyncGateway()
sg_helper.start_sync_gateway(
cluster_config=cluster_config,
url=sg_url,
config="{}/walrus-revs-limit.json".format(SYNC_GATEWAY_CONFIGS)
)
log_info("Running 'test_client_to_sync_gateway_complex_replication_with_revs_limit'")
log_info("ls_url: {}".format(ls_url))
log_info("sg_admin_url: {}".format(sg_admin_url))
log_info("sg_url: {}".format(sg_url))
client = MobileRestClient()
# Test the endpoint, listener does not support users but should have a default response
client.get_session(url=ls_url)
sg_user_channels = ["NBC"]
client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels)
sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name)
ls_db = client.create_database(url=ls_url, name=ls_db_name)
ls_db_docs = client.add_docs(url=ls_url, db=ls_db, number=num_docs, id_prefix=ls_db, channels=sg_user_channels)
assert len(ls_db_docs) == num_docs
# Start replication ls_db -> sg_db
repl_one = client.start_replication(
url=ls_url,
continuous=True,
from_db=ls_db,
to_url=sg_admin_url, to_db=sg_db
)
client.verify_docs_present(url=sg_admin_url, db=sg_db, expected_docs=ls_db_docs)
# Delay is to the updates here due to couchbase/couchbase-lite-ios#1277.
# Basically, if your revs depth is small and someone is updating a doc past the revs depth before a push replication,
# the push replication will have no common ancestor with sync_gateway causing conflicts to be created.
# Adding a delay between updates helps this situation. There is an alternative for CBL mac and CBL NET to change the default revs client depth
# but that is not configurable for Android.
# Currently adding a delay will allow the replication to act as expected for all platforms now.
client.update_docs(url=sg_url, db=sg_db, docs=ls_db_docs, number_updates=num_revs, delay=0.1, auth=sg_session)
client.update_docs(url=ls_url, db=ls_db, docs=ls_db_docs, number_updates=num_revs, delay=0.1)
# Start replication ls_db <- sg_db
repl_two = client.start_replication(
url=ls_url,
continuous=True,
from_url=sg_admin_url, from_db=sg_db,
to_db=ls_db
)
client.wait_for_replication_status_idle(url=ls_url, replication_id=repl_one)
client.wait_for_replication_status_idle(url=ls_url, replication_id=repl_two)
client.compact_database(url=ls_url, db=ls_db)
# LiteServ should only have 20 revisions due to built in client revs limit
client.verify_revs_num_for_docs(url=ls_url, db=ls_db, docs=ls_db_docs, expected_revs_per_doc=20)
# Sync Gateway should have 100 revisions due to the specified revs_limit in the sg config and possible conflict winners from the liteserv db
client.verify_max_revs_num_for_docs(url=sg_url, db=sg_db, docs=ls_db_docs, expected_max_number_revs_per_doc=100, auth=sg_session)
client.delete_conflicts(url=ls_url, db=ls_db, docs=ls_db_docs)
expected_generation = num_revs + 1
client.verify_docs_rev_generations(url=ls_url, db=ls_db, docs=ls_db_docs, expected_generation=expected_generation)
client.verify_docs_rev_generations(url=sg_url, db=sg_db, docs=ls_db_docs, expected_generation=expected_generation, auth=sg_session)
client.delete_docs(url=ls_url, db=ls_db, docs=ls_db_docs)
client.verify_docs_deleted(url=ls_url, db=ls_db, docs=ls_db_docs)
client.verify_docs_deleted(url=sg_admin_url, db=sg_db, docs=ls_db_docs)
ls_db_docs = client.add_docs(url=ls_url, db=ls_db, number=num_docs, id_prefix=ls_db, channels=sg_user_channels)
assert len(ls_db_docs) == 10
expected_revs = num_revs + 20 + 2
client.update_docs(url=ls_url, db=ls_db, docs=ls_db_docs, delay=0.1, number_updates=num_revs)
client.verify_max_revs_num_for_docs(url=ls_url, db=ls_db, docs=ls_db_docs, expected_max_number_revs_per_doc=expected_revs)
expected_generation = (num_revs * 2) + 3
client.verify_docs_rev_generations(url=ls_url, db=ls_db, docs=ls_db_docs, expected_generation=expected_generation)
client.compact_database(url=ls_url, db=ls_db)
client.verify_revs_num_for_docs(url=ls_url, db=ls_db, docs=ls_db_docs, expected_revs_per_doc=20)
client.stop_replication(
url=ls_url,
continuous=True,
from_db=ls_db,
to_url=sg_admin_url, to_db=sg_db
)
client.stop_replication(
url=ls_url,
continuous=True,
from_url=sg_admin_url, from_db=sg_db,
to_db=ls_db
)
client.wait_for_no_replications(url=ls_url)
client.delete_conflicts(url=ls_url, db=ls_db, docs=ls_db_docs)
client.delete_conflicts(url=sg_url, db=sg_db, docs=ls_db_docs, auth=sg_session)
client.delete_docs(url=ls_url, db=ls_db, docs=ls_db_docs)
# Start push pull and verify that all docs are deleted
# Start replication ls_db -> sg_db
repl_one = client.start_replication(
url=ls_url,
continuous=True,
from_db=ls_db,
to_url=sg_admin_url, to_db=sg_db
)
# Start replication ls_db <- sg_db
repl_two = client.start_replication(
url=ls_url,
continuous=True,
from_url=sg_admin_url, from_db=sg_db,
to_db=ls_db
)
client.verify_docs_deleted(url=ls_url, db=ls_db, docs=ls_db_docs)
client.verify_docs_deleted(url=sg_admin_url, db=sg_db, docs=ls_db_docs)
def test_inline_large_attachments(setup_client_syncgateway_test):
"""
1. Start LiteServ and Sync Gateway
2. Create 2 databases on LiteServ (ls_db1, ls_db2)
3. Start continuous push replication from ls_db1 to sg_db
4. Start continuous pull replication from sg_db to ls_db2
5. PUT 5 large inline attachments to ls_db1
6. DELETE the docs on ls_db1
7. PUT same 5 large inline attachments to ls_db1
8. Verify docs replicate to ls_db2
9. Purge ls_db1
10. Verify docs removed
"""
log_info("Running 'test_inline_large_attachments' ...")
sg_url = setup_client_syncgateway_test["sg_url"]
sg_url_admin = setup_client_syncgateway_test["sg_admin_url"]
ls_url = setup_client_syncgateway_test["ls_url"]
log_info("ls_url: {}".format(ls_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
ls_db1 = "ls_db1"
ls_db2 = "ls_db2"
sg_db = "db"
client = MobileRestClient()
client.create_database(ls_url, ls_db1)
client.create_database(ls_url, ls_db2)
# Start continuous push replication from ls_db1 -> sg_db
client.start_replication(url=ls_url, continuous=True, from_db=ls_db1, to_url=sg_url, to_db=sg_db)
# Start continuous push replication from sg_db -> ls_db2
client.start_replication(url=ls_url, continuous=True, from_url=sg_url, from_db=sg_db, to_db=ls_db2)
# doc with 2.36 PNG attachment
attachment_docs = []
for i in range(5):
doc = document.create_doc(
doc_id="large_attach_{}".format(i), attachment_name="golden_gate_large.jpg", channels=["ABC"]
)
attachment_docs.append(doc)
# add large attachments to ls_db1
docs = []
for doc in attachment_docs:
docs.append(client.add_doc(ls_url, ls_db1, doc, use_post=False))
# Delete docs
client.delete_docs(ls_url, ls_db1, docs)
client.verify_docs_deleted(ls_url, ls_db1, docs)
# Recreated docs
recreated_docs = []
for doc in attachment_docs:
recreated_docs.append(client.add_doc(ls_url, ls_db1, doc, use_post=False))
client.verify_docs_present(ls_url, ls_db1, recreated_docs)
client.verify_docs_present(sg_url, sg_db, recreated_docs)
client.verify_docs_present(ls_url, ls_db2, recreated_docs)
purged_docs = client.purge_docs(ls_url, ls_db1, recreated_docs)
log_info(purged_docs)
# All purged docs should have replicated and should be gone now.
# This is currently failing due to some docs not replicating to ls_db2
client.verify_docs_deleted(ls_url, ls_db1, recreated_docs)
client.verify_docs_deleted(sg_url, sg_db, recreated_docs)
client.verify_docs_deleted(ls_url, ls_db2, recreated_docs)
def test_verify_open_revs_with_revs_limit_push_conflict(setup_client_syncgateway_test):
"""Test replication from multiple client dbs to one sync_gateway db
https://github.com/couchbase/couchbase-lite-ios/issues/1277
"""
cluster_config = setup_client_syncgateway_test["cluster_config"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_url = setup_client_syncgateway_test["sg_url"]
sg_admin_url = setup_client_syncgateway_test["sg_admin_url"]
num_docs = 100
num_revs = 20
sg_db = "db"
sg_user_name = "sg_user"
sg_helper = SyncGateway()
sg_helper.start_sync_gateway(
cluster_config=cluster_config,
url=sg_url,
config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS)
)
log_info("Running 'test_verify_open_revs_with_revs_limit_push_conflict'")
log_info("ls_url: {}".format(ls_url))
log_info("sg_admin_url: {}".format(sg_admin_url))
log_info("sg_url: {}".format(sg_url))
log_info("num_docs: {}".format(num_docs))
log_info("num_revs: {}".format(num_revs))
client = MobileRestClient()
# Test the endpoint, listener does not support users but should have a default response
client.get_session(url=ls_url)
sg_user_channels = ["NBC"]
client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels)
sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name)
ls_db = client.create_database(url=ls_url, name="ls_db")
ls_db_docs = client.add_docs(url=ls_url, db=ls_db, number=num_docs, id_prefix="ls_db", channels=sg_user_channels)
assert len(ls_db_docs) == num_docs
# Start replication ls_db -> sg_db
repl_one = client.start_replication(
url=ls_url,
continuous=True,
from_db=ls_db,
to_url=sg_admin_url, to_db=sg_db
)
client.verify_docs_present(url=sg_admin_url, db=sg_db, expected_docs=ls_db_docs)
client.update_docs(url=sg_url, db=sg_db, docs=ls_db_docs, number_updates=num_revs, auth=sg_session)
sg_current_doc = client.get_doc(url=sg_url, db=sg_db, doc_id="ls_db_2", auth=sg_session)
client.update_docs(url=ls_url, db=ls_db, docs=ls_db_docs, number_updates=num_revs)
ls_current_doc = client.get_doc(url=ls_url, db=ls_db, doc_id="ls_db_2")
client.wait_for_replication_status_idle(url=ls_url, replication_id=repl_one)
client.verify_doc_rev_generation(url=ls_url, db=ls_db, doc_id=ls_current_doc["_id"], expected_generation=21)
client.verify_doc_rev_generation(url=sg_url, db=sg_db, doc_id=sg_current_doc["_id"], expected_generation=21, auth=sg_session)
expected_ls_revs = [ls_current_doc["_rev"]]
client.verify_open_revs(url=ls_url, db=ls_db, doc_id=ls_current_doc["_id"], expected_open_revs=expected_ls_revs)
expected_sg_revs = [ls_current_doc["_rev"], sg_current_doc["_rev"]]
client.verify_open_revs(url=sg_admin_url, db=sg_db, doc_id=sg_current_doc["_id"], expected_open_revs=expected_sg_revs)
def test_rebalance_sanity(params_from_base_test_setup):
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
cluster_helper = ClusterKeywords()
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf_path)
topology = cluster_helper.get_cluster_topology(cluster_config)
admin_sg_one = topology["sync_gateways"][0]["admin"]
sg_one_url = topology["sync_gateways"][0]["public"]
cluster_servers = topology["couchbase_servers"]
cbs_one_url = cluster_servers[0]
cbs_two_url = cluster_servers[1]
log_info("Running: 'test_distributed_index_rebalance_sanity'")
log_info("cluster_config: {}".format(cluster_config))
log_info("admin_sg: {}".format(admin_sg_one))
log_info("sg_url: {}".format(sg_one_url))
log_info("cbs_one_url: {}".format(cbs_one_url))
log_info("cbs_two_url: {}".format(cbs_two_url))
sg_db = "db"
num_docs = 100
num_updates = 100
sg_user_name = "seth"
sg_user_password = "******"
channels = ["ABC", "CBS"]
client = MobileRestClient()
cb_server = CouchbaseServer(cbs_one_url)
server_to_remove = CouchbaseServer(cbs_two_url)
client.create_user(admin_sg_one, sg_db, sg_user_name, sg_user_password, channels=channels)
session = client.create_session(admin_sg_one, sg_db, sg_user_name)
with concurrent.futures.ThreadPoolExecutor(5) as executor:
# Add docs to sg
log_info("Adding docs to sync_gateway")
docs = client.add_docs(sg_one_url, sg_db, num_docs, "test_doc", channels=channels, auth=session)
assert len(docs) == num_docs
# Start updating docs and rebalance out one CBS node
log_info("Updating docs on sync_gateway")
update_docs_task = executor.submit(client.update_docs, sg_one_url, sg_db, docs, num_updates, auth=session)
# Run rebalance in background
cb_server.rebalance_out(cluster_servers, server_to_remove)
updated_docs = update_docs_task.result()
log_info(updated_docs)
# Verify docs / revisions present
client.verify_docs_present(sg_one_url, sg_db, updated_docs, auth=session)
# Verify docs revisions in changes feed
client.verify_docs_in_changes(sg_one_url, sg_db, updated_docs, auth=session)
# Rebalance Server back in to the pool
cb_server.add_node(server_to_remove)
cb_server.rebalance_in(cluster_servers, server_to_remove)
def test_listener_two_sync_gateways(setup_client_syncgateway_test):
"""
Port of https://github.com/couchbaselabs/sync-gateway-tests/blob/master/tests/cbl-replication-mismatch-2-gateways.js
Scenario:
1. Start 2 sync_gateways
2. Create sg_db_one db on sync_gateway one
3. Create sg_db_two db on sync_gateway two
4. Create ls_db_one and ls_db_two on Liteserv
5. Setup continuous push / pull replication from ls_db_one <-> sg_db_one
6. Setup continuous push / pull replication from ls_db_two <-> sg_db_two
7. Setup continuous push / pull replication from sg_db_one <-> ls_db_two
8. Setup continuous push / pull replication from sg_db_two <-> ls_db_one
9. Add num_docs / 2 to each liteserv database
10. Verify each database has num_docs docs
11. Verify all_docs in all dbs
12. Verify changes feed for sg_db_one and sg_db_two
13. Verify chnages feed for ls_db_one and ls_db_two
"""
num_docs = 500
ls_url = setup_client_syncgateway_test["ls_url"]
cluster_config = setup_client_syncgateway_test["cluster_config"]
sg_mode = setup_client_syncgateway_test["sg_mode"]
cluster_util = ClusterKeywords()
topology = cluster_util.get_cluster_topology(cluster_config)
sg_one_admin_url = topology["sync_gateways"][0]["admin"]
sg_two_admin_url = topology["sync_gateways"][1]["admin"]
cb_server_url = topology["couchbase_servers"][0]
log_info("Sync Gateway 1 admin url: {}".format(sg_one_admin_url))
log_info("Sync Gateway 2 admin url: {}".format(sg_two_admin_url))
log_info("Couchbase Server url: {}".format(cb_server_url))
c = cluster.Cluster(cluster_config)
sg_config_path = sync_gateway_config_path_for_mode(
"listener_tests/multiple_sync_gateways", sg_mode)
c.reset(sg_config_path=sg_config_path)
ls_db_one = "ls_db1"
ls_db_two = "ls_db2"
sg_db_one = "sg_db1"
sg_db_two = "sg_db2"
log_info("ls_url: {}".format(ls_url))
log_info("sg_one_admin_url: {}".format(sg_one_admin_url))
log_info("sg_two_admin_url: {}".format(sg_two_admin_url))
log_info("num_docs: {}".format(num_docs))
log_info("Running 'test_listener_two_sync_gateways' ...")
client = MobileRestClient()
# Delete sg_db2 on sync_gateway 1
client.delete_database(url=sg_one_admin_url, name=sg_db_two)
# Delete sg_db1 on sync_gateway 2
client.delete_database(url=sg_two_admin_url, name=sg_db_one)
# Create dbs on LiteServ
client.create_database(ls_url, ls_db_one)
client.create_database(ls_url, ls_db_two)
# Start continuous push pull replication ls_db_one <-> sg_db_one
client.start_replication(url=ls_url,
continuous=True,
from_db=ls_db_one,
to_url=sg_one_admin_url,
to_db=sg_db_one)
client.start_replication(url=ls_url,
continuous=True,
from_url=sg_one_admin_url,
from_db=sg_db_one,
to_db=ls_db_one)
# Start continuous push pull replication ls_db_two <-> sg_db_two
client.start_replication(url=ls_url,
continuous=True,
from_db=ls_db_two,
to_url=sg_two_admin_url,
to_db=sg_db_two)
client.start_replication(url=ls_url,
continuous=True,
from_url=sg_two_admin_url,
from_db=sg_db_two,
to_db=ls_db_two)
# Start continuous push pull replication sg_db_one <-> ls_db_two
client.start_replication(url=ls_url,
continuous=True,
from_url=sg_one_admin_url,
from_db=sg_db_one,
to_db=ls_db_two)
client.start_replication(url=ls_url,
continuous=True,
from_db=ls_db_two,
to_url=sg_one_admin_url,
to_db=sg_db_one)
# Start continuous push pull replication sg_db_two <-> ls_db_one
client.start_replication(url=ls_url,
continuous=True,
from_url=sg_two_admin_url,
from_db=sg_db_two,
to_db=ls_db_one)
client.start_replication(url=ls_url,
continuous=True,
from_db=ls_db_one,
to_url=sg_two_admin_url,
to_db=sg_db_two)
ls_db_one_docs = client.add_docs(url=ls_url,
db=ls_db_one,
number=num_docs / 2,
id_prefix="ls_db_one_doc")
assert len(ls_db_one_docs) == num_docs / 2
ls_db_two_docs = client.add_docs(url=ls_url,
db=ls_db_two,
number=num_docs / 2,
id_prefix="ls_db_two_doc")
assert len(ls_db_two_docs) == num_docs / 2
all_docs = client.merge(ls_db_one_docs, ls_db_two_docs)
assert len(all_docs) == 500
# Verify docs replicate to each db
client.verify_docs_present(url=ls_url,
db=ls_db_one,
expected_docs=all_docs)
client.verify_docs_present(url=ls_url,
db=ls_db_two,
expected_docs=all_docs)
client.verify_docs_present(url=sg_one_admin_url,
db=sg_db_one,
expected_docs=all_docs)
client.verify_docs_present(url=sg_two_admin_url,
db=sg_db_two,
expected_docs=all_docs)
# Verify changes feeds for each db
client.verify_docs_in_changes(url=ls_url,
db=ls_db_one,
expected_docs=all_docs)
client.verify_docs_in_changes(url=ls_url,
db=ls_db_two,
expected_docs=all_docs)
client.verify_docs_in_changes(url=sg_one_admin_url,
db=sg_db_one,
expected_docs=all_docs)
client.verify_docs_in_changes(url=sg_two_admin_url,
db=sg_db_two,
expected_docs=all_docs)
def test_listener_two_sync_gateways(setup_client_2sgs_test):
"""
Port of https://github.com/couchbaselabs/sync-gateway-tests/blob/master/tests/cbl-replication-mismatch-2-gateways.js
Scenario:
1. Start 2 sync_gateways
2. Create sg_db_one db on sync_gateway one
3. Create sg_db_two db on sync_gateway two
4. Create ls_db_one and ls_db_two on Liteserv
5. Setup continuous push / pull replication from ls_db_one <-> sg_db_one
6. Setup continuous push / pull replication from ls_db_two <-> sg_db_two
7. Setup continuous push / pull replication from sg_db_one <-> ls_db_two
8. Setup continuous push / pull replication from sg_db_two <-> ls_db_one
9. Add num_docs / 2 to each liteserv database
10. Verify each database has num_docs docs
11. Verify all_docs in all dbs
12. Verify changes feed for sg_db_one and sg_db_two
13. Verify chnages feed for ls_db_one and ls_db_two
"""
num_docs = 500
ls_url = setup_client_2sgs_test["ls_url"]
cluster_config = setup_client_2sgs_test["cluster_config"]
sg_one_admin_url = setup_client_2sgs_test["sg_one_admin_url"]
sg_two_admin_url = setup_client_2sgs_test["sg_two_admin_url"]
sg_util = SyncGateway()
sg_util.start_sync_gateway(cluster_config=cluster_config, url=sg_one_admin_url, config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS))
sg_util.start_sync_gateway(cluster_config=cluster_config, url=sg_two_admin_url, config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS))
ls_db_one = "ls_db1"
ls_db_two = "ls_db2"
sg_db_one = "sg_db1"
sg_db_two = "sg_db2"
log_info("ls_url: {}".format(ls_url))
log_info("sg_one_admin_url: {}".format(sg_one_admin_url))
log_info("sg_two_admin_url: {}".format(sg_two_admin_url))
log_info("num_docs: {}".format(num_docs))
log_info("Running 'test_listener_two_sync_gateways' ...")
client = MobileRestClient()
# Create dbs on sync_gateway
client.create_database(sg_one_admin_url, sg_db_one, "walrus:")
client.create_database(sg_two_admin_url, sg_db_two, "walrus:")
# Create dbs on LiteServ
client.create_database(ls_url, ls_db_one)
client.create_database(ls_url, ls_db_two)
# Start continuous push pull replication ls_db_one <-> sg_db_one
client.start_replication(
url=ls_url, continuous=True,
from_db=ls_db_one,
to_url=sg_one_admin_url, to_db=sg_db_one
)
client.start_replication(
url=ls_url, continuous=True,
from_url=sg_one_admin_url, from_db=sg_db_one,
to_db=ls_db_one
)
# Start continuous push pull replication ls_db_two <-> sg_db_two
client.start_replication(
url=ls_url, continuous=True,
from_db=ls_db_two,
to_url=sg_two_admin_url, to_db=sg_db_two
)
client.start_replication(
url=ls_url, continuous=True,
from_url=sg_two_admin_url, from_db=sg_db_two,
to_db=ls_db_two
)
# Start continuous push pull replication sg_db_one <-> ls_db_two
client.start_replication(
url=ls_url, continuous=True,
from_url=sg_one_admin_url, from_db=sg_db_one,
to_db=ls_db_two
)
client.start_replication(
url=ls_url, continuous=True,
from_db=ls_db_two,
to_url=sg_one_admin_url, to_db=sg_db_one
)
# Start continuous push pull replication sg_db_two <-> ls_db_one
client.start_replication(
url=ls_url, continuous=True,
from_url=sg_two_admin_url, from_db=sg_db_two,
to_db=ls_db_one
)
client.start_replication(
url=ls_url, continuous=True,
from_db=ls_db_one,
to_url=sg_two_admin_url, to_db=sg_db_two
)
ls_db_one_docs = client.add_docs(url=ls_url, db=ls_db_one, number=num_docs / 2, id_prefix="ls_db_one_doc")
assert len(ls_db_one_docs) == num_docs / 2
ls_db_two_docs = client.add_docs(url=ls_url, db=ls_db_two, number=num_docs / 2, id_prefix="ls_db_two_doc")
assert len(ls_db_two_docs) == num_docs / 2
all_docs = client.merge(ls_db_one_docs, ls_db_two_docs)
assert len(all_docs) == 500
# Verify docs replicate to each db
client.verify_docs_present(url=ls_url, db=ls_db_one, expected_docs=all_docs)
client.verify_docs_present(url=ls_url, db=ls_db_two, expected_docs=all_docs)
client.verify_docs_present(url=sg_one_admin_url, db=sg_db_one, expected_docs=all_docs)
client.verify_docs_present(url=sg_two_admin_url, db=sg_db_two, expected_docs=all_docs)
# Verify changes feeds for each db
client.verify_docs_in_changes(url=ls_url, db=ls_db_one, expected_docs=all_docs)
client.verify_docs_in_changes(url=ls_url, db=ls_db_two, expected_docs=all_docs)
client.verify_docs_in_changes(url=sg_one_admin_url, db=sg_db_one, expected_docs=all_docs)
client.verify_docs_in_changes(url=sg_two_admin_url, db=sg_db_two, expected_docs=all_docs)
