def test_rolling_ttl_remove_expirary(params_from_base_test_setup, sg_conf_name):
"""
1. PUT /db/doc1 via SG with property "_exp":3
2. Once per second for 10 seconds, update /db/doc1 with a new revision (also with "_exp":3)
3. Update /db/doc1 with a revision with no expiry
3. Get /db/doc1. Assert response is 200
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_rolling_ttl_remove_expirary'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=3, channels=sg_user_channels)
doc_exp_10_body = document.create_doc(doc_id="exp_10", expiry=10, channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session)
doc_exp_10 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_10_body, auth=sg_user_session)
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=10, expiry=3, delay=1, auth=sg_user_session)
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=1, auth=sg_user_session)
# If expiry was not removed in the last update, this would expire doc_exp_3
time.sleep(5)
# doc_exp_3 should still be around due to removal of expiry
doc_exp_3 = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session)
assert doc_exp_3["_id"] == "exp_3"
# doc_exp_10 should be expired due to the updates (10s) + sleep (5s)
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_10["id"], auth=sg_user_session)
assert he.value[0].startswith("404 Client Error: Not Found for url:")
def test_rolling_ttl_expires(params_from_base_test_setup, sg_conf_name):
"""
1. PUT /db/doc1 via SG with property "_exp":3
2. Update /db/doc1 10 times with a new revision (also with "_exp":3)
3. Wait 5 seconds
4. Get /db/doc1. Assert response is 200
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_rolling_ttl_expires'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=3, channels=sg_user_channels)
doc_exp_10_body = document.create_doc(doc_id="exp_10", expiry=10, channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session)
doc_exp_10 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_10_body, auth=sg_user_session)
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=10, expiry=3, auth=sg_user_session)
# Sleep should allow doc_exp_3 to expire, but still be in the window to get doc_exp_10
time.sleep(5)
# doc_exp_3 should be expired
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session)
assert he.value[0].startswith("404 Client Error: Not Found for url:")
# doc_exp_10 should be available still
doc_exp_10_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_10["id"], auth=sg_user_session)
assert doc_exp_10_result["_id"] == "exp_10"
def test_writing_attachment_to_couchbase_server(params_from_base_test_setup, sg_conf_name):
"""
1. Start sync_gateway with sync function that rejects all writes:
function(doc, oldDoc) {
throw({forbidden:"No writes!"});
}
2. Create a doc with attachment
3. Use CBS sdk to see if attachment doc exists. Doc ID will look like _sync:att:sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0= (where the suffix is the digest)
4. Assert att doc does not exist
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(cluster_config, sg_conf)
topology = cluster_helper.get_cluster_topology(cluster_config)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
sg_db = "db"
bucket = "data-bucket"
log_info("Running 'test_writing_attachment_to_couchbase_server'")
log_info("Using cbs_url: {}".format(cbs_url))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_url_admin: {}".format(sg_url_admin))
log_info("Using sg_db: {}".format(sg_db))
log_info("Using bucket: {}".format(bucket))
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
docs = client.add_docs(url=sg_url, db=sg_db, number=100, id_prefix=sg_db, channels=sg_user_channels, auth=sg_user_session)
assert len(docs) == 100
# Create doc with attachment and push to sync_gateway
doc_with_att = document.create_doc(doc_id="att_doc", content={"sample_key": "sample_val"}, attachment_name="sample_text.txt", channels=sg_user_channels)
client.add_doc(url=sg_url, db=sg_db, doc=doc_with_att, auth=sg_user_session)
server = CouchbaseServer(cbs_url)
# Assert that the attachment doc gets written to couchbase server
server_att_docs = server.get_server_docs_with_prefix(bucket=bucket, prefix="_sync:att:")
num_att_docs = len(server_att_docs)
assert num_att_docs == 1
def test_removing_expiry(params_from_base_test_setup, sg_conf_name):
"""
1. PUT /db/doc1 via SG with property "_exp":3
2. Update /db/doc1 with a new revision with no expiry value
3. After 10 updates, update /db/doc1 with a revision with no expiry
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_removing_expiry'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=3, channels=sg_user_channels)
doc_exp_10_body = document.create_doc(doc_id="exp_10", expiry=10, channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session)
doc_exp_10 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_10_body, auth=sg_user_session)
doc_exp_3_updated = client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=10, auth=sg_user_session)
# Sleep should allow an expiry to happen on doc_exp_3 if it had not been removed.
# Expected behavior is that the doc_exp_3 will still be around due to the removal of the expiry
time.sleep(5)
# doc_exp_3 should no longer have an expiry and should not raise an exception
doc_exp_3_updated_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3_updated["id"], auth=sg_user_session)
assert doc_exp_3_updated_result["_id"] == "exp_3"
# doc_exp_10 should be available still and should not raise an exception
doc_exp_10_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_10["id"], auth=sg_user_session)
assert doc_exp_10_result["_id"] == "exp_10"
def test_setting_expiry_in_bulk_docs(params_from_base_test_setup, sg_conf_name):
"""
1. PUT /db/_bulk_docs with 10 documents. Set the "_exp":3 on 5 of these documents
2. Wait five seconds
3. POST /db/_bulk_get for the 10 documents. Validate that only the 5 non-expiring documents are returned
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_setting_expiry_in_bulk_docs'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
doc_exp_3_bodies = document.create_docs(doc_id_prefix="exp_3", number=5, expiry=3, channels=sg_user_channels)
doc_exp_10_bodies = document.create_docs(doc_id_prefix="exp_10", number=5, expiry=10, channels=sg_user_channels)
bulk_bodies = doc_exp_3_bodies + doc_exp_10_bodies
bulk_docs = client.add_bulk_docs(url=sg_url, db=sg_db, docs=bulk_bodies, auth=sg_user_session)
# Allow exp_3 docs to expire
time.sleep(5)
bulk_get_docs = client.get_bulk_docs(url=sg_url, db=sg_db, docs=bulk_docs, auth=sg_user_session)
expected_ids = ["exp_10_0", "exp_10_1", "exp_10_2", "exp_10_3", "exp_10_4"]
expected_missing_ids = ["exp_3_0", "exp_3_1", "exp_3_2", "exp_3_3", "exp_3_4"]
client.verify_doc_ids_found_in_response(response=bulk_get_docs, expected_doc_ids=expected_ids)
client.verify_doc_ids_not_found_in_response(response=bulk_get_docs, expected_missing_doc_ids=expected_missing_ids)
def test_no_conflicts_enabled(params_from_base_test_setup, sg_conf_name, num_of_docs):
""" @summary : Enable no conflicts and verify conflicts are not created
Test case link : https://docs.google.com/spreadsheets/d/1YwI_gCeoBebQKBybkzoAEoXSc0XLReszDA-mPFQapgk/edit#gid=0
covered #3
Steps:
1. Enable allow_conflicts = false in SG config
2. Add docs to SG.
3. Update the docs few times.
4. Try to create a conflict.
5. Check the revision list for the doc
6. Verify no conflicts can be created.
7. Sync Gateway should respond with a 409 upon trying to add a conflict
8. Should be possible to add more than 1 revision (Default revs_limit = 1000)
"""
# Setup
cluster_config = params_from_base_test_setup["cluster_config"]
topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
no_conflicts_enabled = params_from_base_test_setup["no_conflicts_enabled"]
sg_db = "db"
if not no_conflicts_enabled:
pytest.skip('--no-conflicts is not enabled, so skipping the test')
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
c = cluster.Cluster(cluster_config)
c.reset(sg_conf)
sg_client = MobileRestClient()
channels = ["no-conflicts"]
sg_client.create_user(url=sg_admin_url, db=sg_db, name='autotest', password='******', channels=channels)
autouser_session = sg_client.create_session(url=sg_admin_url, db=sg_db, name='autotest', password='******')
# end of Set up
# 2. Add docs to SG.
sgdoc_bodies = document.create_docs(doc_id_prefix='sg_docs', number=num_of_docs,
attachments_generator=attachment.generate_2_png_10_10, channels=channels)
sg_docs = sg_client.add_bulk_docs(url=sg_url, db=sg_db, docs=sgdoc_bodies, auth=autouser_session)
assert len(sgdoc_bodies) == num_of_docs
# 3. Update the docs few times
sg_client.update_docs(url=sg_url, db=sg_db, docs=sg_docs, number_updates=3, delay=None, auth=autouser_session, channels=channels)
# 4. Try to create a conflict
for doc in sg_docs:
with pytest.raises(HTTPError) as he:
sg_client.add_conflict(url=sg_url, db=sg_db, doc_id=doc["id"], parent_revisions=doc["rev"], new_revision="2-foo",
auth=autouser_session)
assert he.value.message.startswith('409 Client Error: Conflict for url:')
# 6. Update the docs 1 more time
sg_client.update_docs(url=sg_url, db=sg_db, docs=sg_docs, number_updates=1, delay=None, auth=autouser_session, channels=channels)
def test_revs_cache_size(params_from_base_test_setup, sg_conf_name, num_of_docs):
""" @summary Test for no-conflicts with rev_cache size
Test case link : https://docs.google.com/spreadsheets/d/1YwI_gCeoBebQKBybkzoAEoXSc0XLReszDA-mPFQapgk/edit#gid=0
covered #18
Steps:
Note : the sg config have rev_cache_size as 1000 , make sure number of docs is less than 1000 to have the test
work with expected behavior
1. Add docs to SG.
2. Get the docs
3. Verify number of rev_cache_hits is same as number of docs if rev_cache_size is more than number of docs.
"""
# Setup
cluster_config = params_from_base_test_setup["cluster_config"]
topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
retrieved_docs = num_of_docs / 2
sync_gateway_version = params_from_base_test_setup["sync_gateway_version"]
if sync_gateway_version < "2.0":
pytest.skip('It does not work with sg < 2.0 , so skipping the test')
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
c = cluster.Cluster(cluster_config)
c.reset(sg_conf)
sg_client = MobileRestClient()
channels = ["no-conflicts"]
sg_client.create_user(url=sg_admin_url, db=sg_db, name='autotest', password='******', channels=channels)
autouser_session = sg_client.create_session(url=sg_admin_url, db=sg_db, name='autotest', password='******')
# end of Set up
# 2. Add docs to SG.
sgdoc_bodies = document.create_docs(doc_id_prefix="sg_docs", number=num_of_docs, channels=channels)
sg_docs = sg_client.add_bulk_docs(url=sg_url, db=sg_db, docs=sgdoc_bodies, auth=autouser_session)
assert len(sgdoc_bodies) == num_of_docs
# 3. Get all docs
for i in range(retrieved_docs):
doc = sg_docs[i]
sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc["id"], auth=autouser_session)
# 4. Verify there are number of hits should be same as retrieved docs
exp_vars = sg_client.get_expvars(url=sg_admin_url)
revision_cache_hits = exp_vars["syncGateway_stats"]["revisionCache_hits"]
revision_cache_misses = exp_vars["syncGateway_stats"]["revisionCache_misses"]
assert revision_cache_hits == retrieved_docs, "Revision Cache hits did not hit with expected number {}".format(num_of_docs)
assert revision_cache_misses == 0, "Revision Cache misses is not 0"
def test_auto_prune_with_pull(setup_client_syncgateway_test):
"""Sanity test for autopruning with replication
1. Create a database on LiteServ (ls_db)
2. Add doc to sync gateway
3. Update doc 50 times on sync_gateway
4. Set up pull replication from sync_gateway db to LiteServ db
5. Verify number of revisions on client is default (20)
"""
cluster_config = setup_client_syncgateway_test["cluster_config"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_url = setup_client_syncgateway_test["sg_url"]
sg_admin_url = setup_client_syncgateway_test["sg_admin_url"]
client = MobileRestClient()
sg_helper = SyncGateway()
sg_helper.start_sync_gateway(
cluster_config=cluster_config, url=sg_url, config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS)
)
log_info("Running 'test_auto_prune_listener_sanity' ...")
log_info("ls_url: {}".format(ls_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_admin_url: {}".format(sg_admin_url))
num_docs = 1
num_revs = 50
sg_user_channels = ["NBC"]
sg_db = "db"
sg_user_name = "sg_user"
client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels)
sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name)
ls_db = client.create_database(url=ls_url, name="ls_db")
sg_db_docs = client.add_docs(
url=sg_url, db=sg_db, number=num_docs, id_prefix=sg_db, channels=sg_user_channels, auth=sg_session
)
assert len(sg_db_docs) == num_docs
sg_docs_update = client.update_docs(url=sg_url, db=sg_db, docs=sg_db_docs, number_updates=num_revs, auth=sg_session)
# Start continuous replication ls_db <- sg_db
repl_one = client.start_replication(url=ls_url, continuous=True, from_url=sg_admin_url, from_db=sg_db, to_db=ls_db)
client.wait_for_replication_status_idle(url=ls_url, replication_id=repl_one)
client.verify_docs_present(url=ls_url, db=ls_db, expected_docs=sg_docs_update)
client.verify_revs_num_for_docs(url=ls_url, db=ls_db, docs=sg_docs_update, expected_revs_per_doc=20)
def create_users_add_docs_task(user_name, sg_admin_url, sg_url, sg_db,
number_docs_per_user, batch_size, create_delay):
sg_client = MobileRestClient()
# Create user
if user_name.startswith('unique'):
# Doc channel should be unique for each users
channels = [user_name]
elif user_name.startswith('shared'):
# Doc channel should be shared for each doc with this user type
channels = ['shared']
elif user_name.startswith('filtered_channel'):
channels = ['even', 'odd']
elif user_name.startswith('filtered_doc_ids'):
channels = ['terminator']
else:
raise ValueError('Unexpected user type: {}'.format(user_name))
sg_client.create_user(url=sg_admin_url,
db=sg_db,
name=user_name,
password=USER_PASSWORD,
channels=channels)
# Create session
user_auth = sg_client.create_session(url=sg_admin_url,
db=sg_db,
name=user_name,
password=USER_PASSWORD)
# Start bulk doc creation
doc_ids = add_user_docs(client=sg_client,
sg_url=sg_url,
sg_db=sg_db,
user_name=user_name,
user_auth=user_auth,
channels=channels,
number_docs_per_user=number_docs_per_user,
batch_size=batch_size,
create_delay=create_delay)
return user_name, user_auth, doc_ids
def test_load_balance_sanity(params_from_base_test_setup):
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_util = ClusterKeywords()
cluster_util.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf_path
)
topology = cluster_util.get_cluster_topology(cluster_config)
admin_sg_one = topology["sync_gateways"][0]["admin"]
lb_url = "{}:4984".format(topology["load_balancers"][0])
sg_db = "db"
num_docs = 1000
sg_user_name = "seth"
sg_user_password = "******"
channels = ["ABC", "CBS"]
client = MobileRestClient()
user = client.create_user(admin_sg_one, sg_db, sg_user_name, sg_user_password, channels=channels)
session = client.create_session(admin_sg_one, sg_db, sg_user_name)
log_info(user)
log_info(session)
log_info("Adding docs to the load balancer ...")
ct = ChangesTracker(url=lb_url, db=sg_db, auth=session)
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
log_info("Starting ...")
ct_task = executor.submit(ct.start)
log_info("Adding docs ...")
docs = client.add_docs(lb_url, sg_db, num_docs, "test_doc", channels=channels, auth=session)
assert len(docs) == num_docs
log_info("Adding docs done")
wait_for_changes = executor.submit(ct.wait_until, docs)
if wait_for_changes.result():
log_info("Stopping ...")
log_info("Found all docs ...")
executor.submit(ct.stop)
ct_task.result()
else:
executor.submit(ct.stop)
ct_task.result()
raise Exception("Could not find all changes in feed before timeout!!")
def test_load_balance_sanity(params_from_base_test_setup):
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_util = ClusterKeywords()
cluster_util.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf_path
)
topology = cluster_util.get_cluster_topology(cluster_config)
admin_sg_one = topology["sync_gateways"][0]["admin"]
lb_url = topology["load_balancers"][0]
sg_db = "db"
num_docs = 1000
sg_user_name = "seth"
sg_user_password = "******"
channels = ["ABC", "CBS"]
client = MobileRestClient()
user = client.create_user(admin_sg_one, sg_db, sg_user_name, sg_user_password, channels=channels)
session = client.create_session(admin_sg_one, sg_db, sg_user_name)
log_info(user)
log_info(session)
log_info("Adding docs to the load balancer ...")
ct = ChangesTracker(url=lb_url, db=sg_db, auth=session)
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
log_info("Starting ...")
ct_task = executor.submit(ct.start)
log_info("Adding docs ...")
docs = client.add_docs(lb_url, sg_db, num_docs, "test_doc", channels=channels, auth=session)
assert len(docs) == num_docs
log_info("Adding docs done")
wait_for_changes = executor.submit(ct.wait_until, docs)
if wait_for_changes.result():
log_info("Stopping ...")
log_info("Found all docs ...")
executor.submit(ct.stop)
ct_task.result()
else:
executor.submit(ct.stop)
ct_task.result()
raise Exception("Could not find all changes in feed before timeout!!")
def test_longpoll_awaken_channels(params_from_base_test_setup, sg_conf_name):
cluster_conf = params_from_base_test_setup["cluster_config"]
cluster_topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
sg_admin_url = cluster_topology["sync_gateways"][0]["admin"]
sg_url = cluster_topology["sync_gateways"][0]["public"]
log_info("sg_conf: {}".format(sg_conf))
log_info("sg_admin_url: {}".format(sg_admin_url))
log_info("sg_url: {}".format(sg_url))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
adam_user_info = userinfo.UserInfo(name="adam", password="******", channels=["NBC", "ABC"], roles=[])
traun_user_info = userinfo.UserInfo(name="traun", password="******", channels=[], roles=[])
andy_user_info = userinfo.UserInfo(name="andy", password="******", channels=[], roles=[])
sg_db = "db"
doc_id = "adam_doc_0"
client = MobileRestClient()
adam_auth = client.create_user(url=sg_admin_url, db=sg_db,
name=adam_user_info.name, password=adam_user_info.password, channels=adam_user_info.channels)
traun_auth = client.create_user(url=sg_admin_url, db=sg_db,
name=traun_user_info.name, password=traun_user_info.password, channels=traun_user_info.channels)
andy_auth = client.create_user(url=sg_admin_url, db=sg_db,
name=andy_user_info.name, password=andy_user_info.password, channels=andy_user_info.channels)
############################################################
# changes feed wakes with Channel Access via Admin API
############################################################
# Get starting sequence of docs, use the last seq to progress past any _user docs.
adam_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=adam_auth)
traun_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=traun_auth)
andy_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=andy_auth)
with concurrent.futures.ProcessPoolExecutor() as ex:
# Start changes feed for 3 users
adam_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=adam_changes["last_seq"], timeout=10, auth=adam_auth)
traun_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=traun_changes["last_seq"], timeout=10, auth=traun_auth)
andy_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=andy_changes["last_seq"], timeout=10, auth=andy_auth)
# Wait for changes feed to notice there are no changes and enter wait. 2 seconds should be more than enough
time.sleep(2)
# Make sure the changes future is still running and has not exited due to any new changes, the feed should be caught up
# and waiting
assert not adam_changes_task.done()
assert not traun_changes_task.done()
assert not andy_changes_task.done()
# Add add a doc for adam with "NBC" and "ABC" channels
# Add one doc, this should wake up the changes feed
adam_add_docs_task = ex.submit(client.add_docs, url=sg_url, db=sg_db,
number=1, id_prefix="adam_doc",
auth=adam_auth, channels=adam_user_info.channels)
# Wait for docs adds to complete
adam_docs = adam_add_docs_task.result()
assert len(adam_docs) == 1
# Assert that the changes feed woke up and that the doc change was propagated
adam_changes = adam_changes_task.result()
assert len(adam_changes["results"]) == 1
assert adam_changes["results"][0]["id"] == doc_id
# Verify that the changes feed is still listening for Traun and Andy
assert not traun_changes_task.done()
assert not andy_changes_task.done()
# Update the traun and andy to have one of adam's channels
update_traun_user_task = ex.submit(client.update_user, url=sg_admin_url, db=sg_db,
name=traun_user_info.name, password=traun_user_info.password, channels=["NBC"])
traun_auth = update_traun_user_task.result()
update_andy_user_task = ex.submit(client.update_user, url=sg_admin_url, db=sg_db,
name=andy_user_info.name, password=andy_user_info.password, channels=["ABC"])
andy_auth = update_andy_user_task.result()
# Make sure changes feed wakes up and contains at least one change, 2 may be possible if the _user doc is included
# Make sure the first change is 'adam_doc'
traun_changes = traun_changes_task.result()
assert 1 <= len(traun_changes["results"]) <= 2
assert traun_changes["results"][0]["id"] == "adam_doc_0" or traun_changes["results"][0]["id"] == "_user/traun"
andy_changes = andy_changes_task.result()
assert 1 <= len(andy_changes["results"]) <= 2
assert andy_changes["results"][0]["id"] == "adam_doc_0" or andy_changes["results"][0]["id"] == "_user/andy"
# Block until user docs are seen
client.verify_doc_id_in_changes(url=sg_url, db=sg_db, expected_doc_id="_user/adam", auth=adam_auth)
client.verify_doc_id_in_changes(url=sg_url, db=sg_db, expected_doc_id="_user/traun", auth=traun_auth)
client.verify_doc_id_in_changes(url=sg_url, db=sg_db, expected_doc_id="_user/andy", auth=andy_auth)
# Make sure that adams doc shows up in changes due to the fact that the changes feed may be woken up with a _user doc above
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=adam_docs, auth=adam_auth)
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=adam_docs, auth=traun_auth)
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=adam_docs, auth=andy_auth)
############################################################
# changes feed wakes with Channel Removal via Sync function
############################################################
# Get latest last_seq for next test section
adam_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=adam_auth)
traun_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=traun_auth)
andy_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=andy_auth)
with concurrent.futures.ProcessPoolExecutor() as ex:
# Start changes feed for 3 users from latest last_seq
adam_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=adam_changes["last_seq"], timeout=10, auth=adam_auth)
traun_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=traun_changes["last_seq"], timeout=10, auth=traun_auth)
andy_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=andy_changes["last_seq"], timeout=10, auth=andy_auth)
# Wait for changes feed to notice there are no changes and enter wait. 2 seconds should be more than enough
time.sleep(2)
# Make sure the changes future is still running and has not exited due to any new changes, the feed should be caught up
# and waiting
assert not adam_changes_task.done()
assert not traun_changes_task.done()
assert not andy_changes_task.done()
# Remove the channels property from the doc
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_id, auth=traun_auth, channels=[])
# All three changes feeds should wake up and return one result
adam_changes = adam_changes_task.result()
assert len(adam_changes["results"]) == 1
assert adam_changes["results"][0]["removed"] == ["ABC", "NBC"]
traun_changes = traun_changes_task.result()
assert len(traun_changes["results"]) == 1
assert traun_changes["results"][0]["removed"] == ["NBC"]
andy_changes = andy_changes_task.result()
assert len(andy_changes["results"]) == 1
assert andy_changes["results"][0]["removed"] == ["ABC"]
# Verify that users no longer can access the doc
for user_auth in [adam_auth, traun_auth, andy_auth]:
with pytest.raises(requests.exceptions.HTTPError) as excinfo:
client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id, auth=user_auth)
assert "403 Client Error: Forbidden for url:" in excinfo.value.message
############################################################
# changes feed wakes with Channel Grant via Sync function
############################################################
# Get latest last_seq for next test section
adam_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=adam_auth)
traun_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=traun_auth)
andy_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=andy_auth)
admin_auth = client.create_user(url=sg_admin_url, db=sg_db, name="admin", password="******", channels=["admin"])
channel_grant_doc_id = "channel_grant_with_doc_intially"
# Add another doc with no channels
channel_grant_doc_body = document.create_doc(doc_id=channel_grant_doc_id, channels=["admin"])
client.add_doc(url=sg_url, db=sg_db, doc=channel_grant_doc_body, auth=admin_auth)
with concurrent.futures.ProcessPoolExecutor() as ex:
# Start changes feed for 3 users from latest last_seq
adam_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=adam_changes["last_seq"], timeout=10, auth=adam_auth)
traun_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=traun_changes["last_seq"], timeout=10, auth=traun_auth)
andy_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=andy_changes["last_seq"], timeout=10, auth=andy_auth)
# Wait for changes feed to notice there are no changes and enter wait. 2 seconds should be more than enough
time.sleep(2)
# Make sure the changes future is still running and has not exited due to any new changes, the feed should be caught up
# and waiting
assert not adam_changes_task.done()
assert not traun_changes_task.done()
assert not andy_changes_task.done()
# update the grant doc to have channel for all users
update_task = ex.submit(client.update_doc, url=sg_url, db=sg_db, doc_id=channel_grant_doc_id, auth=admin_auth, channels=["admin", "ABC", "NBC"])
updated_doc = update_task.result()
assert updated_doc["rev"].startswith("2-")
# Verify that access grant wakes up changes feed for adam, traun, and Andy
adam_changes = adam_changes_task.result()
assert len(adam_changes["results"]) == 1
assert adam_changes["results"][0]["id"] == "channel_grant_with_doc_intially"
assert adam_changes["results"][0]["changes"][0]["rev"].startswith("2-")
traun_changes = traun_changes_task.result()
assert len(traun_changes["results"]) == 1
assert traun_changes["results"][0]["id"] == "channel_grant_with_doc_intially"
assert traun_changes["results"][0]["changes"][0]["rev"].startswith("2-")
andy_changes = andy_changes_task.result()
assert len(andy_changes["results"]) == 1
assert andy_changes["results"][0]["id"] == "channel_grant_with_doc_intially"
assert andy_changes["results"][0]["changes"][0]["rev"].startswith("2-")
def test_webhooks_crud(params_from_base_test_setup, sg_conf_name, filtered):
""" Tests for webhook notification on import
xattr mode
1. Start sync gateway with autoimport
1. Write 'num_docs_per_client' docs via SDK
1. Write 'num_docs_per_client' docs via SG
1. Verify 'num_docs_per_client' * 2 webhook events (id, rev, body)
1. Update SG docs once each via SDK
1. Update SDK docs once each via SG
1. Verify 'num_docs_per_client' * 2 webhook events (id, rev, body)
1. Delete SG docs via SDK
1. Delete SDK docs via SG
1. Verify 'num_docs_per_client' * 2 webhook events (id, rev, body)
to verify no dups, wait 10s after recieveing expected webhooks
docmeta mode
1. Write 'num_docs_per_client' docs via SG
1. Verify 'num_docs_per_client' webhook events (id, rev, body)
1. Update SG docs once each via SG
1. Verify 'num_docs_per_client' webhook events (id, rev, body)
1. Delete SG docs via SG
1. Verify 'num_docs_per_client' webhook events (id, rev, body)
if filtered, the scenario will add a filtered propery to every other doc.
The webhook validation will only look for the filtered docs
"""
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
log_info('Webhooks filtered?: {}'.format(filtered))
cluster_conf = params_from_base_test_setup['cluster_config']
cluster_topology = params_from_base_test_setup['cluster_topology']
mode = params_from_base_test_setup['mode']
sg_admin_url = cluster_topology['sync_gateways'][0]['admin']
sg_url = cluster_topology['sync_gateways'][0]['public']
cbs_url = cluster_topology['couchbase_servers'][0]
sg_db = 'db'
bucket_name = 'data-bucket'
num_docs_per_client = 100
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_conf)
# Start webhook server on test runner
webhook_server = WebServer()
webhook_server.start()
sg_client = MobileRestClient()
cbs_ip = host_for_url(cbs_url)
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name),
password='******')
sg_info = UserInfo('sg_user', 'pass', channels=['shared'], roles=[])
sdk_info = UserInfo('sdk_user', 'pass', channels=['shared'], roles=[])
sg_client.create_user(url=sg_admin_url,
db=sg_db,
name=sg_info.name,
password=sg_info.password,
channels=sg_info.channels)
sg_auth = sg_client.create_session(url=sg_admin_url,
db=sg_db,
name=sg_info.name,
password=sg_info.password)
# Create sg docs
doc_content = {'aphex': 'twin'}
sg_docs = document.create_docs(doc_id_prefix='sg_user_doc',
number=num_docs_per_client,
content=doc_content,
channels=sg_info.channels)
# Add filtered property to every other doc
count = 0
for sg_doc in sg_docs:
if count % 2 == 0:
sg_doc['filtered'] = True
count += 1
sg_doc_ids = [doc['_id'] for doc in sg_docs]
sg_filtered_doc_ids = [doc['_id'] for doc in sg_docs if 'filtered' in doc]
assert len(sg_doc_ids) == num_docs_per_client
assert len(sg_filtered_doc_ids) == num_docs_per_client / 2
# Create sdk docs
sdk_docs = {
'sdk_user_doc_{}'.format(i): {
'channels': sdk_info.channels,
'content': doc_content
}
for i in range(num_docs_per_client)
}
# Add filtered property to every other doc
count = 0
for _, doc_val in sdk_docs.items():
if count % 2 == 0:
doc_val['filtered'] = True
count += 1
sdk_doc_ids = [doc for doc in sdk_docs]
sdk_filtered_doc_ids = [k for k, v in sdk_docs.items() if 'filtered' in v]
assert len(sdk_doc_ids) == num_docs_per_client
assert len(sdk_filtered_doc_ids) == num_docs_per_client / 2
all_docs = sg_doc_ids + sdk_doc_ids
all_filtered_docs = sg_filtered_doc_ids + sdk_filtered_doc_ids
assert len(all_docs) == num_docs_per_client * 2
# If xattr mode, add sg + sdk docs
# If non xattr mode, add sg docs
add_docs(sg_client=sg_client,
sg_url=sg_url,
sg_db=sg_db,
sg_docs=sg_docs,
sg_auth=sg_auth,
sdk_client=sdk_client,
sdk_docs=sdk_docs,
num_docs_per_client=num_docs_per_client,
xattrs=xattrs_enabled)
# Wait for added docs to trigger webhooks
if xattrs_enabled and filtered:
poll_for_webhook_data(webhook_server, all_filtered_docs, 1,
doc_content)
elif xattrs_enabled and not filtered:
poll_for_webhook_data(webhook_server, all_docs, 1, doc_content)
elif not xattrs_enabled and filtered:
poll_for_webhook_data(webhook_server, sg_filtered_doc_ids, 1,
doc_content)
else:
poll_for_webhook_data(webhook_server, sg_doc_ids, 1, doc_content)
webhook_server.clear_data()
# Update sdk docs from sg
# If xattr mode, update sdk docs from sg, update sg docs from SDK
# If non xattr mode, update sg docs from sg
updated_doc_content = {'brian': 'eno'}
update_docs(sg_client=sg_client,
sg_url=sg_url,
sg_db=sg_db,
sg_doc_ids=sg_doc_ids,
sg_auth=sg_auth,
sdk_client=sdk_client,
sdk_doc_ids=sdk_doc_ids,
updated_doc_content=updated_doc_content,
xattrs=xattrs_enabled)
# Wait for updates to trigger webhooks
if xattrs_enabled and filtered:
poll_for_webhook_data(webhook_server, all_filtered_docs, 2,
updated_doc_content)
elif xattrs_enabled and not filtered:
poll_for_webhook_data(webhook_server, all_docs, 2, updated_doc_content)
elif not xattrs_enabled and filtered:
poll_for_webhook_data(webhook_server, sg_filtered_doc_ids, 2,
updated_doc_content)
else:
poll_for_webhook_data(webhook_server, sg_doc_ids, 2,
updated_doc_content)
webhook_server.clear_data()
delete_docs(sg_client=sg_client,
sg_url=sg_url,
sg_db=sg_db,
sg_doc_ids=sg_doc_ids,
sg_auth=sg_auth,
sdk_client=sdk_client,
sdk_doc_ids=sdk_doc_ids,
xattrs=xattrs_enabled)
# Wait for deletes to trigger webhook events, filter includes all deleted docs
if xattrs_enabled:
poll_for_webhook_data(webhook_server,
all_docs,
3,
updated_doc_content,
deleted=True)
else:
poll_for_webhook_data(webhook_server,
sg_doc_ids,
3,
updated_doc_content,
deleted=True)
webhook_server.clear_data()
# Stop webhook server
webhook_server.stop()
def test_backfill_channels_oneshot_limit_changes(params_from_base_test_setup,
sg_conf_name, grant_type):
"""
Test that checks that docs are backfilled for one shot changes with limit for a access grant (via REST or SYNC)
CHANNEL-REST = Channel is granted to user via REST
CHANNEL-SYNC = Channel is granted to user via sync function access()
ROLE-REST = Role is granted to user via REST
ROLE-SYNC = Role is granted to user via sync function role()
CHANNEL-TO-ROLE-REST = Channel is added to existing role via REST
CHANNEL-TO-ROLE-SYNC = Channel is added to existing role via sync access()
"""
cluster_config = params_from_base_test_setup["cluster_config"]
topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
log_info("grant_type: {}".format(grant_type))
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster = Cluster(cluster_config)
cluster.reset(sg_conf)
client = MobileRestClient()
admin_user_info = userinfo.UserInfo("admin",
"pass",
channels=["A"],
roles=[])
if grant_type == "CHANNEL-TO-ROLE-REST" or grant_type == "CHANNEL-TO-ROLE-SYNC":
client.create_role(url=sg_admin_url,
db=sg_db,
name="empty_role",
channels=[])
user_b_user_info = userinfo.UserInfo("USER_B",
"pass",
channels=["B"],
roles=["empty_role"])
else:
user_b_user_info = userinfo.UserInfo("USER_B",
"pass",
channels=["B"],
roles=[])
# Create users / sessions
client.create_user(url=sg_admin_url,
db=sg_db,
name=admin_user_info.name,
password=admin_user_info.password,
channels=admin_user_info.channels)
client.create_user(url=sg_admin_url,
db=sg_db,
name=user_b_user_info.name,
password=user_b_user_info.password,
channels=user_b_user_info.channels,
roles=user_b_user_info.roles)
admin_session = client.create_session(url=sg_admin_url,
db=sg_db,
name=admin_user_info.name,
password=admin_user_info.password)
user_b_session = client.create_session(url=sg_admin_url,
db=sg_db,
name=user_b_user_info.name,
password=user_b_user_info.password)
# Create 50 "A" channel docs
a_docs = client.add_docs(url=sg_url,
db=sg_db,
number=50,
id_prefix=None,
auth=admin_session,
channels=["A"])
assert len(a_docs) == 50
b_docs = client.add_docs(url=sg_url,
db=sg_db,
number=1,
id_prefix="b_doc",
auth=user_b_session,
channels=["B"])
assert len(b_docs) == 1
# Loop until admin user sees docs in changes
client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=a_docs,
auth=admin_session)
user_doc = {"id": "_user/USER_B", "rev": None}
b_docs.append(user_doc)
# Loop until user_b sees b_doc_0 doc and _user/USER_B doc
client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=b_docs,
auth=user_b_session,
strict=True)
# Get last_seq for user_b
user_b_changes = client.get_changes(url=sg_url,
db=sg_db,
since=0,
auth=user_b_session,
feed="normal")
# Grant access to channel "A"
if grant_type == "CHANNEL-REST":
log_info(
"Granting user access to channel A via Admin REST user update")
# Grant via update to user in Admin API
client.update_user(url=sg_admin_url,
db=sg_db,
name=user_b_user_info.name,
channels=["A", "B"])
elif grant_type == "CHANNEL-SYNC":
log_info("Granting user access to channel A sync function access()")
# Grant via access() in sync_function, then id 'channel_access' will trigger an access(doc.users, doc.channels)
access_doc = document.create_doc("channel_access", channels=["A"])
access_doc["users"] = ["USER_B"]
client.add_doc(url=sg_url,
db=sg_db,
doc=access_doc,
auth=admin_session)
elif grant_type == "ROLE-REST":
log_info("Granting user access to channel A via Admin REST role grant")
# Create role with channel A
client.create_role(url=sg_admin_url,
db=sg_db,
name="channel-A-role",
channels=["A"])
client.update_user(url=sg_admin_url,
db=sg_db,
name="USER_B",
roles=["channel-A-role"])
elif grant_type == "ROLE-SYNC":
log_info(
"Granting user access to channel A via sync function role() grant")
# Create role with channel A
client.create_role(url=sg_admin_url,
db=sg_db,
name="channel-A-role",
channels=["A"])
# Grant via role() in sync_function, then id 'role_access' will trigger an role(doc.users, doc.roles)
role_access_doc = document.create_doc("role_access")
role_access_doc["users"] = ["USER_B"]
role_access_doc["roles"] = ["role:channel-A-role"]
client.add_doc(sg_url,
db=sg_db,
doc=role_access_doc,
auth=admin_session)
elif grant_type == "CHANNEL-TO-ROLE-REST":
# Update the empty_role to have channel "A"
client.update_role(url=sg_admin_url,
db=sg_db,
name="empty_role",
channels=["A"])
elif grant_type == "CHANNEL-TO-ROLE-SYNC":
# Grant empty_role access to channel "A" via sync function
# Grant channel access to role via sync function
access_doc = document.create_doc("channel_grant_to_role")
access_doc["roles"] = ["role:empty_role"]
access_doc["channels"] = ["A"]
client.add_doc(url=sg_url,
db=sg_db,
doc=access_doc,
auth=admin_session,
use_post=True)
else:
pytest.fail("Unsupported grant_type!!!!")
# Create a dictionary keyed on doc id for all of channel A docs
ids_and_revs_from_a_docs = {doc["id"]: doc["rev"] for doc in a_docs}
assert len(ids_and_revs_from_a_docs.keys()) == 50
log_info("Doing 3, 1 shot changes with limit and last seq!")
# Issue 3 oneshot changes with a limit of 20
# Issue one shot changes to make sure access grant is successful, the change may not propagate immediately so retry.
num_retries = 3
count = 0
while True:
if count == num_retries:
raise exceptions.ChangesError(
"Didn't get all expected changes before timing out!")
user_b_changes_after_grant_one = client.get_changes(
url=sg_url,
db=sg_db,
since=user_b_changes["last_seq"],
auth=user_b_session,
feed="normal",
limit=20)
if len(user_b_changes_after_grant_one["results"]) > 0:
# Found changes, break out an validate changes!
break
time.sleep(1)
count += 1
#################
# Changes Req #1
#################
# Expect a user doc in the changes
if grant_type == "CHANNEL-REST" or grant_type == "ROLE-REST":
ids_and_revs_from_a_docs["_user/USER_B"] = None
# User B shoud have recieved 20 docs due to limit
assert len(user_b_changes_after_grant_one["results"]) == 20
for doc in user_b_changes_after_grant_one["results"]:
# cross off keys found from 'a_docs' dictionary
del ids_and_revs_from_a_docs[doc["id"]]
#################
# Changes Req #2
#################
user_b_changes_after_grant_two = client.get_changes(
url=sg_url,
db=sg_db,
since=user_b_changes_after_grant_one["last_seq"],
auth=user_b_session,
feed="normal",
limit=20)
# User B shoud have recieved 20 docs due to limit
assert len(user_b_changes_after_grant_two["results"]) == 20
for doc in user_b_changes_after_grant_two["results"]:
# cross off keys found from 'a_docs' dictionary
del ids_and_revs_from_a_docs[doc["id"]]
#################
# Changes Req #3
#################
user_b_changes_after_grant_three = client.get_changes(
url=sg_url,
db=sg_db,
since=user_b_changes_after_grant_two["last_seq"],
auth=user_b_session,
feed="normal",
limit=20)
# User B should have recieved 10 docs due to limit or 11 docs with with a terminating _user doc
# The terminating user doc only happens with a grant via REST
if grant_type == "CHANNEL-REST" or grant_type == "ROLE-REST":
assert len(user_b_changes_after_grant_three["results"]) == 11
else:
assert len(user_b_changes_after_grant_three["results"]) == 10
for doc in user_b_changes_after_grant_three["results"]:
# cross off non user doc keys found from 'a_docs' dictionary
del ids_and_revs_from_a_docs[doc["id"]]
# Make sure all the docs have been crossed out
assert len(ids_and_revs_from_a_docs) == 0
#################
# Changes Req #4
#################
user_b_changes_after_grant_four = client.get_changes(
url=sg_url,
db=sg_db,
since=user_b_changes_after_grant_three["last_seq"],
auth=user_b_session,
feed="normal",
limit=20)
# Changes should be caught up and there should be no results
assert len(user_b_changes_after_grant_four["results"]) == 0
def test_awaken_backfill_channels_longpoll_changes_with_limit(
params_from_base_test_setup, sg_conf_name, grant_type):
"""
Test that checks that docs are backfilled for logpoll changes with limit for a access grant (via REST or SYNC)
CHANNEL-REST = Channel is granted to user via REST
CHANNEL-SYNC = Channel is granted to user via sync function access()
ROLE-REST = Role is granted to user via REST
ROLE-SYNC = Role is granted to user via sync function role()
CHANNEL-TO-ROLE-REST = Channel is added to existing role via REST
CHANNEL-TO-ROLE-SYNC = Channel is added to existing role via sync access()
"""
cluster_config = params_from_base_test_setup["cluster_config"]
topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
log_info("grant_type: {}".format(grant_type))
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster = Cluster(cluster_config)
cluster.reset(sg_conf)
client = MobileRestClient()
admin_user_info = userinfo.UserInfo("admin",
"pass",
channels=["A"],
roles=[])
if grant_type == "CHANNEL-TO-ROLE-REST" or grant_type == "CHANNEL-TO-ROLE-SYNC":
client.create_role(url=sg_admin_url,
db=sg_db,
name="empty_role",
channels=[])
user_b_user_info = userinfo.UserInfo("USER_B",
"pass",
channels=["B"],
roles=["empty_role"])
else:
user_b_user_info = userinfo.UserInfo("USER_B",
"pass",
channels=["B"],
roles=[])
# Create users / sessions
client.create_user(
url=sg_admin_url,
db=sg_db,
name=admin_user_info.name,
password=admin_user_info.password,
channels=admin_user_info.channels,
)
client.create_user(url=sg_admin_url,
db=sg_db,
name=user_b_user_info.name,
password=user_b_user_info.password,
channels=user_b_user_info.channels,
roles=user_b_user_info.roles)
admin_session = client.create_session(url=sg_admin_url,
db=sg_db,
name=admin_user_info.name,
password=admin_user_info.password)
user_b_session = client.create_session(url=sg_admin_url,
db=sg_db,
name=user_b_user_info.name,
password=user_b_user_info.password)
# Create 50 "A" channel docs
a_docs = client.add_docs(url=sg_url,
db=sg_db,
number=50,
id_prefix=None,
auth=admin_session,
channels=["A"])
assert len(a_docs) == 50
b_docs = client.add_docs(url=sg_url,
db=sg_db,
number=1,
id_prefix="b_doc",
auth=user_b_session,
channels=["B"])
assert len(b_docs) == 1
user_doc = {"id": "_user/USER_B", "rev": None}
b_docs.append(user_doc)
# Loop until user_b sees b_doc_0 doc and _user/USER_B doc
client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=b_docs,
auth=user_b_session,
strict=True)
# Create a dictionary keyed on doc id for all of channel A docs
ids_and_revs_from_a_docs = {doc["id"]: doc["rev"] for doc in a_docs}
assert len(ids_and_revs_from_a_docs.keys()) == 50
# Get last_seq for user_b
user_b_changes = client.get_changes(url=sg_url,
db=sg_db,
since=0,
auth=user_b_session,
feed="normal")
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as ex:
# Start long poll changes feed.
changes_task = ex.submit(client.get_changes,
url=sg_url,
db=sg_db,
since=user_b_changes["last_seq"],
auth=user_b_session,
timeout=10,
limit=20)
# Grant access to channel "A"
if grant_type == "CHANNEL-REST":
log_info(
"Granting user access to channel A via Admin REST user update")
# Grant via update to user in Admin API
client.update_user(url=sg_admin_url,
db=sg_db,
name=user_b_user_info.name,
channels=["A", "B"])
elif grant_type == "CHANNEL-SYNC":
log_info(
"Granting user access to channel A sync function access()")
# Grant via access() in sync_function,
# then id 'channel_access' will trigger an access(doc.users, doc.channels)
access_doc = document.create_doc("channel_access", channels=["A"])
access_doc["users"] = ["USER_B"]
client.add_doc(url=sg_url,
db=sg_db,
doc=access_doc,
auth=admin_session)
elif grant_type == "ROLE-REST":
log_info(
"Granting user access to channel A via Admin REST role grant")
# Create role with channel A
client.create_role(url=sg_admin_url,
db=sg_db,
name="channel-A-role",
channels=["A"])
client.update_user(url=sg_admin_url,
db=sg_db,
name=user_b_user_info.name,
roles=["channel-A-role"])
elif grant_type == "ROLE-SYNC":
log_info(
"Granting user access to channel A via sync function role() grant"
)
# Create role with channel A
client.create_role(url=sg_admin_url,
db=sg_db,
name="channel-A-role",
channels=["A"])
# Grant via role() in sync_function, then id 'role_access' will trigger an role(doc.users, doc.roles)
role_access_doc = document.create_doc("role_access")
role_access_doc["users"] = ["USER_B"]
role_access_doc["roles"] = ["role:channel-A-role"]
client.add_doc(sg_url,
db=sg_db,
doc=role_access_doc,
auth=admin_session)
elif grant_type == "CHANNEL-TO-ROLE-REST":
# Update the empty_role to have channel "A"
client.update_role(url=sg_admin_url,
db=sg_db,
name="empty_role",
channels=["A"])
elif grant_type == "CHANNEL-TO-ROLE-SYNC":
# Grant empty_role access to channel "A" via sync function
# Grant channel access to role via sync function
access_doc = document.create_doc("channel_grant_to_role")
access_doc["roles"] = ["role:empty_role"]
access_doc["channels"] = ["A"]
client.add_doc(url=sg_url,
db=sg_db,
doc=access_doc,
auth=admin_session,
use_post=True)
else:
pytest.fail("Unsupported grant_type!!!!")
# Block on return of longpoll changes, feed should wake up and return 20 results
changes = changes_task.result()
assert len(changes["results"]) == 20
num_requests = 1
# append _user/doc to the doc scratch pad if a REST grant
if grant_type == "CHANNEL-REST" or grant_type == "ROLE-REST":
ids_and_revs_from_a_docs["_user/USER_B"] = None
# Cross the results off from the 'a_docs' dictionary
for doc in changes["results"]:
del ids_and_revs_from_a_docs[doc["id"]]
# Start looping longpoll changes with limit, cross off changes from dictionary each time one is found
# Since 20 changes should be crossed off already, this should execute 2x.
log_info("Starting looping longpoll changes with limit!")
last_seq = changes["last_seq"]
while True:
if len(ids_and_revs_from_a_docs.keys()) == 0:
log_info("All docs were found! Exiting polling loop")
break
changes = client.get_changes(url=sg_url,
db=sg_db,
since=last_seq,
auth=user_b_session,
limit=20,
timeout=10)
num_requests += 1
# There are more than 2 requests, throw an exception.
if num_requests == 2:
assert len(changes["results"]) == 20
elif num_requests == 3:
# This will be 10 or 11 depending on if the _user/ doc is returned
if grant_type == "CHANNEL-REST" or grant_type == "ROLE-REST":
assert len(changes["results"]) == 11
else:
assert len(changes["results"]) == 10
else:
raise exceptions.ChangesError(
"Looping longpoll should only have to perform 3 requests to get all the changes!!"
)
# Cross the results off from the 'a_docs' dictionary.
# This will blow up in docs duplicate docs are sent to changes
for doc in changes["results"]:
del ids_and_revs_from_a_docs[doc["id"]]
last_seq = changes["last_seq"]
# Shanges after longpoll
zero_results = client.get_changes(url=sg_url,
db=sg_db,
since=last_seq,
auth=user_b_session,
feed="normal")
# Changes should be caught up and there should be no results
assert len(zero_results["results"]) == 0
def test_string_expiry_as_iso_8601_date(params_from_base_test_setup, sg_conf_name):
"""
1. Calculate (server time + 3 seconds) as ISO-8601 date (e.g. 2016-01-01T00:00:00.000+00:00)
2. PUT /db/doc1 via SG with property "_exp":"[date]"
PUT /db/doc2 via SG with property "_exp":"2026-01-01T00:00:00.000+00:00"
3. Wait five seconds
4. Get /db/doc1. Assert response is 404
Get /db/doc2. Assert response is 20
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_string_expiry_as_ISO_8601_Date'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
time_util = Time()
iso_datetime = time_util.get_iso_datetime(delta=3)
doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=iso_datetime, channels=sg_user_channels)
doc_exp_years_body = document.create_doc(doc_id="exp_years", expiry="2026-01-01T00:00:00.000+00:00", channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session)
doc_exp_years = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_years_body, auth=sg_user_session)
# Sleep should allow doc_exp_3 to expire
time.sleep(10)
# doc_exp_3 should be expired
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session)
assert he.value[0].startswith("404 Client Error: Not Found for url:")
# doc_exp_years should be available still
doc_exp_years_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_years["id"], auth=sg_user_session)
assert doc_exp_years_result["_id"] == "exp_years"
def test_mobile_opt_in(params_from_base_test_setup, sg_conf_name):
"""
Scenario: Enable mobile opt in sync function in sync-gateway configuration file
- Check xattrs/mobile-opt-in_cc or di json files
- 8 cases covered
- doc : https://docs.google.com/document/d/1XxLIBsjuj_UxTTJs4Iu7C7uZdos8ZEzeckrVc17y3sw/edit
- #1 Create doc via sdk with mobile opt in and verify doc is imported
- #2 Create doc via sdk with mobile opt out and verify doc is not imported
- #3 Create doc via sg with mobile opt in and update via sdk and verify doc is imported
- #4 Create doc via sg with mobile opt out and update via sdk and verify doc is not imported
- Try to update same doc via sg and verify 409 conflict error is thrown
- Create a doc with same doc id and verify doc is created successfully
- #5 Create doc via sg with mobile opt out and update via sdk which created no revisions
- Now do sdk create with mobile opt in should import case #5
- #6 Create doc via sg with mobile opt out and update via sdk with opt in
- Verify type is overrided and doc is imported
- #7 Create doc via sg with mobile opt in and update via sdk with opt out
- Verify type is overrided and doc is not imported
- #8 Disable import in the sg config and have mobile opt in function
Create doc via sdk with mobile property and verify sg update succeeds
- #9 Same config as #8 and have mobile opt in function in config
Create doc via sdk without mobile property and create new doc via sg with same doc id and
verify it succeeds
"""
bucket_name = 'data-bucket'
sg_db = 'db'
cluster_conf = params_from_base_test_setup['cluster_config']
cluster_topology = params_from_base_test_setup['cluster_topology']
mode = params_from_base_test_setup['mode']
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
# This test should only run when using xattr meta storage
if not xattrs_enabled:
pytest.skip('XATTR tests require --xattrs flag')
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
sg_admin_url = cluster_topology['sync_gateways'][0]['admin']
sg_url = cluster_topology['sync_gateways'][0]['public']
cbs_url = cluster_topology['couchbase_servers'][0]
log_info('sg_conf: {}'.format(sg_conf))
log_info('sg_admin_url: {}'.format(sg_admin_url))
log_info('sg_url: {}'.format(sg_url))
log_info('cbs_url: {}'.format(cbs_url))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
# Create clients
sg_client = MobileRestClient()
cbs_ip = host_for_url(cbs_url)
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name), password='******', timeout=SDK_TIMEOUT)
# Create user / session
auto_user_info = UserInfo(name='autotest', password='******', channels=['mobileOptIn'], roles=[])
sg_client.create_user(
url=sg_admin_url,
db=sg_db,
name=auto_user_info.name,
password=auto_user_info.password,
channels=auto_user_info.channels
)
test_auth_session = sg_client.create_session(
url=sg_admin_url,
db=sg_db,
name=auto_user_info.name,
password=auto_user_info.password
)
def update_mobile_prop():
return {
'updates': 0,
'type': 'mobile',
}
def update_non_mobile_prop():
return {
'updates': 0,
'test': 'true',
'type': 'mobile opt out',
}
# Create first doc via SDK with type mobile. Case #1
doc_id1 = 'mobile_opt_in_sdk_doc'
doc = document.create_doc(doc_id=doc_id1, channels=['mobileOptIn'], prop_generator=update_mobile_prop)
sdk_client.upsert(doc_id1, doc)
sg_get_doc1 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id1, auth=test_auth_session)
assert sg_get_doc1['_rev'].startswith('1-') and sg_get_doc1['_id'] == doc_id1
# Additional coverage for case #1
sg_client.update_doc(url=sg_url, db=sg_db, doc_id=doc_id1, number_updates=1, auth=test_auth_session)
sg_get_doc1 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id1, auth=test_auth_session)
assert sg_get_doc1['_rev'].startswith('2-') and sg_get_doc1['_id'] == doc_id1
# Create second doc via SDK with type non mobile. Case #2
doc_id2 = 'mobile_opt_out_sdk_doc'
doc = document.create_doc(doc_id=doc_id2, channels=['mobileOptIn'], prop_generator=update_non_mobile_prop)
sdk_client.upsert(doc_id2, doc)
with pytest.raises(HTTPError) as he:
sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id2, auth=test_auth_session)
log_info(he.value)
assert he.value.message.startswith('404 Client Error: Not Found for url:')
# Create third sg doc with mobile opt in and update via sdk. Case #3
doc_id3 = 'mobile_opt_in_sg_doc'
doc_body = document.create_doc(doc_id=doc_id3, channels=['mobileOptIn'], prop_generator=update_mobile_prop)
doc = sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session)
sg_get_doc3 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id3, auth=test_auth_session)
sg_get_doc3["updated_sdk_via_sg"] = "1"
sdk_client.upsert(doc_id3, sg_get_doc3)
sg_get_doc3 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id3, auth=test_auth_session)
assert sg_get_doc3['_rev'].startswith('2-') and sg_get_doc3['_id'] == doc_id3
log_info("sg get doc3 is {}".format(sg_get_doc3))
# Create fourth sg doc with mobile opt out and update via sdk. Case #4 and case #8
doc_id4 = 'mobile_opt_out_sg_doc'
doc_body = document.create_doc(doc_id=doc_id4, channels=['mobileOptIn'], prop_generator=update_non_mobile_prop)
doc = sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session)
# update vis SDK
sg_get_doc4 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id4, auth=test_auth_session)
rev = sg_get_doc4['_rev']
sg_get_doc4["updated_sdk_via_sg"] = "1"
sdk_client.upsert(doc_id4, sg_get_doc4)
with pytest.raises(HTTPError) as he:
sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id4, auth=test_auth_session)
log_info(he.value)
assert he.value.message.startswith('404 Client Error: Not Found for url:')
# update via SG
with pytest.raises(HTTPError) as he:
sg_client.put_doc(url=sg_url, db=sg_db, doc_id=doc_id4, doc_body={'sg_rewrite': 'True'}, rev=rev, auth=test_auth_session)
log_info(he.value)
assert he.value.message.startswith('409 Client Error: Conflict for url:')
# Create same doc again to verify there is not existing key error covers case #8
doc_body = document.create_doc(doc_id=doc_id4, channels=['mobileOptIn'], prop_generator=update_non_mobile_prop)
sg_get_doc4_1 = sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session)
log_info("4th doc after recreate vis sg is {}".format(sg_get_doc4_1))
assert sg_get_doc4_1['rev'].startswith('1-') and sg_get_doc4_1['id'] == doc_id4
# Create Fifth sg doc with mobile opt in and delete doc which created no revisions i.e tombstone doc
# Now do sdk create with mobile opt in should import case #5
doc_id5 = 'mobile_sdk_recreate_no_activerev'
doc_body = document.create_doc(doc_id=doc_id5, channels=['mobileOptIn'], prop_generator=update_mobile_prop)
doc = sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session)
rev = doc['rev']
sg_client.delete_doc(url=sg_url, db=sg_db, doc_id=doc_id5, rev=rev, auth=test_auth_session)
# At this point no active revisions for this doc, so now update via sdk with mobile opt in should be successful
# in getting doc
doc = document.create_doc(doc_id=doc_id5, channels=['mobileOptIn'], prop_generator=update_mobile_prop)
sdk_client.upsert(doc_id5, doc)
sg_get_doc5 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id5, auth=test_auth_session)
log_info("sg get doc 5 is {}".format(sg_get_doc5))
assert sg_get_doc5['_rev'].startswith('1-') and sg_get_doc5['_id'] == doc_id5
# Create sixth sg doc with mobile opt out and update via sdk with opt in
doc_id6 = 'mobileoptout_sg_doc_sdkupdate_optin'
doc_body = document.create_doc(doc_id=doc_id6, channels=['mobileOptIn'], prop_generator=update_non_mobile_prop)
doc = sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session)
sg_get_doc6 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id6, auth=test_auth_session)
log_info("Sg sixth doc is {}".format(sg_get_doc6))
sg_get_doc6["type"] = "mobile"
sdk_client.upsert(doc_id6, sg_get_doc6)
sg_get_doc6 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id6, auth=test_auth_session)
assert sg_get_doc6['_rev'].startswith('2-') and sg_get_doc6['_id'] == doc_id6
# Create seventh sg doc with mobile opt in and update via sdk with opt out
doc_id7 = 'mobileoptin_sg_doc_sdkupdate_optout'
doc_body = document.create_doc(doc_id=doc_id7, channels=['mobileOptIn'], prop_generator=update_mobile_prop)
doc = sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session)
sg_get_doc7 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id7, auth=test_auth_session)
log_info("Sg sixth doc is {}".format(sg_get_doc7))
sg_get_doc7["type"] = "mobile opt out"
sdk_client.upsert(doc_id7, sg_get_doc7)
with pytest.raises(HTTPError) as he:
sg_get_doc7 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id7, auth=test_auth_session)
log_info(he.value)
assert he.value.message.startswith('404 Client Error: Not Found for url:')
# TODO : verify _changes that it shows tombstone revisions -> it will happen on 2.0
# Create eighth sdk doc with import disabled and add mobile property and update via sg. Case #7
sg_conf_name = "xattrs/mobile_opt_in_no_import"
sg_no_import_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
sg_util = SyncGateway()
sg_util.start_sync_gateways(cluster_config=cluster_conf, url=sg_url, config=sg_no_import_conf)
doc_id8 = 'mobile_opt_in_sg_rewrite_with_importdisabled'
doc_body = document.create_doc(doc_id=doc_id8, channels=['mobileOptIn'], prop_generator=update_mobile_prop)
sdk_client.upsert(doc_id8, doc_body)
with pytest.raises(HTTPError) as he:
sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session)
log_info(he.value)
assert he.value.message.startswith('409 Client Error: Conflict for url:')
sg_client.update_doc(url=sg_url, db=sg_db, doc_id=doc_id8, number_updates=1, auth=test_auth_session)
sg_get_doc8 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id8, auth=test_auth_session)
assert sg_get_doc8['_rev'].startswith('2-') and sg_get_doc8['_id'] == doc_id8
# Create ninth sdk doc with import disabled and add mobile property and update via sg. Case #8
doc_id9 = 'mobile_opt_out_sg_rewrite_with_importdisabled'
doc_body = document.create_doc(doc_id=doc_id9, channels=['mobileOptIn'], prop_generator=update_non_mobile_prop)
sdk_client.upsert(doc_id9, doc_body)
sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session)
# sg_client.update_doc(url=sg_url, db=sg_db, doc_id=doc_id8, number_updates=1, auth=test_auth_session)
sg_get_doc9 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id9, auth=test_auth_session)
assert sg_get_doc9['_rev'].startswith('1-') and sg_get_doc9['_id'] == doc_id9
def test_attachment_revpos_when_ancestor_unavailable(
params_from_base_test_setup, sg_conf_name):
"""
Creates a document with an attachment, then updates that document so that
the body of the revision that originally pushed the document is no
longer available. Add a new revision that's not a child of the
active revision, and validate that it's uploaded successfully.
Example:
1. Document is created with attachment at rev-1
2. Document is updated (strip digests and length, only put revpos & stub) multiple times on the server, goes to rev-10
3. Client attempts to add a new (conflicting) revision 2, with parent rev-1.
4. If the body of rev-1 is no longer available on the server (temporary backup of revision has expired, and is no longer stored
in the in-memory rev cache), we were throwing an error to client
because we couldn't verify based on the _attachments property in rev-1.
5. In this scenario, before returning error, we are now checking if the active revision has a common ancestor with the incoming revision.
If so, we can validate any revpos values equal to or earlier than the common ancestor against the active revision
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
no_conflicts_enabled = params_from_base_test_setup["no_conflicts_enabled"]
if no_conflicts_enabled:
pytest.skip('--no-conflicts is not enabled, so skipping the test')
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(cluster_config, sg_conf)
topology = cluster_helper.get_cluster_topology(cluster_config)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
sg_db = "db"
bucket = "data-bucket"
log_info("Running 'test_attachment_revpos_when_ancestor_unavailable'")
log_info("Using cbs_url: {}".format(cbs_url))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_url_admin: {}".format(sg_url_admin))
log_info("Using sg_db: {}".format(sg_db))
log_info("Using bucket: {}".format(bucket))
channels_list = ["ABC"]
client = MobileRestClient()
sg_util = SyncGateway()
cb_server = couchbaseserver.CouchbaseServer(cbs_url)
user1 = client.create_user(url=sg_url_admin,
db=sg_db,
name="user1",
password="******",
channels=channels_list)
atts = attachment.load_from_data_dir(["sample_text.txt"])
doc_with_att = document.create_doc(doc_id="att_doc",
content={"sample_key": "sample_val"},
attachments=atts,
channels=channels_list)
doc_gen_1 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_with_att,
auth=user1)
client.update_doc(url=sg_url,
db=sg_db,
doc_id=doc_gen_1["id"],
number_updates=10,
auth=user1)
# Clear cached rev doc bodys from server and cycle sync_gateway
sg_util.stop_sync_gateways(cluster_config=cluster_config, url=sg_url)
cb_server.delete_couchbase_server_cached_rev_bodies(bucket=bucket)
sg_util.start_sync_gateways(cluster_config=cluster_config,
url=sg_url,
config=sg_conf)
client.add_conflict(url=sg_url,
db=sg_db,
doc_id=doc_gen_1["id"],
parent_revisions=doc_gen_1["rev"],
new_revision="2-foo",
auth=user1)
def test_writing_attachment_to_couchbase_server(params_from_base_test_setup,
sg_conf_name):
"""
1. Start sync_gateway with sync function that rejects all writes:
function(doc, oldDoc) {
throw({forbidden:"No writes!"});
}
2. Create a doc with attachment
3. Use CBS sdk to see if attachment doc exists. Doc ID will look like _sync:att:sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0= (where the suffix is the digest)
4. Assert att doc does not exist
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(cluster_config, sg_conf)
topology = cluster_helper.get_cluster_topology(cluster_config)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
sg_db = "db"
bucket = "data-bucket"
log_info("Running 'test_writing_attachment_to_couchbase_server'")
log_info("Using cbs_url: {}".format(cbs_url))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_url_admin: {}".format(sg_url_admin))
log_info("Using sg_db: {}".format(sg_db))
log_info("Using bucket: {}".format(bucket))
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
docs = client.add_docs(url=sg_url,
db=sg_db,
number=100,
id_prefix=sg_db,
channels=sg_user_channels,
auth=sg_user_session)
assert len(docs) == 100
# Create doc with attachment and push to sync_gateway
atts = attachment.load_from_data_dir(["sample_text.txt"])
doc_with_att = document.create_doc(doc_id="att_doc",
content={"sample_key": "sample_val"},
attachments=atts,
channels=sg_user_channels)
client.add_doc(url=sg_url,
db=sg_db,
doc=doc_with_att,
auth=sg_user_session)
server = couchbaseserver.CouchbaseServer(cbs_url)
# Assert that the attachment doc gets written to couchbase server
server_att_docs = server.get_server_docs_with_prefix(bucket=bucket,
prefix="_sync:att:")
num_att_docs = len(server_att_docs)
assert num_att_docs == 1
def test_take_all_sgaccels_down(params_from_base_test_setup, sg_conf):
"""
Scenario that takes all sync_gateway accel nodes offline during doc load.
After bring the nodes back online during load, the reshard of the DCP feed is verified.
The changes feed is verified that all docs show up.
1. Start doc load (1000 doc)
2. Take all sg_accel nodes down in parallel
3. Verify node are down
4. Wait for doc adds to complete, store "doc_push_result_1"
5. Verify "doc_push_result_1" docs added
6. Start doc load (1000 docs)
7. Wait for 5. to complete, store "doc_push_result_2"
8. Verify "doc_push_result_2" docs added
9. Start another doc load (1000 docs)
10. Bring up nodes in parallel
11. poll on p-index reshard
12. Wait for 9. to complete, store "doc_push_result_3"
13. Verify "doc_push_result_3" docs added
14. Verify "doc_push_result_1" + "doc_push_result_2" + "doc_push_result_3" show up in _changes feed
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_dcp_reshard_single_sg_accel_goes_down_and_up'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_util = ClusterKeywords()
topology = cluster_util.get_cluster_topology(cluster_conf)
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
num_docs = 1000
client = MobileRestClient()
doc_pusher_user_info = userinfo.UserInfo("doc_pusher", "pass", channels=["A"], roles=[])
doc_pusher_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=doc_pusher_user_info.name,
password=doc_pusher_user_info.password,
channels=doc_pusher_user_info.channels
)
a_user_info = userinfo.UserInfo("a_user", "pass", channels=["A"], roles=[])
client.create_user(
url=sg_admin_url,
db=sg_db,
name=a_user_info.name,
password=a_user_info.password,
channels=a_user_info.channels
)
a_user_session = client.create_session(
url=sg_admin_url,
db=sg_db,
name=a_user_info.name,
password=a_user_info.password
)
# Shutdown all accel nodes in parallel
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as ex:
# Start adding docs
docs_1 = document.create_docs(None, num_docs, channels=doc_pusher_user_info.channels)
docs_1_task = ex.submit(client.add_bulk_docs, url=sg_url, db=sg_db, docs=docs_1, auth=doc_pusher_auth)
# Take down all access nodes
log_info("Shutting down sg_accels: [{}, {}, {}] ...".format(
cluster.sg_accels[0],
cluster.sg_accels[1],
cluster.sg_accels[2]
))
sg_accel_down_task_1 = ex.submit(cluster.sg_accels[0].stop)
sg_accel_down_task_2 = ex.submit(cluster.sg_accels[1].stop)
sg_accel_down_task_3 = ex.submit(cluster.sg_accels[2].stop)
assert sg_accel_down_task_1.result() == 0
assert sg_accel_down_task_2.result() == 0
assert sg_accel_down_task_3.result() == 0
# Block until bulk_docs is complete
doc_push_result_1 = docs_1_task.result()
assert len(doc_push_result_1) == num_docs
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=doc_push_result_1, auth=doc_pusher_auth)
# Load sync_gateway with another batch of docs while the sg_accel nodes are offline
docs_2_bodies = document.create_docs(None, num_docs, channels=doc_pusher_user_info.channels)
docs_push_result_2 = client.add_bulk_docs(url=sg_url, db=sg_db, docs=docs_2_bodies, auth=doc_pusher_auth)
assert len(docs_push_result_2) == num_docs
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=docs_push_result_2, auth=doc_pusher_auth)
# Start loading Sync Gateway with another set of docs while bringing the sg_accel nodes online
docs_3 = document.create_docs(None, num_docs, channels=doc_pusher_user_info.channels)
docs_3_task = ex.submit(client.add_bulk_docs, url=sg_url, db=sg_db, docs=docs_3, auth=doc_pusher_auth)
# Bring all the sg_accel nodes back up
# Take down all access nodes
log_info("Starting sg_accels: [{}, {}, {}] ...".format(
cluster.sg_accels[0],
cluster.sg_accels[1],
cluster.sg_accels[2]
))
sg_accel_up_task_1 = ex.submit(cluster.sg_accels[0].start, sg_conf)
sg_accel_up_task_2 = ex.submit(cluster.sg_accels[1].start, sg_conf)
sg_accel_up_task_3 = ex.submit(cluster.sg_accels[2].start, sg_conf)
assert sg_accel_up_task_1.result() == 0
assert sg_accel_up_task_2.result() == 0
assert sg_accel_up_task_3.result() == 0
# Wait for pindex to reshard correctly
assert cluster.validate_cbgt_pindex_distribution_retry(3)
# Block until second bulk_docs is complete
doc_push_result_3 = docs_3_task.result()
assert len(doc_push_result_3) == num_docs
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=doc_push_result_3, auth=doc_pusher_auth)
# Combine the 3 push results and make sure the changes propagate to a_user
# a_user has access to the doc's channel.
log_info("Verifying all the changes show up for 'a_user' ...")
all_docs = doc_push_result_1 + docs_push_result_2 + doc_push_result_3
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=all_docs, auth=a_user_session, polling_interval=2)
def test_attachment_revpos_when_ancestor_unavailable_active_revision_doesnt_share_ancestor(
params_from_base_test_setup, sg_conf_name):
"""
Creates a document with an attachment, then updates that document so that
the body of the revision that originally pushed the document is no
longer available. Add a new revision that's not a child of the
active revision, and validate that it's uploaded successfully.
Example:
1. Document is created with no attachment at rev-1
2. Server adds revision with attachment at rev-2 {"hello.txt", revpos=2}
2. Document is updated multiple times on the server, goes to rev-4
3. Client attempts to add a new (conflicting) revision 3a, with ancestors rev-2a (with it's own attachment), rev-1.
4. When client attempts to push rev-3a with attachment stub {"hello.txt", revpos=2}. Should throw an error, since the revpos
of the attachment is later than the common ancestor (rev-1)
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
no_conflicts_enabled = params_from_base_test_setup["no_conflicts_enabled"]
if no_conflicts_enabled:
pytest.skip('--no-conflicts is not enabled, so skipping the test')
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(cluster_config, sg_conf)
topology = cluster_helper.get_cluster_topology(cluster_config)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
sg_db = "db"
bucket = "data-bucket"
log_info(
"Running 'test_attachment_revpos_when_ancestor_unavailable_active_revision_doesnt_share_ancestor'"
)
log_info("Using cbs_url: {}".format(cbs_url))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_url_admin: {}".format(sg_url_admin))
log_info("Using sg_db: {}".format(sg_db))
log_info("Using bucket: {}".format(bucket))
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
doc = document.create_doc(doc_id="doc_1",
content={"sample_key": "sample_val"},
channels=sg_user_channels)
doc_gen_1 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc,
auth=sg_user_session)
client.update_doc(url=sg_url,
db=sg_db,
doc_id=doc_gen_1["id"],
attachment_name="sample_text.txt",
auth=sg_user_session)
client.update_doc(url=sg_url,
db=sg_db,
doc_id=doc_gen_1["id"],
auth=sg_user_session)
client.update_doc(url=sg_url,
db=sg_db,
doc_id=doc_gen_1["id"],
auth=sg_user_session)
parent_rev_list = ["2-foo2", doc_gen_1["rev"]]
# Sync Gateway should error since it has no references attachment in its ancestors
with pytest.raises(HTTPError) as he:
client.add_conflict(url=sg_url,
db=sg_db,
doc_id=doc_gen_1["id"],
parent_revisions=parent_rev_list,
new_revision="3-foo3",
auth=sg_user_session)
assert he.value[0].startswith("400 Client Error: Bad Request for url: ")
def test_server_goes_down_sanity(params_from_base_test_setup):
"""
1. Start with a two node couchbase server cluster
2. Starting adding docs
3. Kill one of the server nodes and signal completion
4. Stop adding docs
5. Verify that that the expected docs are present and in the changes feed.
6. Start server again and add to cluster
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_version = get_sg_version(cluster_config)
if compare_versions(sg_version, '1.5') < 0:
pytest.skip("This test needs multiple URLs in the SG config, not supported by SG < 1.5")
cluster_helper = ClusterKeywords()
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf_path)
topology = cluster_helper.get_cluster_topology(cluster_config)
admin_sg = topology["sync_gateways"][0]["admin"]
sg_url = topology["sync_gateways"][0]["public"]
coucbase_servers = topology["couchbase_servers"]
cbs_one_url = coucbase_servers[0]
cbs_two_url = coucbase_servers[1]
log_info("Running: 'test_server_goes_down_sanity'")
log_info("cluster_config: {}".format(cluster_config))
log_info("admin_sg: {}".format(admin_sg))
log_info("sg_url: {}".format(sg_url))
log_info("cbs_one_url: {}".format(cbs_one_url))
log_info("cbs_two_url: {}".format(cbs_two_url))
sg_db = "db"
num_docs = 100
sg_user_name = "seth"
sg_user_password = "******"
channels = ["ABC", "CBS"]
client = MobileRestClient()
main_server = couchbaseserver.CouchbaseServer(cbs_one_url)
flakey_server = couchbaseserver.CouchbaseServer(cbs_two_url)
client.create_user(admin_sg, sg_db, sg_user_name, sg_user_password, channels=channels)
session = client.create_session(admin_sg, sg_db, sg_user_name)
# Stop second server
flakey_server.stop()
# Try to add 100 docs in a loop until all succeed, if the never do, fail with timeout
errors = num_docs
# Wait 30 seconds for auto failover
# (Minimum value suggested - http://docs.couchbase.com/admin/admin/Tasks/tasks-nodeFailover.html)
# + 15 seconds to add docs
timeout = 45
start = time.time()
successful_add = False
while not successful_add:
# Fail tests if all docs do not succeed before timeout
if (time.time() - start) > timeout:
# Bring server back up before failing the test
flakey_server.start()
main_server.rebalance_in(coucbase_servers, flakey_server)
raise TimeoutError("Failed to successfully put docs before timeout")
try:
docs = client.add_docs(url=sg_url, db=sg_db, number=num_docs, id_prefix=None, auth=session, channels=channels)
# If the above add doc does not throw, it was a successfull add.
successful_add = True
except requests.exceptions.HTTPError as he:
log_info("Failed to add docs: {}".format(he))
log_info("Seeing: {} errors".format(errors))
time.sleep(1)
assert len(docs) == 100
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=docs, auth=session)
try:
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=docs, auth=session, polling_interval=5)
except keywords.exceptions.TimeoutException:
# timeout verifying docs. Bring server back in to restore topology, then fail
# Failing due to https://github.com/couchbase/sync_gateway/issues/2197
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
raise keywords.exceptions.TimeoutException("Failed to get all changes")
# Test succeeded without timeout, bring server back into topology
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
# Make sure all docs were not added before server was
log_info("test_server_goes_down_sanity complete!")
def test_server_goes_down_rebuild_channels(params_from_base_test_setup):
"""
1. Start with a two node couchbase server cluster
2. Starting adding docs
3. Kill one of the server nodes and signal completion
4. Stop adding docs
5. Verify that that the expected docs are present and in the changes feed.
6. Start server again and add to cluster
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_version = get_sg_version(cluster_config)
if compare_versions(sg_version, '1.5') < 0:
pytest.skip("This test needs multiple URLs in the SG config, not supported by SG < 1.5")
cluster_helper = ClusterKeywords()
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf_path)
topology = cluster_helper.get_cluster_topology(cluster_config)
admin_sg = topology["sync_gateways"][0]["admin"]
sg_url = topology["sync_gateways"][0]["public"]
coucbase_servers = topology["couchbase_servers"]
cbs_one_url = coucbase_servers[0]
cbs_two_url = coucbase_servers[1]
log_info("Running: 'test_server_goes_down_sanity'")
log_info("cluster_config: {}".format(cluster_config))
log_info("admin_sg: {}".format(admin_sg))
log_info("sg_url: {}".format(sg_url))
log_info("cbs_one_url: {}".format(cbs_one_url))
log_info("cbs_two_url: {}".format(cbs_two_url))
sg_db = "db"
num_docs = 100
admin_user_info = userinfo.UserInfo(
name="admin",
password="******",
channels=["ABC"],
roles=[]
)
seth_user_info = userinfo.UserInfo(
name="seth",
password="******",
channels=["ABC"],
roles=[]
)
client = MobileRestClient()
main_server = couchbaseserver.CouchbaseServer(cbs_one_url)
flakey_server = couchbaseserver.CouchbaseServer(cbs_two_url)
admin_auth = client.create_user(
admin_sg,
sg_db,
admin_user_info.name,
admin_user_info.password,
channels=admin_user_info.channels
)
client.create_user(
admin_sg,
sg_db,
seth_user_info.name,
seth_user_info.password,
channels=seth_user_info.channels
)
seth_session = client.create_session(admin_sg, sg_db, seth_user_info.name)
# allow any user docs to make it to changes
initial_changes = client.get_changes(url=sg_url, db=sg_db, since=0, auth=seth_session)
# push docs from admin
docs = client.add_docs(
url=sg_url,
db=sg_db,
number=num_docs,
id_prefix=None,
channels=admin_user_info.channels,
auth=admin_auth
)
assert len(docs) == num_docs
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=docs, auth=seth_session)
changes_before_failover = client.get_changes(url=sg_url, db=sg_db, since=initial_changes["last_seq"], auth=seth_session)
assert len(changes_before_failover["results"]) == num_docs
# Stop server via 'service stop'
flakey_server.stop()
start = time.time()
while True:
# Fail tests if all docs do not succeed before timeout
if (time.time() - start) > 60:
# Bring server back up before failing the test
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
raise keywords.exceptions.TimeoutError("Failed to rebuild changes")
try:
# Poll until failover happens (~30 second)
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=docs, auth=seth_session)
# changes requests succeeded, exit loop
break
except requests.exceptions.HTTPError:
# Changes will fail until failover of the down server happens. Wait and try again.
log_info("/db/_changes failed due to server down. Retrying ...")
time.sleep(1)
# Verify no new changes
changes = client.get_changes(
url=sg_url,
db=sg_db,
since=changes_before_failover["last_seq"],
auth=seth_session,
feed="normal"
)
assert len(changes["results"]) == 0
# Check that all changes are intact from initial changes request
changes = client.get_changes(url=sg_url, db=sg_db, since=initial_changes["last_seq"], auth=seth_session)
assert len(changes["results"]) == num_docs
coucbase_servers = topology["couchbase_servers"]
# Test succeeded without timeout, bring server back into topology
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
def test_rebalance_sanity(params_from_base_test_setup):
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_version = get_sg_version(cluster_config)
if compare_versions(sg_version, '1.5') < 0:
pytest.skip("This test needs multiple URLs in the SG config, not supported by SG < 1.5")
cluster_helper = ClusterKeywords()
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf_path)
topology = cluster_helper.get_cluster_topology(cluster_config)
admin_sg_one = topology["sync_gateways"][0]["admin"]
sg_one_url = topology["sync_gateways"][0]["public"]
cluster_servers = topology["couchbase_servers"]
cbs_one_url = cluster_servers[0]
cbs_two_url = cluster_servers[1]
log_info("Running: 'test_distributed_index_rebalance_sanity'")
log_info("cluster_config: {}".format(cluster_config))
log_info("admin_sg: {}".format(admin_sg_one))
log_info("sg_url: {}".format(sg_one_url))
log_info("cbs_one_url: {}".format(cbs_one_url))
log_info("cbs_two_url: {}".format(cbs_two_url))
sg_db = "db"
num_docs = 100
num_updates = 100
sg_user_name = "seth"
sg_user_password = "******"
channels = ["ABC", "CBS"]
client = MobileRestClient()
cb_server = couchbaseserver.CouchbaseServer(cbs_one_url)
server_to_remove = couchbaseserver.CouchbaseServer(cbs_two_url)
client.create_user(admin_sg_one, sg_db, sg_user_name, sg_user_password, channels=channels)
session = client.create_session(admin_sg_one, sg_db, sg_user_name)
with concurrent.futures.ThreadPoolExecutor(5) as executor:
# Add docs to sg
log_info("Adding docs to sync_gateway")
docs = client.add_docs(sg_one_url, sg_db, num_docs, "test_doc", channels=channels, auth=session)
assert len(docs) == num_docs
# Start updating docs and rebalance out one CBS node
log_info("Updating docs on sync_gateway")
update_docs_task = executor.submit(client.update_docs, sg_one_url, sg_db, docs, num_updates, auth=session)
# Run rebalance in background
cb_server.rebalance_out(cluster_servers, server_to_remove)
updated_docs = update_docs_task.result()
log_info(updated_docs)
# Verify docs / revisions present
client.verify_docs_present(sg_one_url, sg_db, updated_docs, auth=session)
# Verify docs revisions in changes feed
client.verify_docs_in_changes(sg_one_url, sg_db, updated_docs, auth=session)
# Rebalance Server back in to the pool
cb_server.add_node(server_to_remove)
cb_server.rebalance_in(cluster_servers, server_to_remove)
def test_longpoll_awaken_via_sync_role(params_from_base_test_setup, sg_conf_name):
"""
Test that longpoll changes feed wakes up on role() in sync_function
The contrived sync_function below is used:
function(doc, oldDoc){
if(doc._id == "role_doc_0") {
console.log("granting_access!");
role(["adam", "traun", "andy"], "role:techno");
}
channel(doc, doc.channels);
}
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
cluster_topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
sg_admin_url = cluster_topology["sync_gateways"][0]["admin"]
sg_url = cluster_topology["sync_gateways"][0]["public"]
log_info("sg_conf: {}".format(sg_conf))
log_info("sg_admin_url: {}".format(sg_admin_url))
log_info("sg_url: {}".format(sg_url))
sg_db = "db"
techno_role = "techno"
techno_channel = "aphex"
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
client = MobileRestClient()
client.create_role(url=sg_admin_url, db=sg_db, name=techno_role, channels=[techno_channel])
admin_user_info = userinfo.UserInfo(name="admin", password="******", channels=[], roles=[techno_role])
adam_user_info = userinfo.UserInfo(name="adam", password="******", channels=[], roles=[])
traun_user_info = userinfo.UserInfo(name="traun", password="******", channels=[], roles=[])
andy_user_info = userinfo.UserInfo(name="andy", password="******", channels=[], roles=[])
admin_auth = client.create_user(url=sg_admin_url, db=sg_db,
name=admin_user_info.name, password=admin_user_info.password, roles=adam_user_info.roles)
adam_auth = client.create_user(url=sg_admin_url, db=sg_db,
name=adam_user_info.name, password=adam_user_info.password)
traun_auth = client.create_user(url=sg_admin_url, db=sg_db,
name=traun_user_info.name, password=traun_user_info.password)
andy_auth = client.create_user(url=sg_admin_url, db=sg_db,
name=andy_user_info.name, password=andy_user_info.password)
client.add_docs(url=sg_url, db=sg_db, number=1, id_prefix="techno_doc", channels=[techno_channel], auth=admin_auth)
# Get starting sequence of docs, use the last seq to progress past any _user docs.
adam_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=adam_auth)
traun_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=traun_auth)
andy_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=andy_auth)
with concurrent.futures.ProcessPoolExecutor() as ex:
# Start changes feed for 3 users from latest last_seq
adam_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=adam_changes["last_seq"], timeout=10, auth=adam_auth)
traun_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=traun_changes["last_seq"], timeout=10, auth=traun_auth)
andy_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=andy_changes["last_seq"], timeout=10, auth=andy_auth)
# Wait for changes feed to notice there are no changes and enter wait. 2 seconds should be more than enough
time.sleep(2)
# Make sure the changes future is still running and has not exited due to any new changes, the feed should be caught up
# and waiting
assert not adam_changes_task.done()
assert not traun_changes_task.done()
assert not andy_changes_task.done()
# Grant adam, traun and andy access to the "NATGEO" channel
client.add_docs(url=sg_url, db=sg_db, number=1, id_prefix="role_doc", channels=[], auth=admin_auth)
# Changes feed should wake up with the natgeo_0 doc
adam_changes = adam_changes_task.result()
assert len(adam_changes["results"]) == 1
assert adam_changes["results"][0]["id"] == "techno_doc_0"
assert adam_changes["results"][0]["changes"][0]["rev"].startswith("1-")
traun_changes = traun_changes_task.result()
assert len(traun_changes["results"]) == 1
assert traun_changes["results"][0]["id"] == "techno_doc_0"
assert traun_changes["results"][0]["changes"][0]["rev"].startswith("1-")
andy_changes = andy_changes_task.result()
assert len(andy_changes["results"]) == 1
assert andy_changes["results"][0]["id"] == "techno_doc_0"
assert andy_changes["results"][0]["changes"][0]["rev"].startswith("1-")
# Assert that the changes are caught up and should recieve no new changes from last_seq
# Test for https://github.com/couchbase/sync_gateway/issues/2186
adam_changes = client.get_changes(url=sg_url, db=sg_db, since=adam_changes["last_seq"], auth=adam_auth, timeout=1)
assert len(adam_changes["results"]) == 0
traun_changes = client.get_changes(url=sg_url, db=sg_db, since=traun_changes["last_seq"], auth=traun_auth, timeout=1)
assert len(traun_changes["results"]) == 0
andy_changes = client.get_changes(url=sg_url, db=sg_db, since=andy_changes["last_seq"], auth=andy_auth, timeout=1)
assert len(andy_changes["results"]) == 0
def test_stale_revision_should_not_be_in_the_index(setup_client_syncgateway_test):
"""original ticket: https://github.com/couchbase/couchbase-lite-android/issues/855
scenario:
1. Running sync_gateway
2. Create database and starts both push and pull replicators through client REST API
3. Create two or more views through client REST API
4. Add doc, and verify doc is index with current revision through client REST API
5. Make sure document is pushed to sync gateway through sync gateway REST API
6. Update doc with sync gateway (not client side) through sync gateway REST API
7. Make sure updated document is pull replicated to client through client REST API
8. Make sure updated document is indexed through client REST API
9. Make sure stale revision is deleted from index. through client REST API
10. Pass criteria
"""
cluster_config = setup_client_syncgateway_test["cluster_config"]
sg_mode = setup_client_syncgateway_test["sg_mode"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_url = setup_client_syncgateway_test["sg_url"]
sg_admin_url = setup_client_syncgateway_test["sg_admin_url"]
num_docs = 10
num_revs = 100
d_doc_name = "dd"
sg_db = "db"
sg_user_name = "sg_user"
sg_config = sync_gateway_config_path_for_mode("listener_tests/listener_tests", sg_mode)
c = cluster.Cluster(config=cluster_config)
c.reset(sg_config_path=sg_config)
log_info("Running 'test_stale_revision_should_not_be_in_the_index'")
log_info("ls_url: {}".format(ls_url))
log_info("sg_admin_url: {}".format(sg_admin_url))
log_info("sg_url: {}".format(sg_url))
log_info("num_docs: {}".format(num_docs))
log_info("num_revs: {}".format(num_revs))
client = MobileRestClient()
sg_user_channels = ["NBC"]
client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels)
sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name)
view = """{
"language" : "javascript",
"views" : {
"content_view" : {
"map" : "function(doc, meta) { if (doc.content) { emit(doc._id, doc._rev); } }"
},
"update_view" : {
"map" : "function(doc, meta) { emit(doc.updates, null); }"
}
}
}"""
ls_db = client.create_database(url=ls_url, name="ls_db")
# Setup continuous push / pull replication from ls_db1 to sg_db
client.start_replication(
url=ls_url,
continuous=True,
from_db=ls_db,
to_url=sg_admin_url, to_db=sg_db
)
client.start_replication(
url=ls_url,
continuous=True,
from_url=sg_admin_url, from_db=sg_db,
to_db=ls_db
)
design_doc_id = client.add_design_doc(url=ls_url, db=ls_db, name=d_doc_name, doc=view)
client.get_doc(url=ls_url, db=ls_db, doc_id=design_doc_id)
doc_body = document.create_doc(doc_id="doc_1", content={"hi": "I should be in the view"}, channels=sg_user_channels)
log_info(doc_body)
doc_body_2 = document.create_doc(doc_id="doc_2", channels=sg_user_channels)
doc = client.add_doc(url=ls_url, db=ls_db, doc=doc_body)
doc_2 = client.add_doc(url=ls_url, db=ls_db, doc=doc_body_2)
content_view_rows = client.get_view(url=ls_url, db=ls_db, design_doc_name=d_doc_name, view_name="content_view")
client.verify_view_row_num(view_response=content_view_rows, expected_num_rows=1)
update_view_rows = client.get_view(url=ls_url, db=ls_db, design_doc_name=d_doc_name, view_name="update_view")
client.verify_view_row_num(view_response=update_view_rows, expected_num_rows=2)
expected_docs_list = [doc, doc_2]
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=expected_docs_list, auth=sg_session)
updated_doc = client.update_doc(url=sg_url, db=sg_db, doc_id=doc["id"], number_updates=10, auth=sg_session)
client.verify_docs_present(url=ls_url, db=ls_db, expected_docs=updated_doc)
content_view_rows_2 = client.get_view(url=ls_url, db=ls_db, design_doc_name=d_doc_name, view_name="content_view")
client.verify_view_row_num(view_response=content_view_rows_2, expected_num_rows=1)
client.verify_view_contains_keys(view_response=content_view_rows_2, keys=doc["id"])
client.verify_view_contains_values(view_response=content_view_rows_2, values=updated_doc["rev"])
def test_backfill_channels_oneshot_limit_changes(params_from_base_test_setup, sg_conf_name, grant_type):
cluster_config = params_from_base_test_setup["cluster_config"]
topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
log_info("grant_type: {}".format(grant_type))
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster = Cluster(cluster_config)
cluster.reset(sg_conf)
client = MobileRestClient()
admin_user_info = userinfo.UserInfo("admin", "pass", channels=["A"], roles=[])
user_b_user_info = userinfo.UserInfo("USER_B", "pass", channels=["B"], roles=[])
# Create users / sessions
client.create_user(url=sg_admin_url, db=sg_db,
name=admin_user_info.name, password=admin_user_info.password, channels=admin_user_info.channels)
client.create_user(url=sg_admin_url, db=sg_db,
name=user_b_user_info.name, password=user_b_user_info.password, channels=user_b_user_info.channels)
admin_session = client.create_session(url=sg_admin_url, db=sg_db, name=admin_user_info.name, password=admin_user_info.password)
user_b_session = client.create_session(url=sg_admin_url, db=sg_db, name=user_b_user_info.name, password=user_b_user_info.password)
# Create 50 "A" channel docs
a_docs = client.add_docs(url=sg_url, db=sg_db, number=50, id_prefix=None, auth=admin_session, channels=["A"])
assert len(a_docs) == 50
b_docs = client.add_docs(url=sg_url, db=sg_db, number=1, id_prefix="b_doc", auth=user_b_session, channels=["B"])
assert len(b_docs) == 1
user_doc = {"id": "_user/USER_B", "rev": None}
b_docs.append(user_doc)
# Loop until user_b sees b_doc_0 doc and _user/USER_B doc
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=b_docs, auth=user_b_session, strict=True)
# Get last_seq for user_b
user_b_changes = client.get_changes(url=sg_url, db=sg_db, since=0, auth=user_b_session, feed="normal")
# Grant access to channel "A"
if grant_type == "CHANNEL-REST":
log_info("Granting user access to channel A via Admin REST user update")
# Grant via update to user in Admin API
client.update_user(url=sg_admin_url, db=sg_db,
name=user_b_user_info.name, channels=["A", "B"])
elif grant_type == "CHANNEL-SYNC":
log_info("Granting user access to channel A sync function access()")
# Grant via access() in sync_function, then id 'channel_access' will trigger an access(doc.users, doc.channels)
access_doc = document.create_doc("channel_access", channels=["A"])
access_doc["users"] = ["USER_B"]
client.add_doc(url=sg_url, db=sg_db, doc=access_doc, auth=admin_session)
elif grant_type == "ROLE-REST":
log_info("Granting user access to channel A via Admin REST role grant")
# Create role with channel A
client.create_role(url=sg_admin_url, db=sg_db, name="channel-A-role", channels=["A"])
client.update_user(url=sg_admin_url, db=sg_db, name="USER_B", roles=["channel-A-role"])
elif grant_type == "ROLE-SYNC":
log_info("Granting user access to channel A via sync function role() grant")
# Create role with channel A
client.create_role(url=sg_admin_url, db=sg_db, name="channel-A-role", channels=["A"])
# Grant via role() in sync_function, then id 'role_access' will trigger an role(doc.users, doc.roles)
role_access_doc = document.create_doc("role_access")
role_access_doc["users"] = ["USER_B"]
role_access_doc["roles"] = ["role:channel-A-role"]
client.add_doc(sg_url, db=sg_db, doc=role_access_doc, auth=admin_session)
else:
pytest.fail("Unsupported grant_type!!!!")
# Create a dictionary keyed on doc id for all of channel A docs
ids_and_revs_from_a_docs = {doc["id"]: doc["rev"] for doc in a_docs}
assert len(ids_and_revs_from_a_docs.keys()) == 50
log_info("Doing 3, 1 shot changes with limit and last seq!")
# Issue 3 oneshot changes with a limit of 20
#################
# Changes Req #1
#################
user_b_changes_after_grant_one = client.get_changes(url=sg_url, db=sg_db,
since=user_b_changes["last_seq"], auth=user_b_session, feed="normal", limit=20)
# User B shoud have recieved 20 docs due to limit
assert len(user_b_changes_after_grant_one["results"]) == 20
for doc in user_b_changes_after_grant_one["results"]:
# cross off keys found from 'a_docs' dictionary
del ids_and_revs_from_a_docs[doc["id"]]
#################
# Changes Req #2
#################
user_b_changes_after_grant_two = client.get_changes(url=sg_url, db=sg_db,
since=user_b_changes_after_grant_one["last_seq"],
auth=user_b_session, feed="normal", limit=20)
# User B shoud have recieved 20 docs due to limit
assert len(user_b_changes_after_grant_two["results"]) == 20
for doc in user_b_changes_after_grant_two["results"]:
# cross off keys found from 'a_docs' dictionary
del ids_and_revs_from_a_docs[doc["id"]]
#################
# Changes Req #3
#################
user_b_changes_after_grant_three = client.get_changes(url=sg_url, db=sg_db,
since=user_b_changes_after_grant_two["last_seq"],
auth=user_b_session, feed="normal", limit=20)
# User B shoud have recieved 20 docs due to limit
assert len(user_b_changes_after_grant_three["results"]) == 10
for doc in user_b_changes_after_grant_three["results"]:
# cross off keys found from 'a_docs' dictionary
del ids_and_revs_from_a_docs[doc["id"]]
# Make sure all the docs have been crossed out
assert len(ids_and_revs_from_a_docs) == 0
#################
# Changes Req #4
#################
user_b_changes_after_grant_four = client.get_changes(url=sg_url, db=sg_db,
since=user_b_changes_after_grant_three["last_seq"],
auth=user_b_session, feed="normal", limit=20)
# Changes should be caught up and there should be no results
assert len(user_b_changes_after_grant_four["results"]) == 0
def test_attachment_revpos_when_ancestor_unavailable(params_from_base_test_setup, sg_conf_name):
"""
Creates a document with an attachment, then updates that document so that
the body of the revision that originally pushed the document is no
longer available. Add a new revision that's not a child of the
active revision, and validate that it's uploaded successfully.
Example:
1. Document is created with attachment at rev-1
2. Document is updated (strip digests and length, only put revpos & stub) multiple times on the server, goes to rev-4
3. Client attempts to add a new (conflicting) revision 2, with parent rev-1.
4. If the body of rev-1 is no longer available on the server (temporary backup of revision has expired, and is no longer stored
in the in-memory rev cache), we were throwing an error to client
because we couldn't verify based on the _attachments property in rev-1.
5. In this scenario, before returning error, we are now checking if the active revision has a common ancestor with the incoming revision.
If so, we can validate any revpos values equal to or earlier than the common ancestor against the active revision
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(cluster_config, sg_conf)
topology = cluster_helper.get_cluster_topology(cluster_config)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
sg_db = "db"
bucket = "data-bucket"
log_info("Running 'test_attachment_revpos_when_ancestor_unavailable'")
log_info("Using cbs_url: {}".format(cbs_url))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_url_admin: {}".format(sg_url_admin))
log_info("Using sg_db: {}".format(sg_db))
log_info("Using bucket: {}".format(bucket))
channels_list = ["ABC"]
client = MobileRestClient()
sg_util = SyncGateway()
cb_server = CouchbaseServer(cbs_url)
user1 = client.create_user(url=sg_url_admin, db=sg_db, name="user1", password="******", channels=channels_list)
doc_with_att = document.create_doc(doc_id="att_doc", content={"sample_key": "sample_val"}, attachment_name="sample_text.txt", channels=channels_list)
doc_gen_1 = client.add_doc(url=sg_url, db=sg_db, doc=doc_with_att, auth=user1)
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_gen_1["id"], number_updates=10, auth=user1)
# Clear cached rev doc bodys from server and cycle sync_gateway
sg_util.stop_sync_gateway(cluster_config=cluster_config, url=sg_url)
cb_server.delete_couchbase_server_cached_rev_bodies(bucket=bucket)
sg_util.start_sync_gateway(cluster_config=cluster_config, url=sg_url, config=sg_conf)
client.add_conflict(
url=sg_url, db=sg_db,
doc_id=doc_gen_1["id"],
parent_revisions=doc_gen_1["rev"],
new_revision="2-foo",
auth=user1
)
def test_take_all_sgaccels_down(params_from_base_test_setup, sg_conf):
"""
Scenario that takes all sync_gateway accel nodes offline during doc load.
After bring the nodes back online during load, the reshard of the DCP feed is verified.
The changes feed is verified that all docs show up.
1. Start doc load (1000 doc)
2. Take all sg_accel nodes down in parallel
3. Verify node are down
4. Wait for doc adds to complete, store "doc_push_result_1"
5. Verify "doc_push_result_1" docs added
6. Start doc load (1000 docs)
7. Wait for 5. to complete, store "doc_push_result_2"
8. Verify "doc_push_result_2" docs added
9. Start another doc load (1000 docs)
10. Bring up nodes in parallel
11. poll on p-index reshard
12. Wait for 9. to complete, store "doc_push_result_3"
13. Verify "doc_push_result_3" docs added
14. Verify "doc_push_result_1" + "doc_push_result_2" + "doc_push_result_3" show up in _changes feed
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_dcp_reshard_single_sg_accel_goes_down_and_up'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_util = ClusterKeywords()
topology = cluster_util.get_cluster_topology(cluster_conf)
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
num_docs = 1000
client = MobileRestClient()
doc_pusher_user_info = userinfo.UserInfo("doc_pusher",
"pass",
channels=["A"],
roles=[])
doc_pusher_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=doc_pusher_user_info.name,
password=doc_pusher_user_info.password,
channels=doc_pusher_user_info.channels)
a_user_info = userinfo.UserInfo("a_user", "pass", channels=["A"], roles=[])
client.create_user(url=sg_admin_url,
db=sg_db,
name=a_user_info.name,
password=a_user_info.password,
channels=a_user_info.channels)
a_user_session = client.create_session(url=sg_admin_url,
db=sg_db,
name=a_user_info.name,
password=a_user_info.password)
# Shutdown all accel nodes in parallel
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as ex:
# Start adding docs
docs_1 = document.create_docs(None,
num_docs,
channels=doc_pusher_user_info.channels)
docs_1_task = ex.submit(client.add_bulk_docs,
url=sg_url,
db=sg_db,
docs=docs_1,
auth=doc_pusher_auth)
# Take down all access nodes
log_info("Shutting down sg_accels: [{}, {}, {}] ...".format(
cluster.sg_accels[0], cluster.sg_accels[1], cluster.sg_accels[2]))
sg_accel_down_task_1 = ex.submit(cluster.sg_accels[0].stop)
sg_accel_down_task_2 = ex.submit(cluster.sg_accels[1].stop)
sg_accel_down_task_3 = ex.submit(cluster.sg_accels[2].stop)
assert sg_accel_down_task_1.result() == 0
assert sg_accel_down_task_2.result() == 0
assert sg_accel_down_task_3.result() == 0
# Block until bulk_docs is complete
doc_push_result_1 = docs_1_task.result()
assert len(doc_push_result_1) == num_docs
client.verify_docs_present(url=sg_url,
db=sg_db,
expected_docs=doc_push_result_1,
auth=doc_pusher_auth)
# Load sync_gateway with another batch of docs while the sg_accel nodes are offline
docs_2_bodies = document.create_docs(
None, num_docs, channels=doc_pusher_user_info.channels)
docs_push_result_2 = client.add_bulk_docs(url=sg_url,
db=sg_db,
docs=docs_2_bodies,
auth=doc_pusher_auth)
assert len(docs_push_result_2) == num_docs
client.verify_docs_present(url=sg_url,
db=sg_db,
expected_docs=docs_push_result_2,
auth=doc_pusher_auth)
# Start loading Sync Gateway with another set of docs while bringing the sg_accel nodes online
docs_3 = document.create_docs(None,
num_docs,
channels=doc_pusher_user_info.channels)
docs_3_task = ex.submit(client.add_bulk_docs,
url=sg_url,
db=sg_db,
docs=docs_3,
auth=doc_pusher_auth)
# Bring all the sg_accel nodes back up
# Take down all access nodes
log_info("Starting sg_accels: [{}, {}, {}] ...".format(
cluster.sg_accels[0], cluster.sg_accels[1], cluster.sg_accels[2]))
sg_accel_up_task_1 = ex.submit(cluster.sg_accels[0].start, sg_conf)
sg_accel_up_task_2 = ex.submit(cluster.sg_accels[1].start, sg_conf)
sg_accel_up_task_3 = ex.submit(cluster.sg_accels[2].start, sg_conf)
assert sg_accel_up_task_1.result() == 0
assert sg_accel_up_task_2.result() == 0
assert sg_accel_up_task_3.result() == 0
# Wait for pindex to reshard correctly
assert cluster.validate_cbgt_pindex_distribution_retry(3)
# Block until second bulk_docs is complete
doc_push_result_3 = docs_3_task.result()
assert len(doc_push_result_3) == num_docs
client.verify_docs_present(url=sg_url,
db=sg_db,
expected_docs=doc_push_result_3,
auth=doc_pusher_auth)
# Combine the 3 push results and make sure the changes propagate to a_user
# a_user has access to the doc's channel.
log_info("Verifying all the changes show up for 'a_user' ...")
all_docs = doc_push_result_1 + docs_push_result_2 + doc_push_result_3
client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=all_docs,
auth=a_user_session,
polling_interval=2)
def test_attachment_revpos_when_ancestor_unavailable_active_revision_doesnt_share_ancestor(params_from_base_test_setup, sg_conf_name):
"""
Creates a document with an attachment, then updates that document so that
the body of the revision that originally pushed the document is no
longer available. Add a new revision that's not a child of the
active revision, and validate that it's uploaded successfully.
Example:
1. Document is created with no attachment at rev-1
2. Server adds revision with attachment at rev-2 {"hello.txt", revpos=2}
2. Document is updated multiple times on the server, goes to rev-4
3. Client attempts to add a new (conflicting) revision 3a, with ancestors rev-2a (with it's own attachment), rev-1.
4. When client attempts to push rev-3a with attachment stub {"hello.txt", revpos=2}. Should throw an error, since the revpos
of the attachment is later than the common ancestor (rev-1)
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(cluster_config, sg_conf)
topology = cluster_helper.get_cluster_topology(cluster_config)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
sg_db = "db"
bucket = "data-bucket"
log_info("Running 'test_attachment_revpos_when_ancestor_unavailable_active_revision_doesnt_share_ancestor'")
log_info("Using cbs_url: {}".format(cbs_url))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_url_admin: {}".format(sg_url_admin))
log_info("Using sg_db: {}".format(sg_db))
log_info("Using bucket: {}".format(bucket))
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
doc = document.create_doc(doc_id="doc_1", content={"sample_key": "sample_val"}, channels=sg_user_channels)
doc_gen_1 = client.add_doc(url=sg_url, db=sg_db, doc=doc, auth=sg_user_session)
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_gen_1["id"], attachment_name="sample_text.txt", auth=sg_user_session)
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_gen_1["id"], auth=sg_user_session)
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_gen_1["id"], auth=sg_user_session)
parent_rev_list = ["2-foo2", doc_gen_1["rev"]]
# Sync Gateway should error since it has no references attachment in its ancestors
with pytest.raises(HTTPError) as he:
client.add_conflict(
url=sg_url,
db=sg_db,
doc_id=doc_gen_1["id"],
parent_revisions=parent_rev_list,
new_revision="3-foo3",
auth=sg_user_session
)
assert he.value[0].startswith("400 Client Error: Bad Request for url: ")
def test_non_winning_revisions(params_from_base_test_setup, sg_conf_name):
""" Add non-winning revisions to the revision tree and ensure
that the changes feed returns the correct revisions
Steps:
- Create a doc
- Add 5 revs
- changes, assert rev starts with "6-" from 0, store "last_seq_1"
- Create a conflict off first revision ("2-foo") (POST docs, new_edits == false)
- changes, assert rev starts with "6-" from "last_seq_1", store "last_seq_2"
- changes, assert rev starts with "6-" from 0
- Add a "3-foo" rev with rev "2-foo" as parent
- changes, assert rev starts with "6-" from "last_seq_2", store "last_seq_3"
- changes, assert rev starts with "6-" from 0
- add tombstone rev as child of "6-" i.e. issue delete on ("6-")
- changes, assert rev starts with "3-foo" from "last_seq_3"
- changes, assert rev starts with "3-foo" from 0
"""
cluster_config = params_from_base_test_setup["cluster_config"]
topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
c = cluster.Cluster(cluster_config)
c.reset(sg_conf)
client = MobileRestClient()
seth_user_info = userinfo.UserInfo(
name="seth",
password="******",
channels=["NATGEO"],
roles=[]
)
seth_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=seth_user_info.name,
password=seth_user_info.password,
channels=seth_user_info.channels
)
test_doc_body = document.create_doc(doc_id="test_doc", channels=seth_user_info.channels)
rev_gen_1_doc = client.add_doc(url=sg_url, db=sg_db, doc=test_doc_body, auth=seth_auth)
rev_gen_6_doc = client.update_doc(url=sg_url, db=sg_db, doc_id=rev_gen_1_doc["id"], number_updates=5, auth=seth_auth)
assert rev_gen_6_doc["rev"].startswith("6-")
# Get changes until rev generation 6 document shows up
start = time.time()
last_seq = 0
while True:
if time.time() - start > keywords.constants.CLIENT_REQUEST_TIMEOUT:
raise keywords.exceptions.TimeoutError("Wait for Replication Status Idle: TIMEOUT")
changes_1 = client.get_changes(url=sg_url, db=sg_db, since=last_seq, auth=seth_auth, skip_user_docs=True)
last_seq = changes_1["last_seq"]
# break when expected rev shows up in changes feed
if changes_1["results"] and changes_1["results"][0]["changes"][0]["rev"].startswith("6-"):
break
assert len(changes_1["results"]) == 1
assert changes_1["results"][0]["id"] == "test_doc"
assert changes_1["results"][0]["changes"][0]["rev"].startswith("6-")
# Create a conflict off of rev one
rev_gen_2_doc_conflict = client.add_conflict(
url=sg_url,
db=sg_db,
doc_id=rev_gen_1_doc["id"],
parent_revisions=rev_gen_1_doc["rev"],
new_revision="2-foo",
auth=seth_auth
)
assert rev_gen_2_doc_conflict["id"] == "test_doc"
assert rev_gen_2_doc_conflict["rev"] == "2-foo"
# Issue changes since changes_1 last_seq above
changes_2 = client.get_changes(url=sg_url, db=sg_db, since=changes_1["last_seq"], auth=seth_auth)
assert len(changes_2["results"]) == 1
assert changes_2["results"][0]["id"] == "test_doc"
assert changes_2["results"][0]["changes"][0]["rev"].startswith("6-")
# Issue changes since 0, strip user doc and make sure the doc is still the '6-' rev
changes_from_0_one = client.get_changes(url=sg_url, db=sg_db, since=0, auth=seth_auth, skip_user_docs=True)
assert len(changes_from_0_one["results"]) == 1
assert changes_from_0_one["results"][0]["id"] == "test_doc"
assert changes_from_0_one["results"][0]["changes"][0]["rev"].startswith("6-")
# Create a 3-foo rev with 2-foo as the parent
rev_gen_3_doc_conflict = client.add_conflict(
url=sg_url,
db=sg_db,
doc_id=rev_gen_2_doc_conflict["id"],
parent_revisions=rev_gen_2_doc_conflict["rev"],
new_revision="3-foo",
auth=seth_auth
)
assert rev_gen_3_doc_conflict["id"] == "test_doc"
assert rev_gen_3_doc_conflict["rev"] == "3-foo"
# Issue changes since changes_2 last_seq above
changes_3 = client.get_changes(url=sg_url, db=sg_db, since=changes_2["last_seq"], auth=seth_auth)
assert len(changes_3["results"]) == 1
assert changes_3["results"][0]["id"] == "test_doc"
assert changes_3["results"][0]["changes"][0]["rev"].startswith("6-")
# Issue changes since 0, strip user doc and make sure the doc is still the '6-' rev
changes_from_0_two = client.get_changes(url=sg_url, db=sg_db, since=0, auth=seth_auth, skip_user_docs=True)
assert len(changes_from_0_two["results"]) == 1
assert changes_from_0_two["results"][0]["id"] == "test_doc"
assert changes_from_0_two["results"][0]["changes"][0]["rev"].startswith("6-")
# Delete test_doc at rev 6-*
client.delete_doc(url=sg_url, db=sg_db, doc_id=rev_gen_6_doc["id"], rev=rev_gen_6_doc["rev"], auth=seth_auth)
# Issue changes since changes_3 last_seq above
changes_4 = client.get_changes(url=sg_url, db=sg_db, since=changes_3["last_seq"], auth=seth_auth)
assert len(changes_4["results"]) == 1
assert changes_4["results"][0]["id"] == "test_doc"
assert changes_4["results"][0]["changes"][0]["rev"] == "3-foo"
# Issue a oneshot changes since changes_4 last_seq and assert no results are returned
changes_5 = client.get_changes(url=sg_url, db=sg_db, since=changes_4["last_seq"], feed="normal", auth=seth_auth)
assert len(changes_5["results"]) == 0
def test_upgrade_cbl(setup_client_syncgateway_test):
"""
1. install previous version of couchbase lite.
2. Create docs in the lite.
3. upgrade to latest version of couchbase lite.
4. Verfiy docs still exits and accessible
"""
sg_db = "db"
ls_db = "ls_db"
channels = ["auto"]
num_docs = 30000
cluster_config = setup_client_syncgateway_test["cluster_config"]
sg_mode = setup_client_syncgateway_test["sg_mode"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_one_admin = setup_client_syncgateway_test["sg_admin_url"]
sg_one_public = setup_client_syncgateway_test["sg_url"]
liteserv = setup_client_syncgateway_test["liteserv"]
device_enabled = setup_client_syncgateway_test["device_enabled"]
liteserv_platform = setup_client_syncgateway_test["liteserv_platform"]
liteserv_version = setup_client_syncgateway_test["liteserv_version"]
if liteserv_platform.lower() == "android" or liteserv_platform.lower() == "net-msft" or device_enabled:
# There is a signature match issue on Android and older version ios does not have app for devices
pytest.skip('upgrade lite serv app does not work on Android and there is no app for ios device created' +
' for older version, so skipping the test')
sg_config = sync_gateway_config_path_for_mode("listener_tests/listener_tests", sg_mode)
c = cluster.Cluster(config=cluster_config)
c.reset(sg_config_path=sg_config)
log_info("ls_url: {}".format(ls_url))
log_info("sg_one_admin: {}".format(sg_one_admin))
log_info("sg_one_public: {}".format(sg_one_public))
test_name = "test_upgrade"
# Downloading 1.4.0-3, we are not paramatrizing here for older version as this is not going to work
# so hardcodig now rather than adding new parameter
log_info("Downloading older version of LiteServ ...")
liteserv.download(version_build="1.4.0-3")
# Install LiteServ
if device_enabled and liteserv_platform == "ios":
liteserv.stop()
liteserv.install_device()
ls_url = liteserv.start_device("{}/logs/{}-{}-{}.txt".format(RESULTS_DIR, type(liteserv).__name__,
test_name, datetime.datetime.now()))
else:
liteserv.stop()
liteserv.install()
log_info("Liteserv starts now ....")
ls_url = liteserv.start("{}/logs/{}-{}-{}.txt".format(RESULTS_DIR, type(liteserv).__name__,
test_name, datetime.datetime.now()))
client = MobileRestClient()
client.create_user(sg_one_admin, sg_db, "test", password="******", channels=channels)
session = client.create_session(sg_one_admin, sg_db, "test")
client.create_database(url=ls_url, name=ls_db)
# Create 'num_docs' docs on LiteServ
docs = client.add_docs(url=ls_url, db=ls_db, number=num_docs, id_prefix="seeded_doc",
generator="four_k", channels=channels)
assert len(docs) == num_docs
client.verify_docs_present(url=ls_url, db=ls_db, expected_docs=docs, timeout=240)
with ThreadPoolExecutor(max_workers=2) as tpe:
download_cbl_tpe = tpe.submit(download_install, liteserv, liteserv_version)
client.start_replication(url=ls_url, continuous=True, from_db=ls_db, to_url=sg_one_admin,
to_db=sg_db)
download_cbl_tpe.result()
log_info("download is done")
liteserv.stop()
ls_url = liteserv.start("{}/logs/{}-{}-{}.txt".format(RESULTS_DIR, type(liteserv).__name__,
test_name, datetime.datetime.now()))
client.start_replication(url=ls_url, continuous=True, from_db=ls_db, to_url=sg_one_admin,
to_db=sg_db)
client.verify_docs_present(url=ls_url, db=ls_db, expected_docs=docs, timeout=240)
client.verify_docs_present(url=sg_one_public, db=sg_db, expected_docs=docs, auth=session, timeout=600)
def test_backfill_channels_oneshot_changes(params_from_base_test_setup,
sg_conf_name, grant_type):
"""
Test that checks that docs are backfilled for one shot changes for a access grant (via REST or SYNC)
CHANNEL-REST = Channel is granted to user via REST
CHANNEL-SYNC = Channel is granted to user via sync function access()
ROLE-REST = Role is granted to user via REST
ROLE-SYNC = Role is granted to user via sync function role()
CHANNEL-TO-ROLE-REST = Channel is added to existing role via REST
CHANNEL-TO-ROLE-SYNC = Channel is added to existing role via sync access()
"""
cluster_config = params_from_base_test_setup["cluster_config"]
topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
log_info("grant_type: {}".format(grant_type))
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster = Cluster(cluster_config)
cluster.reset(sg_conf)
client = MobileRestClient()
admin_user_info = userinfo.UserInfo("admin",
"pass",
channels=["A"],
roles=[])
if grant_type == "CHANNEL-TO-ROLE-REST" or grant_type == "CHANNEL-TO-ROLE-SYNC":
client.create_role(url=sg_admin_url,
db=sg_db,
name="empty_role",
channels=[])
user_b_user_info = userinfo.UserInfo("USER_B",
"pass",
channels=["B"],
roles=["empty_role"])
else:
user_b_user_info = userinfo.UserInfo("USER_B",
"pass",
channels=["B"],
roles=[])
# Create users / sessions
client.create_user(url=sg_admin_url,
db=sg_db,
name=admin_user_info.name,
password=admin_user_info.password,
channels=admin_user_info.channels)
client.create_user(url=sg_admin_url,
db=sg_db,
name=user_b_user_info.name,
password=user_b_user_info.password,
channels=user_b_user_info.channels,
roles=user_b_user_info.roles)
admin_session = client.create_session(url=sg_admin_url,
db=sg_db,
name=admin_user_info.name,
password=admin_user_info.password)
user_b_session = client.create_session(url=sg_admin_url,
db=sg_db,
name=user_b_user_info.name,
password=user_b_user_info.password)
# Create 50 "A" channel docs
a_docs = client.add_docs(url=sg_url,
db=sg_db,
number=50,
id_prefix=None,
auth=admin_session,
channels=["A"])
assert len(a_docs) == 50
b_docs = client.add_docs(url=sg_url,
db=sg_db,
number=1,
id_prefix="b_doc",
auth=user_b_session,
channels=["B"])
assert len(b_docs) == 1
user_doc = {"id": "_user/USER_B", "rev": None}
b_docs.append(user_doc)
# Loop until admin user sees docs in changes
client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=a_docs,
auth=admin_session)
# Loop until user_b sees b_doc_0 doc and _user/USER_B doc
client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=b_docs,
auth=user_b_session,
strict=True)
# Get last_seq for user_b
user_b_changes = client.get_changes(url=sg_url,
db=sg_db,
since=0,
auth=user_b_session,
feed="normal")
# Grant access to channel "A"
if grant_type == "CHANNEL-REST":
log_info(
"Granting user access to channel A via Admin REST user update")
# Grant via update to user in Admin API
client.update_user(url=sg_admin_url,
db=sg_db,
name=user_b_user_info.name,
channels=["A", "B"])
elif grant_type == "CHANNEL-SYNC":
log_info("Granting user access to channel A sync function access()")
# Grant via access() in sync_function, then id 'channel_access' will trigger an access(doc.users, doc.channels)
access_doc = document.create_doc("channel_access", channels=["A"])
access_doc["users"] = ["USER_B"]
client.add_doc(url=sg_url,
db=sg_db,
doc=access_doc,
auth=admin_session)
elif grant_type == "ROLE-REST":
log_info("Granting user access to channel A via Admin REST role grant")
# Create role with channel A
client.create_role(url=sg_admin_url,
db=sg_db,
name="channel-A-role",
channels=["A"])
client.update_user(url=sg_admin_url,
db=sg_db,
name="USER_B",
channels=["B"],
roles=["channel-A-role"])
elif grant_type == "ROLE-SYNC":
log_info(
"Granting user access to channel A via sync function role() grant")
# Create role with channel A
client.create_role(url=sg_admin_url,
db=sg_db,
name="channel-A-role",
channels=["A"])
# Grant via role() in sync_function, then id 'role_access' will trigger an role(doc.users, doc.roles)
role_access_doc = document.create_doc("role_access")
role_access_doc["users"] = ["USER_B"]
role_access_doc["roles"] = ["role:channel-A-role"]
client.add_doc(sg_url,
db=sg_db,
doc=role_access_doc,
auth=admin_session)
elif grant_type == "CHANNEL-TO-ROLE-REST":
# Update the empty_role to have channel "A"
client.update_role(url=sg_admin_url,
db=sg_db,
name="empty_role",
channels=["A"])
elif grant_type == "CHANNEL-TO-ROLE-SYNC":
# Grant empty_role access to channel "A" via sync function
# Grant channel access to role via sync function
access_doc = document.create_doc("channel_grant_to_role")
access_doc["roles"] = ["role:empty_role"]
access_doc["channels"] = ["A"]
client.add_doc(url=sg_url,
db=sg_db,
doc=access_doc,
auth=admin_session,
use_post=True)
else:
pytest.fail("Unsupported grant_type!!!!")
# Issue one shot changes to make sure access grant is successful, the change may not propagate immediately so retry.
num_retries = 3
count = 0
while True:
if count == num_retries:
raise exceptions.ChangesError(
"Didn't get all expected changes before timing out!")
user_b_changes_after_grant = client.get_changes(
url=sg_url,
db=sg_db,
since=user_b_changes["last_seq"],
auth=user_b_session,
feed="normal")
if len(user_b_changes_after_grant["results"]) > 0:
# Found changes, break out an validate changes!
break
time.sleep(1)
count += 1
# User B shoud have recieved 51 docs (a_docs + 1 _user/USER_B doc) if a REST grant or 50 changes if the grant
# is via the sync function
changes_results = user_b_changes_after_grant["results"]
assert 50 <= len(changes_results) <= 51
# Create a dictionary of id rev pair of all the docs that are not "_user/" docs from changes
ids_and_revs_from_user_changes = {
change["id"]: change["changes"][0]["rev"]
for change in changes_results if not change["id"].startswith("_user/")
}
assert len(ids_and_revs_from_user_changes) == 50
# Create a list of id rev pair of all of the channel A docs
ids_and_revs_from_a_docs = {doc["id"]: doc["rev"] for doc in a_docs}
assert len(ids_and_revs_from_a_docs) == 50
# Check that the changes and the a_docs are identical in id and rev
assert ids_and_revs_from_user_changes == ids_and_revs_from_a_docs
# Get changes from last_seq of the changes request after the grant. There should be no new changes
user_b_changes = client.get_changes(
url=sg_url,
db=sg_db,
since=user_b_changes_after_grant["last_seq"],
auth=user_b_session,
feed="normal")
assert len(user_b_changes["results"]) == 0
def test_backfill_channels_looping_longpoll_changes(params_from_base_test_setup, sg_conf_name, grant_type):
cluster_config = params_from_base_test_setup["cluster_config"]
topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
log_info("grant_type: {}".format(grant_type))
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster = Cluster(cluster_config)
cluster.reset(sg_conf)
client = MobileRestClient()
admin_user_info = userinfo.UserInfo("admin", "pass", channels=["A"], roles=[])
user_b_user_info = userinfo.UserInfo("USER_B", "pass", channels=["B"], roles=[])
# Create users / sessions
client.create_user(url=sg_admin_url, db=sg_db,
name=admin_user_info.name, password=admin_user_info.password, channels=admin_user_info.channels)
client.create_user(url=sg_admin_url, db=sg_db,
name=user_b_user_info.name, password=user_b_user_info.password, channels=user_b_user_info.channels)
admin_session = client.create_session(url=sg_admin_url, db=sg_db, name=admin_user_info.name, password=admin_user_info.password)
user_b_session = client.create_session(url=sg_admin_url, db=sg_db, name=user_b_user_info.name, password=user_b_user_info.password)
# Create 50 "A" channel docs
a_docs = client.add_docs(url=sg_url, db=sg_db, number=50, id_prefix=None, auth=admin_session, channels=["A"])
assert len(a_docs) == 50
b_docs = client.add_docs(url=sg_url, db=sg_db, number=1, id_prefix="b_doc", auth=user_b_session, channels=["B"])
assert len(b_docs) == 1
user_doc = {"id": "_user/USER_B", "rev": None}
b_docs.append(user_doc)
# Loop until user_b sees b_doc_0 doc and _user/USER_B doc
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=b_docs, auth=user_b_session, strict=True)
# Create a dictionary keyed on doc id for all of channel A docs
ids_and_revs_from_a_docs = {doc["id"]: doc["rev"] for doc in a_docs}
assert len(ids_and_revs_from_a_docs.keys()) == 50
# Get last_seq for user_b
user_b_changes = client.get_changes(url=sg_url, db=sg_db, since=0, auth=user_b_session, feed="normal")
with concurrent.futures.ProcessPoolExecutor() as ex:
# Start long poll changes feed.
changes_task = ex.submit(client.get_changes,
url=sg_url, db=sg_db, since=user_b_changes["last_seq"], auth=user_b_session, timeout=10, limit=20)
# Grant access to channel "A"
if grant_type == "CHANNEL-REST":
log_info("Granting user access to channel A via Admin REST user update")
# Grant via update to user in Admin API
client.update_user(url=sg_admin_url, db=sg_db,
name=user_b_user_info.name, channels=["A", "B"])
elif grant_type == "CHANNEL-SYNC":
log_info("Granting user access to channel A sync function access()")
# Grant via access() in sync_function, then id 'channel_access' will trigger an access(doc.users, doc.channels)
access_doc = document.create_doc("channel_access", channels=["A"])
access_doc["users"] = ["USER_B"]
client.add_doc(url=sg_url, db=sg_db, doc=access_doc, auth=admin_session)
elif grant_type == "ROLE-REST":
log_info("Granting user access to channel A via Admin REST role grant")
# Create role with channel A
client.create_role(url=sg_admin_url, db=sg_db, name="channel-A-role", channels=["A"])
client.update_user(url=sg_admin_url, db=sg_db, name="USER_B", roles=["channel-A-role"])
elif grant_type == "ROLE-SYNC":
log_info("Granting user access to channel A via sync function role() grant")
# Create role with channel A
client.create_role(url=sg_admin_url, db=sg_db, name="channel-A-role", channels=["A"])
# Grant via role() in sync_function, then id 'role_access' will trigger an role(doc.users, doc.roles)
role_access_doc = document.create_doc("role_access")
role_access_doc["users"] = ["USER_B"]
role_access_doc["roles"] = ["role:channel-A-role"]
client.add_doc(sg_url, db=sg_db, doc=role_access_doc, auth=admin_session)
else:
pytest.fail("Unsupported grant_type!!!!")
# Block on return of longpoll changes, feed should wake up and return 20 results
changes = changes_task.result()
assert len(changes["results"]) == 20
num_requests = 1
# Cross the results off from the 'a_docs' dictionary
for doc in changes["results"]:
del ids_and_revs_from_a_docs[doc["id"]]
# Start looping longpoll changes with limit, cross off changes from dictionary each time one is found
# Since 20 changes should be crossed off already, this should execute 2x.
log_info("Starting looping longpoll changes with limit!")
last_seq = changes["last_seq"]
while True:
if len(ids_and_revs_from_a_docs.keys()) == 0:
log_info("All docs were found! Exiting polling loop")
break
changes = client.get_changes(url=sg_url, db=sg_db, since=last_seq, auth=user_b_session, limit=20, timeout=10)
num_requests += 1
# There are more than 2 requests, throw an exception.
if num_requests == 2:
assert len(changes["results"]) == 20
elif num_requests == 3:
# This will be 10 or 11 depending on if the _user/ doc is returned
assert 10 <= len(changes["results"]) <= 11
else:
raise exceptions.ChangesError("Looping longpoll should only have to perform 3 requests to get all the changes!!")
# Cross the results off from the 'a_docs' dictionary.
# This will blow up in docs duplicate docs are sent to changes
for doc in changes["results"]:
if doc["id"] != "_user/USER_B":
del ids_and_revs_from_a_docs[doc["id"]]
last_seq = changes["last_seq"]
# Shanges after longpoll
zero_results = client.get_changes(url=sg_url, db=sg_db,
since=last_seq,
auth=user_b_session, feed="normal")
# Changes should be caught up and there should be no results
assert len(zero_results["results"]) == 0
def test_backfill_channel_grant_to_role_longpoll(params_from_base_test_setup,
sg_conf_name, grant_type,
channels_to_grant):
"""
Test that check that docs are backfilled for a channel grant (via REST or SYNC) to existing role
1. Create a 'grantee' user with an empty role
2. 'pusher' user adds docs with channel(s) that will later be granted to 'grantee'
3. Verify that the 'pusher' sees the docs on its changes feed
4. Grant the 'grantee's role access to the pushers channels (either via REST or via sync function)
5. Verify that 'grantee' gets all of the docs after the grant
"""
cluster_config = params_from_base_test_setup["cluster_config"]
topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
num_docs_per_channel = 100
empty_role_name = "empty_role"
log_info("grant_type: {}".format(grant_type))
log_info("channels to grant access to: {}".format(channels_to_grant))
is_multi_channel_grant = False
if len(channels_to_grant) == 3:
is_multi_channel_grant = True
log_info("is_multi_channel_grant: {}".format(is_multi_channel_grant))
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster = Cluster(cluster_config)
cluster.reset(sg_conf)
client = MobileRestClient()
client.create_role(url=sg_admin_url,
db=sg_db,
name=empty_role_name,
channels=[])
pusher_info = userinfo.UserInfo("pusher",
"pass",
channels=channels_to_grant,
roles=[])
grantee_info = userinfo.UserInfo("grantee",
"pass",
channels=[],
roles=[empty_role_name])
# Create users
client.create_user(url=sg_admin_url,
db=sg_db,
name=pusher_info.name,
password=pusher_info.password,
channels=pusher_info.channels,
roles=pusher_info.roles)
pusher_session = client.create_session(url=sg_admin_url,
db=sg_db,
name=pusher_info.name,
password=pusher_info.password)
client.create_user(url=sg_admin_url,
db=sg_db,
name=grantee_info.name,
password=grantee_info.password,
channels=grantee_info.channels,
roles=grantee_info.roles)
grantee_session = client.create_session(url=sg_admin_url,
db=sg_db,
name=grantee_info.name,
password=grantee_info.password)
pusher_changes = client.get_changes(url=sg_url,
db=sg_db,
since=0,
auth=pusher_session)
# Make sure _user docs shows up in the changes feed
assert len(pusher_changes["results"]
) == 1 and pusher_changes["results"][0]["id"] == "_user/pusher"
# Add docs with the appropriate channels
a_docs = client.add_docs(url=sg_url,
db=sg_db,
number=num_docs_per_channel,
id_prefix=None,
auth=pusher_session,
channels=["A"])
assert len(a_docs) == 100
expected_docs = a_docs
if is_multi_channel_grant:
b_docs = client.add_docs(url=sg_url,
db=sg_db,
number=num_docs_per_channel,
id_prefix=None,
auth=pusher_session,
channels=["B"])
assert len(b_docs) == 100
expected_docs += b_docs
c_docs = client.add_docs(url=sg_url,
db=sg_db,
number=num_docs_per_channel,
id_prefix=None,
auth=pusher_session,
channels=["C"])
assert len(c_docs) == 100
expected_docs += c_docs
# Wait for all docs to show up on the changes feed before access grant
client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=expected_docs,
auth=pusher_session)
# Get changes for granted before grant and assert the only changes is the user doc
grantee_changes_before_grant = client.get_changes(url=sg_url,
db=sg_db,
since=0,
auth=grantee_session)
assert len(grantee_changes_before_grant["results"]) == 1
assert grantee_changes_before_grant["results"][0]["id"] == "_user/grantee"
if grant_type == "CHANNEL-REST":
# Grant channel access to role via REST
client.update_role(url=sg_admin_url,
db=sg_db,
name=empty_role_name,
channels=channels_to_grant)
elif grant_type == "CHANNEL-SYNC":
# Grant channel access to role via sync function
access_doc = document.create_doc(doc_id="channel_grant_to_role")
access_doc["roles"] = ["role:{}".format(empty_role_name)]
access_doc["channels"] = channels_to_grant
client.add_doc(url=sg_url,
db=sg_db,
doc=access_doc,
auth=pusher_session,
use_post=True)
# Issue changes request after grant
grantee_changes_post_grant = client.get_changes(
url=sg_url,
db=sg_db,
since=grantee_changes_before_grant["last_seq"],
auth=grantee_session,
feed="longpoll")
# grantee should have all the docs now
if is_multi_channel_grant:
# Check that the grantee gets all of the docs for channels ["A", "B", "C"]
assert len(
grantee_changes_post_grant["results"]) == num_docs_per_channel * 3
else:
# Check that the grantee gets all of the docs for channels ["A"]
assert len(
grantee_changes_post_grant["results"]) == num_docs_per_channel
# Disable this conditional if https://github.com/couchbase/sync_gateway/issues/2277 is fixed
if mode == "di":
# Issue one more changes request from the post grant last seq and make sure there are no other changes
grantee_changes_post_post_grant = client.get_changes(
url=sg_url,
db=sg_db,
since=grantee_changes_post_grant["last_seq"],
auth=grantee_session,
feed="normal")
assert len(grantee_changes_post_post_grant["results"]) == 0
def test_string_expiry_as_iso_8601_date(params_from_base_test_setup,
sg_conf_name):
"""
1. Calculate (server time + 3 seconds) as ISO-8601 date (e.g. 2016-01-01T00:00:00.000+00:00)
2. PUT /db/doc1 via SG with property "_exp":"[date]"
PUT /db/doc2 via SG with property "_exp":"2026-01-01T00:00:00.000+00:00"
3. Wait five seconds
4. Get /db/doc1. Assert response is 404
Get /db/doc2. Assert response is 20
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_string_expiry_as_ISO_8601_Date'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
bucket_name = "data-bucket"
cbs_ip = host_for_url(cbs_url)
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name),
password='******')
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
time_util = Time()
iso_datetime = time_util.get_iso_datetime(delta=3)
doc_exp_3_body = document.create_doc(doc_id="exp_3",
expiry=iso_datetime,
channels=sg_user_channels)
doc_exp_years_body = document.create_doc(
doc_id="exp_years",
expiry="2026-01-01T00:00:00.000+00:00",
channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_3_body,
auth=sg_user_session)
doc_exp_years = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_years_body,
auth=sg_user_session)
# Sleep should allow doc_exp_3 to expire
time.sleep(10)
# doc_exp_3 should be expired
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3["id"],
auth=sg_user_session)
# In XATTR mode, the expiry results in a tombstone
# In Doc Meta mode, the expiry results in a purge
if xattrs_enabled:
assert he.value[0].startswith("403 Client Error: Forbidden for url:")
else:
assert he.value[0].startswith("404 Client Error: Not Found for url:")
verify_doc_deletion_on_server(doc_id=doc_exp_3["id"],
sdk_client=sdk_client,
sg_client=client,
sg_admin_url=sg_url_admin,
sg_db=sg_db,
xattrs_enabled=xattrs_enabled)
# doc_exp_years should be available still
doc_exp_years_result = client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_years["id"],
auth=sg_user_session)
assert doc_exp_years_result["_id"] == "exp_years"
def test_auto_prune_listener_keeps_conflicts_sanity(setup_client_syncgateway_test):
""""
1. Create db on LiteServ and add docs
2. Create db on sync_gateway and add docs with the same id
3. Create one shot push / pull replication
4. Update LiteServ 50 times
5. Assert that pruned conflict is still present
6. Delete the current revision and check that a GET returns the old conflict as the current rev
"""
cluster_config = setup_client_syncgateway_test["cluster_config"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_url = setup_client_syncgateway_test["sg_url"]
sg_admin_url = setup_client_syncgateway_test["sg_admin_url"]
client = MobileRestClient()
sg_helper = SyncGateway()
sg_helper.start_sync_gateway(
cluster_config=cluster_config, url=sg_url, config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS)
)
log_info("Running 'test_auto_prune_listener_keeps_conflicts_sanity' ...")
log_info("ls_url: {}".format(ls_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_admin_url: {}".format(sg_admin_url))
num_docs = 1
num_revs = 100
sg_db = "db"
ls_db = "ls_db"
sg_user_name = "sg_user"
sg_user_channels = ["NBC"]
client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels)
sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name)
ls_db = client.create_database(url=ls_url, name=ls_db)
# Create docs with same prefix to create conflicts when the dbs complete 1 shot replication
ls_db_docs = client.add_docs(url=ls_url, db=ls_db, number=num_docs, id_prefix="doc", channels=sg_user_channels)
assert len(ls_db_docs) == num_docs
sg_db_docs = client.add_docs(
url=sg_url, db=sg_db, number=num_docs, id_prefix="doc", channels=sg_user_channels, auth=sg_session
)
assert len(sg_db_docs) == num_docs
# Setup one shot pull replication and wait for idle.
client.start_replication(url=ls_url, continuous=False, from_url=sg_admin_url, from_db=sg_db, to_db=ls_db)
client.wait_for_no_replications(url=ls_url)
# There should now be a conflict on the client
conflicting_revs = client.get_conflict_revs(url=ls_url, db=ls_db, doc=ls_db_docs[0])
# Get the doc with conflict rev
client.get_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"], rev=conflicting_revs[0])
# Update doc past revs limit and make sure conflict is still available
updated_doc = client.update_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"], number_updates=num_revs)
client.get_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"], rev=conflicting_revs[0])
# Delete doc and ensure that the conflict is now the current rev
client.delete_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"], rev=updated_doc["rev"])
current_doc = client.get_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"])
assert current_doc["_rev"] == conflicting_revs[0]
def test_removing_expiry(params_from_base_test_setup, sg_conf_name):
"""
1. PUT /db/doc1 via SG with property "_exp":3
2. Update /db/doc1 with a new revision with no expiry value
3. After 10 updates, update /db/doc1 with a revision with no expiry
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_removing_expiry'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
doc_exp_3_body = document.create_doc(doc_id="exp_3",
expiry=3,
channels=sg_user_channels)
doc_exp_10_body = document.create_doc(doc_id="exp_10",
expiry=10,
channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_3_body,
auth=sg_user_session)
doc_exp_10 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_10_body,
auth=sg_user_session)
doc_exp_3_updated = client.update_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3["id"],
number_updates=10,
auth=sg_user_session)
# Sleep should allow an expiry to happen on doc_exp_3 if it had not been removed.
# Expected behavior is that the doc_exp_3 will still be around due to the removal of the expiry
time.sleep(5)
# doc_exp_3 should no longer have an expiry and should not raise an exception
doc_exp_3_updated_result = client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3_updated["id"],
auth=sg_user_session)
assert doc_exp_3_updated_result["_id"] == "exp_3"
# doc_exp_10 should be available still and should not raise an exception
doc_exp_10_result = client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_10["id"],
auth=sg_user_session)
assert doc_exp_10_result["_id"] == "exp_10"
def test_longpoll_awaken_doc_add_update(params_from_base_test_setup, sg_conf_name):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
cluster_topology = params_from_base_test_setup["cluster_topology"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
sg_admin_url = cluster_topology["sync_gateways"][0]["admin"]
sg_url = cluster_topology["sync_gateways"][0]["public"]
log_info("sg_conf: {}".format(sg_conf))
log_info("sg_admin_url: {}".format(sg_admin_url))
log_info("sg_url: {}".format(sg_url))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
adam_user_info = userinfo.UserInfo(name="adam", password="******", channels=["NBC"], roles=[])
traun_user_info = userinfo.UserInfo(name="traun", password="******", channels=["CBS"], roles=[])
andy_user_info = userinfo.UserInfo(name="andy", password="******", channels=["MTV"], roles=[])
sg_db = "db"
client = MobileRestClient()
adam_auth = client.create_user(url=sg_admin_url, db=sg_db,
name=adam_user_info.name, password=adam_user_info.password, channels=adam_user_info.channels)
traun_auth = client.create_user(url=sg_admin_url, db=sg_db,
name=traun_user_info.name, password=traun_user_info.password, channels=traun_user_info.channels)
andy_auth = client.create_user(url=sg_admin_url, db=sg_db,
name=andy_user_info.name, password=andy_user_info.password, channels=andy_user_info.channels)
# Get starting sequence of docs, use the last seq to progress past any user docs
adam_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=adam_auth)
traun_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=traun_auth)
andy_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=andy_auth)
with concurrent.futures.ProcessPoolExecutor() as ex:
adam_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=adam_changes["last_seq"], timeout=30, auth=adam_auth)
traun_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=traun_changes["last_seq"], timeout=30, auth=traun_auth)
andy_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=andy_changes["last_seq"], timeout=30, auth=andy_auth)
# Wait for changes feed to notice there are no changes and enter wait. 2 seconds should be more than enough
time.sleep(2)
# Make sure the changes future is still running and has not exited due to any new changes, the feed should be caught up
# and waiting
assert not adam_changes_task.done()
assert not traun_changes_task.done()
assert not andy_changes_task.done()
# Add 3 docs with adam auth, one with adam channels, one with traun channels, one with andy auth
# Tests the changes feed wakes up for user created docs and non user created docs
adam_add_doc_task_1 = ex.submit(client.add_docs, url=sg_url, db=sg_db,
number=1, id_prefix="adam_doc",
auth=adam_auth, channels=adam_user_info.channels)
adam_add_doc_task_2 = ex.submit(client.add_docs, url=sg_url, db=sg_db,
number=1, id_prefix="traun_doc",
auth=adam_auth, channels=traun_user_info.channels)
adam_add_doc_task_3 = ex.submit(client.add_docs, url=sg_url, db=sg_db,
number=1, id_prefix="andy_doc",
auth=adam_auth, channels=andy_user_info.channels)
# Wait for docs adds to complete
adam_doc_1 = adam_add_doc_task_1.result()
assert len(adam_doc_1) == 1
adam_doc_2 = adam_add_doc_task_2.result()
assert len(adam_doc_2) == 1
adam_doc_3 = adam_add_doc_task_3.result()
assert len(adam_doc_3) == 1
# Assert that the changes feed woke up and that the doc change was propagated
adam_changes = adam_changes_task.result()
assert len(adam_changes["results"]) == 1
assert adam_changes["results"][0]["id"] == "adam_doc_0"
traun_changes = traun_changes_task.result()
assert len(traun_changes["results"]) == 1
assert traun_changes["results"][0]["id"] == "traun_doc_0"
andy_changes = andy_changes_task.result()
assert len(andy_changes["results"]) == 1
assert andy_changes["results"][0]["id"] == "andy_doc_0"
# Start another longpoll changes request
adam_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=adam_changes["last_seq"], auth=adam_auth)
traun_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=traun_changes["last_seq"], auth=traun_auth)
andy_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=andy_changes["last_seq"], auth=andy_auth)
# Wait for changes feed to notice there are no changes and enter wait. 2 seconds should be more than enough
time.sleep(2)
# Make sure the changes future is still running and has not exited due to any new changes, the feed should be caught up
# and waiting
assert not adam_changes_task.done()
assert not traun_changes_task.done()
assert not andy_changes_task.done()
# Check to see that doc updates to own user doc wake up changes feed
adam_update_docs_task = ex.submit(client.update_docs, url=sg_url, db=sg_db, docs=adam_doc_1, number_updates=1, auth=adam_auth)
traun_update_docs_task = ex.submit(client.update_docs, url=sg_url, db=sg_db, docs=adam_doc_2, number_updates=1, auth=traun_auth)
andy_update_docs_task = ex.submit(client.update_docs, url=sg_url, db=sg_db, docs=adam_doc_3, number_updates=1, auth=andy_auth)
# Wait for docs updates to complete
adam_updated_docs = adam_update_docs_task.result()
assert len(adam_updated_docs) == 1
assert adam_updated_docs[0]["rev"].startswith("2-")
traun_updated_docs = traun_update_docs_task.result()
assert len(traun_updated_docs) == 1
assert traun_updated_docs[0]["rev"].startswith("2-")
andy_updated_docs = andy_update_docs_task.result()
assert len(andy_updated_docs) == 1
assert andy_updated_docs[0]["rev"].startswith("2-")
# Assert that the changes feed woke up and that the doc updates was propagated
adam_changes = adam_changes_task.result()
assert len(adam_changes["results"]) == 1
assert adam_changes["results"][0]["id"] == "adam_doc_0"
rev_from_change = int(adam_changes["results"][0]["changes"][0]["rev"].split("-")[0])
assert rev_from_change == 2
traun_changes = traun_changes_task.result()
assert len(traun_changes["results"]) == 1
assert traun_changes["results"][0]["id"] == "traun_doc_0"
rev_from_change = int(traun_changes["results"][0]["changes"][0]["rev"].split("-")[0])
assert rev_from_change == 2
andy_changes = andy_changes_task.result()
assert len(andy_changes["results"]) == 1
assert andy_changes["results"][0]["id"] == "andy_doc_0"
rev_from_change = int(andy_changes["results"][0]["changes"][0]["rev"].split("-")[0])
assert rev_from_change == 2
# Start another longpoll changes request from the last_seq
adam_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=adam_changes["last_seq"], auth=adam_auth)
traun_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=traun_changes["last_seq"], auth=traun_auth)
andy_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=andy_changes["last_seq"], auth=andy_auth)
# Wait for changes feed to notice there are no changes and enter wait. 2 seconds should be more than enough
time.sleep(2)
# Make sure the changes future is still running and has not exited due to any new changes, the feed should be caught up
# and waiting
assert not adam_changes_task.done()
assert not traun_changes_task.done()
assert not andy_changes_task.done()
# Add doc with access to all channels
adam_add_doc_task_4 = ex.submit(client.add_docs, url=sg_url, db=sg_db,
number=1, id_prefix="all_doc",
auth=adam_auth,
channels=adam_user_info.channels + traun_user_info.channels + andy_user_info.channels)
all_doc = adam_add_doc_task_4.result()
assert len(all_doc) == 1
# Assert that the changes feed woke up and that the doc add was propagated to all users
adam_changes = adam_changes_task.result()
assert len(adam_changes["results"]) == 1
assert adam_changes["results"][0]["id"] == "all_doc_0"
rev_from_change = int(adam_changes["results"][0]["changes"][0]["rev"].split("-")[0])
assert rev_from_change == 1
traun_changes = traun_changes_task.result()
assert len(traun_changes["results"]) == 1
assert traun_changes["results"][0]["id"] == "all_doc_0"
rev_from_change = int(traun_changes["results"][0]["changes"][0]["rev"].split("-")[0])
assert rev_from_change == 1
andy_changes = andy_changes_task.result()
assert len(andy_changes["results"]) == 1
assert andy_changes["results"][0]["id"] == "all_doc_0"
rev_from_change = int(andy_changes["results"][0]["changes"][0]["rev"].split("-")[0])
assert rev_from_change == 1
# Start another longpoll changes request from the last_seq
adam_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=adam_changes["last_seq"], auth=adam_auth)
traun_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=traun_changes["last_seq"], auth=traun_auth)
andy_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=andy_changes["last_seq"], auth=andy_auth)
# Wait for changes feed to notice there are no changes and enter wait. 2 seconds should be more than enough
time.sleep(2)
# Make sure the changes future is still running and has not exited due to any new changes, the feed should be caught up
# and waiting
assert not adam_changes_task.done()
assert not traun_changes_task.done()
assert not andy_changes_task.done()
# Update doc with access to all channels
adam_update_docs_task_2 = ex.submit(client.update_docs, url=sg_url, db=sg_db, docs=all_doc, number_updates=1, auth=adam_auth)
# Wait for docs updates to complete
all_doc_updated = adam_update_docs_task_2.result()
assert len(all_doc_updated) == 1
assert all_doc_updated[0]["rev"].startswith("2-")
# Assert that the changes feed woke up and that the doc update was propagated to all users
adam_changes = adam_changes_task.result()
assert len(adam_changes["results"]) == 1
assert adam_changes["results"][0]["id"] == "all_doc_0"
rev_from_change = int(adam_changes["results"][0]["changes"][0]["rev"].split("-")[0])
assert rev_from_change == 2
traun_changes = traun_changes_task.result()
assert len(traun_changes["results"]) == 1
assert traun_changes["results"][0]["id"] == "all_doc_0"
rev_from_change = int(traun_changes["results"][0]["changes"][0]["rev"].split("-")[0])
assert rev_from_change == 2
andy_changes = andy_changes_task.result()
assert len(andy_changes["results"]) == 1
assert andy_changes["results"][0]["id"] == "all_doc_0"
rev_from_change = int(andy_changes["results"][0]["changes"][0]["rev"].split("-")[0])
assert rev_from_change == 2
def test_setting_expiry_in_bulk_docs(params_from_base_test_setup,
sg_conf_name):
"""
1. PUT /db/_bulk_docs with 10 documents. Set the "_exp":3 on 5 of these documents
2. Wait five seconds
3. POST /db/_bulk_get for the 10 documents. Validate that only the 5 non-expiring documents are returned
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_setting_expiry_in_bulk_docs'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
bucket_name = "data-bucket"
cbs_ip = host_for_url(cbs_url)
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name),
password='******')
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
doc_exp_3_bodies = document.create_docs(doc_id_prefix="exp_3",
number=5,
expiry=3,
channels=sg_user_channels)
doc_exp_10_bodies = document.create_docs(doc_id_prefix="exp_10",
number=5,
expiry=10,
channels=sg_user_channels)
bulk_bodies = doc_exp_3_bodies + doc_exp_10_bodies
bulk_docs = client.add_bulk_docs(url=sg_url,
db=sg_db,
docs=bulk_bodies,
auth=sg_user_session)
# Allow exp_3 docs to expire
time.sleep(5)
bulk_docs_ids = [doc["id"] for doc in bulk_docs]
expected_ids = ["exp_10_0", "exp_10_1", "exp_10_2", "exp_10_3", "exp_10_4"]
expected_missing_ids = [
"exp_3_0", "exp_3_1", "exp_3_2", "exp_3_3", "exp_3_4"
]
bulk_get_docs, errors = client.get_bulk_docs(url=sg_url,
db=sg_db,
doc_ids=bulk_docs_ids,
auth=sg_user_session,
validate=False)
assert len(bulk_get_docs) == len(expected_ids)
assert len(errors) == len(expected_missing_ids)
bulk_get_doc_ids = [doc["_id"] for doc in bulk_get_docs]
error_ids = [doc["id"] for doc in errors]
assert bulk_get_doc_ids == expected_ids
assert error_ids == expected_missing_ids
client.verify_doc_ids_found_in_response(response=bulk_get_docs,
expected_doc_ids=expected_ids)
client.verify_doc_ids_not_found_in_response(
response=errors, expected_missing_doc_ids=expected_missing_ids)
for expired_doc in error_ids:
verify_doc_deletion_on_server(doc_id=expired_doc,
sdk_client=sdk_client,
sg_client=client,
sg_admin_url=sg_url_admin,
sg_db=sg_db,
xattrs_enabled=xattrs_enabled)
def test_longpoll_awaken_roles(params_from_base_test_setup, sg_conf_name):
cluster_conf = params_from_base_test_setup["cluster_config"]
cluster_topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
sg_admin_url = cluster_topology["sync_gateways"][0]["admin"]
sg_url = cluster_topology["sync_gateways"][0]["public"]
log_info("sg_conf: {}".format(sg_conf))
log_info("sg_admin_url: {}".format(sg_admin_url))
log_info("sg_url: {}".format(sg_url))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
admin_role = "admin_role"
admin_channel = "admin_channel"
admin_user_info = userinfo.UserInfo(name="admin", password="******", channels=[], roles=[admin_role])
adam_user_info = userinfo.UserInfo(name="adam", password="******", channels=[], roles=[])
traun_user_info = userinfo.UserInfo(name="traun", password="******", channels=[], roles=[])
andy_user_info = userinfo.UserInfo(name="andy", password="******", channels=[], roles=[])
sg_db = "db"
client = MobileRestClient()
# Create a role on sync_gateway
client.create_role(url=sg_admin_url, db=sg_db, name=admin_role, channels=[admin_channel])
# Create users with no channels or roles
admin_auth = client.create_user(url=sg_admin_url, db=sg_db,
name=admin_user_info.name, password=admin_user_info.password, roles=[admin_role])
adam_auth = client.create_user(url=sg_admin_url, db=sg_db,
name=adam_user_info.name, password=adam_user_info.password)
traun_auth = client.create_user(url=sg_admin_url, db=sg_db,
name=traun_user_info.name, password=traun_user_info.password)
andy_auth = client.create_user(url=sg_admin_url, db=sg_db,
name=andy_user_info.name, password=andy_user_info.password)
################################
# change feed wakes for role add
################################
# Get starting sequence of docs, use the last seq to progress past any _user docs.
adam_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=adam_auth)
traun_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=traun_auth)
andy_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=andy_auth)
# Add doc with channel associated with the admin role
admin_doc = client.add_docs(url=sg_url, db=sg_db, number=1, id_prefix="admin_doc", auth=admin_auth, channels=[admin_channel])
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=admin_doc, auth=admin_auth)
with concurrent.futures.ProcessPoolExecutor() as ex:
# Start changes feed for 3 users from latest last_seq
adam_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=adam_changes["last_seq"], timeout=10, auth=adam_auth)
traun_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=traun_changes["last_seq"], timeout=10, auth=traun_auth)
andy_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=andy_changes["last_seq"], timeout=10, auth=andy_auth)
# Wait for changes feed to notice there are no changes and enter wait. 2 seconds should be more than enough
time.sleep(2)
# Make sure the changes future is still running and has not exited due to any new changes, the feed should be caught up
# and waiting
assert not adam_changes_task.done()
assert not traun_changes_task.done()
assert not andy_changes_task.done()
adam_auth = client.update_user(url=sg_admin_url, db=sg_db,
name=adam_user_info.name, password=adam_user_info.password, roles=[admin_role])
traun_auth = client.update_user(url=sg_admin_url, db=sg_db,
name=traun_user_info.name, password=traun_user_info.password, roles=[admin_role])
andy_auth = client.update_user(url=sg_admin_url, db=sg_db,
name=andy_user_info.name, password=andy_user_info.password, roles=[admin_role])
adam_changes = adam_changes_task.result()
assert 1 <= len(adam_changes["results"]) <= 2
assert adam_changes["results"][0]["id"] == "admin_doc_0" or adam_changes["results"][0]["id"] == "_user/adam"
traun_changes = traun_changes_task.result()
assert 1 <= len(traun_changes["results"]) <= 2
assert traun_changes["results"][0]["id"] == "admin_doc_0" or traun_changes["results"][0]["id"] == "_user/traun"
andy_changes = andy_changes_task.result()
assert 1 <= len(andy_changes["results"]) <= 2
assert andy_changes["results"][0]["id"] == "admin_doc_0" or andy_changes["results"][0]["id"] == "_user/andy"
# Check that the user docs all show up in changes feed
client.verify_doc_id_in_changes(url=sg_url, db=sg_db, expected_doc_id="_user/adam", auth=adam_auth)
client.verify_doc_id_in_changes(url=sg_url, db=sg_db, expected_doc_id="_user/traun", auth=traun_auth)
client.verify_doc_id_in_changes(url=sg_url, db=sg_db, expected_doc_id="_user/andy", auth=andy_auth)
# Check that the admin doc made it to all the changes feeds
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=admin_doc, auth=adam_auth)
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=admin_doc, auth=traun_auth)
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=admin_doc, auth=andy_auth)
# At this point, each user should have a changes feed that is caught up for the next section
###########################################
# change feed wakes for channel add to role
###########################################
abc_channel = "ABC"
abc_pusher_info = userinfo.UserInfo(name="abc_pusher", password="******", channels=[abc_channel], roles=[])
abc_pusher_auth = client.create_user(url=sg_admin_url, db=sg_db, name=abc_pusher_info.name,
password=abc_pusher_info.password, channels=abc_pusher_info.channels)
# Add doc with ABC channel
client.add_docs(url=sg_url, db=sg_db, number=1, id_prefix="abc_doc", auth=abc_pusher_auth, channels=[abc_channel])
# Get latest last_seq for next test section
adam_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=adam_auth)
traun_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=traun_auth)
andy_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=andy_auth)
with concurrent.futures.ProcessPoolExecutor() as ex:
# Start changes feed for 3 users from latest last_seq
adam_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=adam_changes["last_seq"], timeout=10, auth=adam_auth)
traun_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=traun_changes["last_seq"], timeout=10, auth=traun_auth)
andy_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=andy_changes["last_seq"], timeout=10, auth=andy_auth)
# Wait for changes feed to notice there are no changes and enter wait. 2 seconds should be more than enough
time.sleep(2)
# Make sure the changes future is still running and has not exited due to any new changes, the feed should be caught up
# and waiting
assert not adam_changes_task.done()
assert not traun_changes_task.done()
assert not andy_changes_task.done()
# Update admin role to include ABC channel
# Since adam, traun, and andy are assigned to that role, they should wake up and get the 'abc_pusher_0' doc
client.update_role(url=sg_admin_url, db=sg_db, name=admin_role, channels=[admin_channel, abc_channel])
adam_changes = adam_changes_task.result()
assert len(adam_changes["results"]) == 1
assert adam_changes["results"][0]["id"] == "abc_doc_0"
traun_changes = traun_changes_task.result()
assert len(traun_changes["results"]) == 1
assert traun_changes["results"][0]["id"] == "abc_doc_0"
andy_changes = adam_changes_task.result()
assert len(andy_changes["results"]) == 1
assert andy_changes["results"][0]["id"] == "abc_doc_0"
def test_initial_pull_replication(setup_client_syncgateway_test, continuous):
"""
1. Prepare sync-gateway to have 10000 documents.
2. Create a single shot / continuous pull replicator and to pull the docs into a database.
3. Verify if all of the docs get pulled.
Referenced issue: couchbase/couchbase-lite-android#955.
"""
sg_db = "db"
ls_db = "ls_db"
num_docs = 10000
cluster_config = setup_client_syncgateway_test["cluster_config"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_one_admin = setup_client_syncgateway_test["sg_admin_url"]
sg_one_public = setup_client_syncgateway_test["sg_url"]
sg_helper = SyncGateway()
sg_helper.start_sync_gateway(
cluster_config=cluster_config,
url=sg_one_public,
config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS)
)
log_info("Running 'test_initial_pull_replication', continuous: {}".format(continuous))
log_info("ls_url: {}".format(ls_url))
log_info("sg_one_admin: {}".format(sg_one_admin))
log_info("sg_one_public: {}".format(sg_one_public))
client = MobileRestClient()
client.create_user(sg_one_admin, sg_db, "seth", password="******", channels=["ABC", "NBC"])
session = client.create_session(sg_one_admin, sg_db, "seth")
# Create 'num_docs' docs on sync_gateway
docs = client.add_docs(
url=sg_one_public,
db=sg_db,
number=num_docs,
id_prefix="seeded_doc",
generator="four_k",
auth=session
)
assert len(docs) == num_docs
client.create_database(url=ls_url, name=ls_db)
# Start oneshot pull replication
repl_id = client.start_replication(
url=ls_url,
continuous=continuous,
from_url=sg_one_admin,
from_db=sg_db,
to_db=ls_db
)
start = time.time()
if continuous:
log_info("Waiting for replication status 'Idle' for: {}".format(repl_id))
# Android will report IDLE status, and drop into the 'verify_docs_present' below
# due to https://github.com/couchbase/couchbase-lite-java-core/issues/1409
client.wait_for_replication_status_idle(ls_url, repl_id)
else:
log_info("Waiting for no replications: {}".format(repl_id))
client.wait_for_no_replications(ls_url)
# Verify docs replicated to client
client.verify_docs_present(url=ls_url, db=ls_db, expected_docs=docs, timeout=240)
all_docs_replicated_time = time.time() - start
log_info("Replication took: {}s".format(all_docs_replicated_time))
# Verify docs show up in client's changes feed
client.verify_docs_in_changes(url=ls_url, db=ls_db, expected_docs=docs)
replications = client.get_replications(url=ls_url)
if continuous:
assert len(replications) == 1, "There should only be one replication running"
assert replications[0]["status"] == "Idle", "Replication Status should be 'Idle'"
assert replications[0]["continuous"], "Running replication should be continuous"
# Only .NET has an 'error' property
if "error" in replications[0]:
assert len(replications[0]["error"]) == 0
else:
assert len(replications) == 0, "No replications should be running"
def test_remove_add_channels_to_doc(params_from_base_test_setup, sg_conf_name):
cluster_config = params_from_base_test_setup["cluster_config"]
topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster = Cluster(cluster_config)
cluster.reset(sg_conf)
client = MobileRestClient()
admin_user_info = userinfo.UserInfo("admin", "pass", channels=["A", "B"], roles=[])
a_user_info = userinfo.UserInfo("a_user", "pass", channels=["A"], roles=[])
admin_user_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=admin_user_info.name,
password=admin_user_info.password,
channels=admin_user_info.channels,
)
a_user_auth = client.create_user(
url=sg_admin_url, db=sg_db, name=a_user_info.name, password=a_user_info.password, channels=a_user_info.channels
)
a_docs = client.add_docs(
url=sg_url, db=sg_db, number=50, id_prefix="a_doc", auth=admin_user_auth, channels=admin_user_info.channels
)
# Build dictionay of a_docs
a_docs_id_rev = {doc["id"]: doc["rev"] for doc in a_docs}
assert len(a_docs_id_rev) == 50
# Wait for all docs to show up in changes
client.verify_doc_id_in_changes(sg_url, sg_db, expected_doc_id="_user/a_user", auth=a_user_auth)
client.verify_docs_in_changes(sg_url, sg_db, expected_docs=a_docs, auth=a_user_auth)
# Get changes for 'a_user'
a_user_changes = client.get_changes(url=sg_url, db=sg_db, since=0, auth=a_user_auth, feed="normal")
# 'a_user' should get 50 'a_doc_*' doc and 1 '_user/a_user' doc
assert len(a_user_changes["results"]) == 51
###########################
# Remove Channels from doc
###########################
# Copy a_docs_id_rev to dictionary to scratch off values
remove_docs_scratch_off = a_docs_id_rev.copy()
assert len(remove_docs_scratch_off) == 50
# Use admin user to update the docs to remove 'A' from the channels property on the doc and add 'B'
client.update_docs(url=sg_url, db=sg_db, docs=a_docs, number_updates=1, auth=admin_user_auth, channels=["B"])
# Longpoll loop requires due to the delay that changes take to permeate to the client
changes_timeout = 10
start = time.time()
last_seq = a_user_changes["last_seq"]
while True:
# If take longer than 10 seconds, fail the test
if time.time() - start > changes_timeout:
raise keywords.exceptions.TimeoutException("Could not find all expected docs in changs feed")
# We found everything, exit loop!
if remove_docs_scratch_off == {}:
log_info("All expected docs found to be removed")
break
# Get changes for 'a_user' from last_seq
a_user_changes = client.get_changes(url=sg_url, db=sg_db, since=last_seq, auth=a_user_auth, timeout=10)
assert len(a_user_changes["results"]) > 0
# Loop over changes found and perform the following
# 1. Check that the docs is flagged with 'removed'
# 2. Cross off the doc fromt the the 'remove_docs_scratch_off'
for change in a_user_changes["results"]:
assert change["removed"] == ["A"]
assert change["changes"][0]["rev"].startswith("2-")
# This will blow up if any change is not found in that dictionary
del remove_docs_scratch_off[change["id"]]
# Update last_seq
last_seq = a_user_changes["last_seq"]
# Issue changes request from 'last_seq' and verify that the changes are up to date and returns no results
a_user_changes = client.get_changes(url=sg_url, db=sg_db, since=last_seq, auth=a_user_auth, feed="normal")
assert len(a_user_changes["results"]) == 0
#########################
# Add Channels to doc
#########################
# Copy the a_docs_id_rev dictionary for scratch ovee
add_docs_scratch_off = a_docs_id_rev.copy()
assert len(add_docs_scratch_off) == 50
# Use admin user to update the docs to add ['A'] back to document channels
client.update_docs(url=sg_url, db=sg_db, docs=a_docs, number_updates=1, auth=admin_user_auth, channels=["A"])
# Longpoll loop requires due to the delay that changes take to permeate to the client
changes_timeout = 10
start = time.time()
last_seq = a_user_changes["last_seq"]
while True:
# If take longer than 10 seconds, fail the test
if time.time() - start > changes_timeout:
raise keywords.exceptions.TimeoutException("Could not find all expected docs in changs feed")
# We found everything, exit loop!
if add_docs_scratch_off == {}:
log_info("All expected docs found to be removed")
break
# Get changes for 'a_user' from last_seq
a_user_changes = client.get_changes(url=sg_url, db=sg_db, since=last_seq, auth=a_user_auth, timeout=10)
assert len(a_user_changes["results"]) > 0
# Loop over changes found and perform the following
# 1. Check that the docs has a 3rd gen rev prefix
# 2. Cross off the doc fromt the the 'add_docs_scratch_off'
for change in a_user_changes["results"]:
assert change["changes"][0]["rev"].startswith("3-")
# This will blow up if any change is not found in that dictionary
del add_docs_scratch_off[change["id"]]
# Update last_seq
last_seq = a_user_changes["last_seq"]
# Issue changes request from 'last_seq' and verify that the changes are up to date and returns no results
a_user_changes = client.get_changes(url=sg_url, db=sg_db, since=last_seq, auth=a_user_auth, feed="normal")
assert len(a_user_changes["results"]) == 0
def test_verify_open_revs_with_revs_limit_push_conflict(setup_client_syncgateway_test):
"""Test replication from multiple client dbs to one sync_gateway db
https://github.com/couchbase/couchbase-lite-ios/issues/1277
"""
cluster_config = setup_client_syncgateway_test["cluster_config"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_url = setup_client_syncgateway_test["sg_url"]
sg_admin_url = setup_client_syncgateway_test["sg_admin_url"]
num_docs = 100
num_revs = 20
sg_db = "db"
sg_user_name = "sg_user"
sg_helper = SyncGateway()
sg_helper.start_sync_gateway(
cluster_config=cluster_config,
url=sg_url,
config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS)
)
log_info("Running 'test_verify_open_revs_with_revs_limit_push_conflict'")
log_info("ls_url: {}".format(ls_url))
log_info("sg_admin_url: {}".format(sg_admin_url))
log_info("sg_url: {}".format(sg_url))
log_info("num_docs: {}".format(num_docs))
log_info("num_revs: {}".format(num_revs))
client = MobileRestClient()
# Test the endpoint, listener does not support users but should have a default response
client.get_session(url=ls_url)
sg_user_channels = ["NBC"]
client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels)
sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name)
ls_db = client.create_database(url=ls_url, name="ls_db")
ls_db_docs = client.add_docs(url=ls_url, db=ls_db, number=num_docs, id_prefix="ls_db", channels=sg_user_channels)
assert len(ls_db_docs) == num_docs
# Start replication ls_db -> sg_db
repl_one = client.start_replication(
url=ls_url,
continuous=True,
from_db=ls_db,
to_url=sg_admin_url, to_db=sg_db
)
client.verify_docs_present(url=sg_admin_url, db=sg_db, expected_docs=ls_db_docs)
client.update_docs(url=sg_url, db=sg_db, docs=ls_db_docs, number_updates=num_revs, auth=sg_session)
sg_current_doc = client.get_doc(url=sg_url, db=sg_db, doc_id="ls_db_2", auth=sg_session)
client.update_docs(url=ls_url, db=ls_db, docs=ls_db_docs, number_updates=num_revs)
ls_current_doc = client.get_doc(url=ls_url, db=ls_db, doc_id="ls_db_2")
client.wait_for_replication_status_idle(url=ls_url, replication_id=repl_one)
client.verify_doc_rev_generation(url=ls_url, db=ls_db, doc_id=ls_current_doc["_id"], expected_generation=21)
client.verify_doc_rev_generation(url=sg_url, db=sg_db, doc_id=sg_current_doc["_id"], expected_generation=21, auth=sg_session)
expected_ls_revs = [ls_current_doc["_rev"]]
client.verify_open_revs(url=ls_url, db=ls_db, doc_id=ls_current_doc["_id"], expected_open_revs=expected_ls_revs)
expected_sg_revs = [ls_current_doc["_rev"], sg_current_doc["_rev"]]
client.verify_open_revs(url=sg_admin_url, db=sg_db, doc_id=sg_current_doc["_id"], expected_open_revs=expected_sg_revs)
def test_backfill_channels_oneshot_changes(params_from_base_test_setup, sg_conf_name, grant_type):
cluster_config = params_from_base_test_setup["cluster_config"]
topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
log_info("grant_type: {}".format(grant_type))
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster = Cluster(cluster_config)
cluster.reset(sg_conf)
client = MobileRestClient()
admin_user_info = userinfo.UserInfo("admin", "pass", channels=["A"], roles=[])
user_b_user_info = userinfo.UserInfo("USER_B", "pass", channels=["B"], roles=[])
# Create users / sessions
client.create_user(url=sg_admin_url, db=sg_db,
name=admin_user_info.name, password=admin_user_info.password, channels=admin_user_info.channels)
client.create_user(url=sg_admin_url, db=sg_db,
name=user_b_user_info.name, password=user_b_user_info.password, channels=user_b_user_info.channels)
admin_session = client.create_session(url=sg_admin_url, db=sg_db, name=admin_user_info.name, password=admin_user_info.password)
user_b_session = client.create_session(url=sg_admin_url, db=sg_db, name=user_b_user_info.name, password=user_b_user_info.password)
# Create 50 "A" channel docs
a_docs = client.add_docs(url=sg_url, db=sg_db, number=50, id_prefix=None, auth=admin_session, channels=["A"])
assert len(a_docs) == 50
b_docs = client.add_docs(url=sg_url, db=sg_db, number=1, id_prefix="b_doc", auth=user_b_session, channels=["B"])
assert len(b_docs) == 1
user_doc = {"id": "_user/USER_B", "rev": None}
b_docs.append(user_doc)
# Loop until user_b sees b_doc_0 doc and _user/USER_B doc
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=b_docs, auth=user_b_session, strict=True)
# Get last_seq for user_b
user_b_changes = client.get_changes(url=sg_url, db=sg_db, since=0, auth=user_b_session, feed="normal")
# Grant access to channel "A"
if grant_type == "CHANNEL-REST":
log_info("Granting user access to channel A via Admin REST user update")
# Grant via update to user in Admin API
client.update_user(url=sg_admin_url, db=sg_db,
name=user_b_user_info.name, channels=["A", "B"])
elif grant_type == "CHANNEL-SYNC":
log_info("Granting user access to channel A sync function access()")
# Grant via access() in sync_function, then id 'channel_access' will trigger an access(doc.users, doc.channels)
access_doc = document.create_doc("channel_access", channels=["A"])
access_doc["users"] = ["USER_B"]
client.add_doc(url=sg_url, db=sg_db, doc=access_doc, auth=admin_session)
elif grant_type == "ROLE-REST":
log_info("Granting user access to channel A via Admin REST role grant")
# Create role with channel A
client.create_role(url=sg_admin_url, db=sg_db, name="channel-A-role", channels=["A"])
client.update_user(url=sg_admin_url, db=sg_db, name="USER_B", roles=["channel-A-role"])
elif grant_type == "ROLE-SYNC":
log_info("Granting user access to channel A via sync function role() grant")
# Create role with channel A
client.create_role(url=sg_admin_url, db=sg_db, name="channel-A-role", channels=["A"])
# Grant via role() in sync_function, then id 'role_access' will trigger an role(doc.users, doc.roles)
role_access_doc = document.create_doc("role_access")
role_access_doc["users"] = ["USER_B"]
role_access_doc["roles"] = ["role:channel-A-role"]
client.add_doc(sg_url, db=sg_db, doc=role_access_doc, auth=admin_session)
else:
pytest.fail("Unsupported grant_type!!!!")
user_b_changes_after_grant = client.get_changes(url=sg_url, db=sg_db,
since=user_b_changes["last_seq"], auth=user_b_session, feed="normal")
# User B shoud have recieved 51 docs (a_docs + 1 _user/USER_B doc) if a REST grant or 50 changes if the grant
# is via the sync function
changes_results = user_b_changes_after_grant["results"]
assert 50 <= len(changes_results) <= 51
# Create a dictionary of id rev pair of all the docs that are not "_user/" docs from changes
ids_and_revs_from_user_changes = {
change["id"]: change["changes"][0]["rev"]
for change in changes_results if not change["id"].startswith("_user/")
}
assert len(ids_and_revs_from_user_changes) == 50
# Create a list of id rev pair of all of the channel A docs
ids_and_revs_from_a_docs = {doc["id"]: doc["rev"] for doc in a_docs}
assert len(ids_and_revs_from_a_docs) == 50
# Check that the changes and the a_docs are identical in id and rev
assert ids_and_revs_from_user_changes == ids_and_revs_from_a_docs
# Get changes from last_seq of the changes request after the grant. There should be no new changes
user_b_changes = client.get_changes(url=sg_url, db=sg_db,
since=user_b_changes_after_grant["last_seq"], auth=user_b_session, feed="normal")
assert len(user_b_changes["results"]) == 0
def test_string_expiry_as_unix_date(params_from_base_test_setup, sg_conf_name):
"""
1. Calculate (server time + 3 seconds) as unix time (i.e. Epoch time, e.g. 1466465122)
2. PUT /db/doc1 via SG with property "_exp":"[unix time]"
PUT /db/doc2 via SG with property "_exp":"1767225600" (Jan 1 2026) Note: the maximum epoch time supported by CBS is maxUint32, or Sun 07 Feb 2106, in case you want to move it out further than 2026.
3. Wait five seconds
4. Get /db/doc1. Assert response is 404
Get /db/doc2. Assert response is 200
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_string_expiry_as_unix_date'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
time_util = Time()
unix_time_3s_ahead = time_util.get_unix_timestamp(delta=3)
# Convert unix timestamp to string
unix_time_3s_ahead_string = str(unix_time_3s_ahead)
# Using string representation for unix time
doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=unix_time_3s_ahead_string, channels=sg_user_channels)
doc_exp_years_body = document.create_doc(doc_id="exp_years", expiry="1767225600", channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session)
doc_exp_years = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_years_body, auth=sg_user_session)
# Sleep should allow doc_exp_3 to expire
time.sleep(10)
# doc_exp_3 should be expired
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session)
assert he.value[0].startswith("404 Client Error: Not Found for url:")
# doc_exp_years should be available still
doc_exp_years_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_years["id"], auth=sg_user_session)
assert doc_exp_years_result["_id"] == "exp_years"
def test_string_expiry_as_unix_date(params_from_base_test_setup, sg_conf_name):
"""
1. Calculate (server time + 3 seconds) as unix time (i.e. Epoch time, e.g. 1466465122)
2. PUT /db/doc1 via SG with property "_exp":"[unix time]"
PUT /db/doc2 via SG with property "_exp":"1767225600" (Jan 1 2026) Note: the maximum epoch time supported by CBS is maxUint32, or Sun 07 Feb 2106, in case you want to move it out further than 2026.
3. Wait five seconds
4. Get /db/doc1. Assert response is 404
Get /db/doc2. Assert response is 200
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_string_expiry_as_unix_date'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
bucket_name = "data-bucket"
cbs_ip = host_for_url(cbs_url)
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name),
password='******')
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
time_util = Time()
unix_time_3s_ahead = time_util.get_unix_timestamp(delta=3)
# Convert unix timestamp to string
unix_time_3s_ahead_string = str(unix_time_3s_ahead)
# Using string representation for unix time
doc_exp_3_body = document.create_doc(doc_id="exp_3",
expiry=unix_time_3s_ahead_string,
channels=sg_user_channels)
doc_exp_years_body = document.create_doc(doc_id="exp_years",
expiry="1767225600",
channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_3_body,
auth=sg_user_session)
doc_exp_years = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_years_body,
auth=sg_user_session)
# Sleep should allow doc_exp_3 to expire
time.sleep(10)
# doc_exp_3 should be expired
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3["id"],
auth=sg_user_session)
# In XATTR mode, the expiry results in a tombstone
# In Doc Meta mode, the expiry results in a purge
if xattrs_enabled:
assert he.value[0].startswith("403 Client Error: Forbidden for url:")
else:
assert he.value[0].startswith("404 Client Error: Not Found for url:")
verify_doc_deletion_on_server(doc_id=doc_exp_3["id"],
sdk_client=sdk_client,
sg_client=client,
sg_admin_url=sg_url_admin,
sg_db=sg_db,
xattrs_enabled=xattrs_enabled)
# doc_exp_years should be available still
doc_exp_years_result = client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_years["id"],
auth=sg_user_session)
assert doc_exp_years_result["_id"] == "exp_years"
def test_server_goes_down_rebuild_channels(params_from_base_test_setup):
"""
1. Start with a two node couchbase server cluster
2. Starting adding docs
3. Kill one of the server nodes and signal completion
4. Stop adding docs
5. Verify that that the expected docs are present and in the changes feed.
6. Start server again and add to cluster
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
cluster_helper = ClusterKeywords()
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf_path)
topology = cluster_helper.get_cluster_topology(cluster_config)
admin_sg = topology["sync_gateways"][0]["admin"]
sg_url = topology["sync_gateways"][0]["public"]
coucbase_servers = topology["couchbase_servers"]
cbs_one_url = coucbase_servers[0]
cbs_two_url = coucbase_servers[1]
log_info("Running: 'test_server_goes_down_sanity'")
log_info("cluster_config: {}".format(cluster_config))
log_info("admin_sg: {}".format(admin_sg))
log_info("sg_url: {}".format(sg_url))
log_info("cbs_one_url: {}".format(cbs_one_url))
log_info("cbs_two_url: {}".format(cbs_two_url))
sg_db = "db"
num_docs = 100
admin_user_info = userinfo.UserInfo(
name="admin",
password="******",
channels=["ABC"],
roles=[]
)
seth_user_info = userinfo.UserInfo(
name="seth",
password="******",
channels=["ABC"],
roles=[]
)
client = MobileRestClient()
main_server = CouchbaseServer(cbs_one_url)
flakey_server = CouchbaseServer(cbs_two_url)
admin_auth = client.create_user(
admin_sg,
sg_db,
admin_user_info.name,
admin_user_info.password,
channels=admin_user_info.channels
)
client.create_user(
admin_sg,
sg_db,
seth_user_info.name,
seth_user_info.password,
channels=seth_user_info.channels
)
seth_session = client.create_session(admin_sg, sg_db, seth_user_info.name)
# allow any user docs to make it to changes
initial_changes = client.get_changes(url=sg_url, db=sg_db, since=0, auth=seth_session)
# push docs from admin
docs = client.add_docs(
url=sg_url,
db=sg_db,
number=num_docs,
id_prefix=None,
channels=admin_user_info.channels,
auth=admin_auth
)
assert len(docs) == num_docs
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=docs, auth=seth_session)
changes_before_failover = client.get_changes(url=sg_url, db=sg_db, since=initial_changes["last_seq"], auth=seth_session)
assert len(changes_before_failover["results"]) == num_docs
# Stop server via 'service stop'
flakey_server.stop()
start = time.time()
while True:
# Fail tests if all docs do not succeed before timeout
if (time.time() - start) > 60:
# Bring server back up before failing the test
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
raise keywords.exceptions.TimeoutError("Failed to rebuild changes")
try:
# Poll until failover happens (~30 second)
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=docs, auth=seth_session)
# changes requests succeeded, exit loop
break
except requests.exceptions.HTTPError:
# Changes will fail until failover of the down server happens. Wait and try again.
log_info("/db/_changes failed due to server down. Retrying ...")
time.sleep(1)
# Verify no new changes
changes = client.get_changes(
url=sg_url,
db=sg_db,
since=changes_before_failover["last_seq"],
auth=seth_session,
feed="normal"
)
assert len(changes["results"]) == 0
# Check that all changes are intact from initial changes request
changes = client.get_changes(url=sg_url, db=sg_db, since=initial_changes["last_seq"], auth=seth_session)
assert len(changes["results"]) == num_docs
coucbase_servers = topology["couchbase_servers"]
# Test succeeded without timeout, bring server back into topology
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
def test_numeric_expiry_as_ttl(params_from_base_test_setup, sg_conf_name):
"""
1. PUT /db/doc1 via SG with property "_exp":3
PUT /db/doc2 via SG with property "_exp":10
2. Wait five seconds
3. Get /db/doc1. Assert response is 404
Get /db/doc2. Assert response is 200
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_numeric_expiry_as_ttl'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
bucket_name = "data-bucket"
cbs_ip = host_for_url(cbs_url)
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name),
password='******')
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
doc_exp_3_body = document.create_doc(doc_id="exp_3",
expiry=3,
channels=sg_user_channels)
doc_exp_10_body = document.create_doc(doc_id="exp_10",
expiry=10,
channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_3_body,
auth=sg_user_session)
doc_exp_10 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_10_body,
auth=sg_user_session)
# Sleep should allow doc_exp_3 to expire, but still be in the window to get doc_exp_10
time.sleep(5)
# doc_exp_3 should be expired
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3["id"],
auth=sg_user_session)
# In XATTR mode, the expiry results in a tombstone
# In Doc Meta mode, the expiry results in a purge
if xattrs_enabled:
assert he.value[0].startswith("403 Client Error: Forbidden for url:")
else:
assert he.value[0].startswith("404 Client Error: Not Found for url:")
verify_doc_deletion_on_server(doc_id=doc_exp_3["id"],
sdk_client=sdk_client,
sg_client=client,
sg_admin_url=sg_url_admin,
sg_db=sg_db,
xattrs_enabled=xattrs_enabled)
# doc_exp_10 should be available still
doc_exp_10_result = client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_10["id"],
auth=sg_user_session)
assert doc_exp_10_result["_id"] == "exp_10"
def test_rebalance_sanity(params_from_base_test_setup):
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
cluster_helper = ClusterKeywords()
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf_path)
topology = cluster_helper.get_cluster_topology(cluster_config)
admin_sg_one = topology["sync_gateways"][0]["admin"]
sg_one_url = topology["sync_gateways"][0]["public"]
cluster_servers = topology["couchbase_servers"]
cbs_one_url = cluster_servers[0]
cbs_two_url = cluster_servers[1]
log_info("Running: 'test_distributed_index_rebalance_sanity'")
log_info("cluster_config: {}".format(cluster_config))
log_info("admin_sg: {}".format(admin_sg_one))
log_info("sg_url: {}".format(sg_one_url))
log_info("cbs_one_url: {}".format(cbs_one_url))
log_info("cbs_two_url: {}".format(cbs_two_url))
sg_db = "db"
num_docs = 100
num_updates = 100
sg_user_name = "seth"
sg_user_password = "******"
channels = ["ABC", "CBS"]
client = MobileRestClient()
cb_server = CouchbaseServer(cbs_one_url)
server_to_remove = CouchbaseServer(cbs_two_url)
client.create_user(admin_sg_one, sg_db, sg_user_name, sg_user_password, channels=channels)
session = client.create_session(admin_sg_one, sg_db, sg_user_name)
with concurrent.futures.ThreadPoolExecutor(5) as executor:
# Add docs to sg
log_info("Adding docs to sync_gateway")
docs = client.add_docs(sg_one_url, sg_db, num_docs, "test_doc", channels=channels, auth=session)
assert len(docs) == num_docs
# Start updating docs and rebalance out one CBS node
log_info("Updating docs on sync_gateway")
update_docs_task = executor.submit(client.update_docs, sg_one_url, sg_db, docs, num_updates, auth=session)
# Run rebalance in background
cb_server.rebalance_out(cluster_servers, server_to_remove)
updated_docs = update_docs_task.result()
log_info(updated_docs)
# Verify docs / revisions present
client.verify_docs_present(sg_one_url, sg_db, updated_docs, auth=session)
# Verify docs revisions in changes feed
client.verify_docs_in_changes(sg_one_url, sg_db, updated_docs, auth=session)
# Rebalance Server back in to the pool
cb_server.add_node(server_to_remove)
cb_server.rebalance_in(cluster_servers, server_to_remove)
def test_rolling_ttl_remove_expirary(params_from_base_test_setup,
sg_conf_name):
"""
1. PUT /db/doc1 via SG with property "_exp":3
2. Once per second for 10 seconds, update /db/doc1 with a new revision (also with "_exp":3)
3. Update /db/doc1 with a revision with no expiry
3. Get /db/doc1. Assert response is 200
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_rolling_ttl_remove_expirary'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
bucket_name = "data-bucket"
cbs_ip = host_for_url(cbs_url)
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name),
password='******')
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
doc_exp_3_body = document.create_doc(doc_id="exp_3",
expiry=3,
channels=sg_user_channels)
doc_exp_10_body = document.create_doc(doc_id="exp_10",
expiry=10,
channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_3_body,
auth=sg_user_session)
doc_exp_10 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_10_body,
auth=sg_user_session)
client.update_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3["id"],
number_updates=10,
expiry=3,
delay=1,
auth=sg_user_session)
client.update_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3["id"],
number_updates=1,
auth=sg_user_session)
# If expiry was not removed in the last update, this would expire doc_exp_3
time.sleep(5)
# doc_exp_3 should still be around due to removal of expiry
doc_exp_3 = client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3["id"],
auth=sg_user_session)
assert doc_exp_3["_id"] == "exp_3"
# doc_exp_10 should be expired due to the updates (10s) + sleep (5s)
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_10["id"],
auth=sg_user_session)
# In XATTR mode, the expiry results in a tombstone
# In Doc Meta mode, the expiry results in a purge
if xattrs_enabled:
assert he.value[0].startswith("403 Client Error: Forbidden for url:")
else:
assert he.value[0].startswith("404 Client Error: Not Found for url:")
verify_doc_deletion_on_server(doc_id=doc_exp_10["id"],
sdk_client=sdk_client,
sg_client=client,
sg_admin_url=sg_url_admin,
sg_db=sg_db,
xattrs_enabled=xattrs_enabled)
def test_server_goes_down_sanity(params_from_base_test_setup):
"""
1. Start with a two node couchbase server cluster
2. Starting adding docs
3. Kill one of the server nodes and signal completion
4. Stop adding docs
5. Verify that that the expected docs are present and in the changes feed.
6. Start server again and add to cluster
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
cluster_helper = ClusterKeywords()
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf_path)
topology = cluster_helper.get_cluster_topology(cluster_config)
admin_sg = topology["sync_gateways"][0]["admin"]
sg_url = topology["sync_gateways"][0]["public"]
coucbase_servers = topology["couchbase_servers"]
cbs_one_url = coucbase_servers[0]
cbs_two_url = coucbase_servers[1]
log_info("Running: 'test_server_goes_down_sanity'")
log_info("cluster_config: {}".format(cluster_config))
log_info("admin_sg: {}".format(admin_sg))
log_info("sg_url: {}".format(sg_url))
log_info("cbs_one_url: {}".format(cbs_one_url))
log_info("cbs_two_url: {}".format(cbs_two_url))
sg_db = "db"
num_docs = 100
sg_user_name = "seth"
sg_user_password = "******"
channels = ["ABC", "CBS"]
client = MobileRestClient()
main_server = CouchbaseServer(cbs_one_url)
flakey_server = CouchbaseServer(cbs_two_url)
client.create_user(admin_sg, sg_db, sg_user_name, sg_user_password, channels=channels)
session = client.create_session(admin_sg, sg_db, sg_user_name)
# Stop second server
flakey_server.stop()
# Try to add 100 docs in a loop until all succeed, if the never do, fail with timeout
errors = num_docs
# Wait 30 seconds for auto failover
# (Minimum value suggested - http://docs.couchbase.com/admin/admin/Tasks/tasks-nodeFailover.html)
# + 15 seconds to add docs
timeout = 45
start = time.time()
successful_add = False
while not successful_add:
# Fail tests if all docs do not succeed before timeout
if (time.time() - start) > timeout:
# Bring server back up before failing the test
flakey_server.start()
main_server.rebalance_in(coucbase_servers, flakey_server)
raise TimeoutError("Failed to successfully put docs before timeout")
try:
docs = client.add_docs(url=sg_url, db=sg_db, number=num_docs, id_prefix=None, auth=session, channels=channels)
# If the above add doc does not throw, it was a successfull add.
successful_add = True
except requests.exceptions.HTTPError as he:
log_info("Failed to add docs: {}".format(he))
log_info("Seeing: {} errors".format(errors))
time.sleep(1)
assert len(docs) == 100
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=docs, auth=session)
try:
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=docs, auth=session, polling_interval=5)
except keywords.exceptions.TimeoutException:
# timeout verifying docs. Bring server back in to restore topology, then fail
# Failing due to https://github.com/couchbase/sync_gateway/issues/2197
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
raise keywords.exceptions.TimeoutException("Failed to get all changes")
# Test succeeded without timeout, bring server back into topology
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
# Make sure all docs were not added before server was
log_info("test_server_goes_down_sanity complete!")
def test_initial_push_replication(setup_client_syncgateway_test, continuous):
"""
1. Prepare LiteServ to have 10000 documents.
2. Create a single shot push / continuous replicator and to push the docs into a sync_gateway database.
3. Verify if all of the docs get pushed.
"""
sg_db = "db"
ls_db = "ls_db"
seth_channels = ["ABC", "NBC"]
num_docs = 10000
cluster_config = setup_client_syncgateway_test["cluster_config"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_one_admin = setup_client_syncgateway_test["sg_admin_url"]
sg_one_public = setup_client_syncgateway_test["sg_url"]
sg_helper = SyncGateway()
sg_helper.start_sync_gateway(
cluster_config=cluster_config,
url=sg_one_public,
config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS)
)
log_info("Running 'test_initial_push_replication', continuous: {}".format(continuous))
log_info("ls_url: {}".format(ls_url))
log_info("sg_one_admin: {}".format(sg_one_admin))
log_info("sg_one_public: {}".format(sg_one_public))
client = MobileRestClient()
client.create_user(sg_one_admin, sg_db, "seth", password="******", channels=seth_channels)
session = client.create_session(sg_one_admin, sg_db, "seth")
client.create_database(url=ls_url, name=ls_db)
# Create 'num_docs' docs on LiteServ
docs = client.add_docs(
url=ls_url,
db=ls_db,
number=num_docs,
id_prefix="seeded_doc",
generator="four_k",
channels=seth_channels
)
assert len(docs) == num_docs
# Start push replication
repl_id = client.start_replication(
url=ls_url,
continuous=continuous,
from_db=ls_db,
to_url=sg_one_admin,
to_db=sg_db
)
if continuous:
log_info("Waiting for replication status 'Idle' for: {}".format(repl_id))
client.wait_for_replication_status_idle(ls_url, repl_id)
else:
log_info("Waiting for no replications: {}".format(repl_id))
client.wait_for_no_replications(ls_url)
# Verify docs replicated to sync_gateway
client.verify_docs_present(url=sg_one_public, db=sg_db, expected_docs=docs, auth=session)
# Verify docs show up in sync_gateway's changes feed
client.verify_docs_in_changes(url=sg_one_public, db=sg_db, expected_docs=docs, auth=session)
replications = client.get_replications(url=ls_url)
if continuous:
assert len(replications) == 1, "There should only be one replication running"
assert replications[0]["status"] == "Idle", "Replication Status should be 'Idle'"
assert replications[0]["continuous"], "Running replication should be continuous"
# Only .NET has an 'error' property
if "error" in replications[0]:
assert len(replications[0]["error"]) == 0
else:
assert len(replications) == 0, "No replications should be running"
def test_winning_conflict_branch_revisions(params_from_base_test_setup,
sg_conf_name):
""" Add winning conflict revisions to the revision tree and ensure
that the changes feed returns the correct revisions
Steps:
- Add a doc ('test_doc')
- Add 5 revs to 'test_doc'
- POST _changes, assert rev starts with "6-" from 0, store "last_seq_1"
- Create a conflict off first revision ("2-foo") (POST docs, new_edits == false)
- Append 5 revisions to the conflicting branch
(3-foo with 2-foo as parent, 4-foo with 3-foo as parent ... 7-foo with 6-foo as parent)
- GET 'test_doc' and verify that the rev is '7-foo'
- POST _changes, assert returns 7-foo
"""
cluster_config = params_from_base_test_setup["cluster_config"]
topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
no_conflicts_enabled = params_from_base_test_setup["no_conflicts_enabled"]
if no_conflicts_enabled:
pytest.skip(
'--no-conflicts is enabled, this test needs to create conflicts, so skipping the test'
)
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
c = cluster.Cluster(cluster_config)
c.reset(sg_conf)
client = MobileRestClient()
seth_user_info = userinfo.UserInfo(name="seth",
password="******",
channels=["NATGEO"],
roles=[])
seth_auth = client.create_user(url=sg_admin_url,
db=sg_db,
name=seth_user_info.name,
password=seth_user_info.password,
channels=seth_user_info.channels)
test_doc_body = document.create_doc(doc_id="test_doc",
channels=seth_user_info.channels)
rev_gen_1_doc = client.add_doc(url=sg_url,
db=sg_db,
doc=test_doc_body,
auth=seth_auth)
rev_gen_6_doc = client.update_doc(url=sg_url,
db=sg_db,
doc_id=rev_gen_1_doc["id"],
number_updates=5,
auth=seth_auth)
assert rev_gen_6_doc["rev"].startswith("6-")
# Wait until doc shows up in changes feed
last_seq = 0
start = time.time()
while True:
if time.time() - start > keywords.constants.CLIENT_REQUEST_TIMEOUT:
raise keywords.exceptions.TimeoutError(
"Wait for Replication Status Idle: TIMEOUT")
changes_1 = client.get_changes(url=sg_url,
db=sg_db,
since=last_seq,
auth=seth_auth,
skip_user_docs=True)
last_seq = changes_1["last_seq"]
if len(changes_1["results"]) > 0 and changes_1["results"][0][
"changes"][0]["rev"].startswith("6-"):
break
assert len(changes_1["results"]) == 1
assert changes_1["results"][0]["id"] == "test_doc"
assert changes_1["results"][0]["changes"][0]["rev"].startswith("6-")
# Create a conflict off of rev one
rev_gen_1_doc_conflict = client.add_conflict(
url=sg_url,
db=sg_db,
doc_id=rev_gen_1_doc["id"],
parent_revisions=rev_gen_1_doc["rev"],
new_revision="2-foo",
auth=seth_auth)
# Update the conflicting branch 5x
rev_gen = 3
for _ in range(5):
rev_gen_1_doc_conflict = client.add_conflict(
url=sg_url,
db=sg_db,
doc_id=rev_gen_1_doc["id"],
parent_revisions=rev_gen_1_doc_conflict["rev"],
new_revision="{}-foo".format(rev_gen),
auth=seth_auth)
rev_gen += 1
# Wait until doc shows up in changes feed from last_seq from where last changes loop from above left off
start = time.time()
while True:
if time.time() - start > keywords.constants.CLIENT_REQUEST_TIMEOUT:
raise keywords.exceptions.TimeoutError(
"Wait for Replication Status Idle: TIMEOUT")
changes_2 = client.get_changes(url=sg_url,
db=sg_db,
since=last_seq,
auth=seth_auth)
last_seq = changes_2["last_seq"]
if changes_2["results"][0]["changes"][0]["rev"] == "7-foo" and len(
changes_2["results"]) > 0:
break
# Verify that the the "7-foo" rev is return on the changes feed
assert len(changes_2["results"]) == 1
assert changes_2["results"][0]["id"] == "test_doc"
assert changes_2["results"][0]["changes"][0]["rev"] == "7-foo"
def test_client_to_sync_gateway_complex_replication_with_revs_limit(setup_client_syncgateway_test):
""" Ported from sync_gateway tests repo
... 1. Clear server buckets
... 2. Restart liteserv with _session
... 3. Restart sync_gateway wil that config
... 4. Create db on LiteServ
... 5. Add numDocs to LiteServ db
... 6. Setup push replication from LiteServ db to sync_gateway
... 7. Verify doc present on sync_gateway (number of docs)
... 8. Update sg docs numRevs * 4 = 480
... 9. Update docs on LiteServ db numRevs * 4 = 480
... 10. Setup pull replication from sg -> liteserv db
... 11. Verify all docs are replicated
... 12. compact LiteServ db (POST _compact)
... 13. Verify number of revs in LiteServ db (?revs_info=true) check rev status == available fail if revs available > revs limit
... 14. Delete LiteServ db conflicts (?conflicts=true) DELETE _conflicts
... 15. Create numDoc number of docs in LiteServ db
... 16. Update LiteServ db docs numRevs * 5 (600)
... 17. Verify LiteServ db revs is < 602
... 18. Verify LiteServ db docs revs prefix (9 * numRevs + 3)
... 19. Compact LiteServ db
... 20. Verify number of revs <= 10
... 21. Delete LiteServ docs
... 22. Delete Server bucket
... 23. Delete LiteServ db
"""
ls_db_name = "ls_db"
sg_db = "db"
sg_user_name = "sg_user"
num_docs = 10
num_revs = 100
cluster_config = setup_client_syncgateway_test["cluster_config"]
ls_url = setup_client_syncgateway_test["ls_url"]
sg_url = setup_client_syncgateway_test["sg_url"]
sg_admin_url = setup_client_syncgateway_test["sg_admin_url"]
sg_helper = SyncGateway()
sg_helper.start_sync_gateway(
cluster_config=cluster_config,
url=sg_url,
config="{}/walrus-revs-limit.json".format(SYNC_GATEWAY_CONFIGS)
)
log_info("Running 'test_client_to_sync_gateway_complex_replication_with_revs_limit'")
log_info("ls_url: {}".format(ls_url))
log_info("sg_admin_url: {}".format(sg_admin_url))
log_info("sg_url: {}".format(sg_url))
client = MobileRestClient()
# Test the endpoint, listener does not support users but should have a default response
client.get_session(url=ls_url)
sg_user_channels = ["NBC"]
client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels)
sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name)
ls_db = client.create_database(url=ls_url, name=ls_db_name)
ls_db_docs = client.add_docs(url=ls_url, db=ls_db, number=num_docs, id_prefix=ls_db, channels=sg_user_channels)
assert len(ls_db_docs) == num_docs
# Start replication ls_db -> sg_db
repl_one = client.start_replication(
url=ls_url,
continuous=True,
from_db=ls_db,
to_url=sg_admin_url, to_db=sg_db
)
client.verify_docs_present(url=sg_admin_url, db=sg_db, expected_docs=ls_db_docs)
# Delay is to the updates here due to couchbase/couchbase-lite-ios#1277.
# Basically, if your revs depth is small and someone is updating a doc past the revs depth before a push replication,
# the push replication will have no common ancestor with sync_gateway causing conflicts to be created.
# Adding a delay between updates helps this situation. There is an alternative for CBL mac and CBL NET to change the default revs client depth
# but that is not configurable for Android.
# Currently adding a delay will allow the replication to act as expected for all platforms now.
client.update_docs(url=sg_url, db=sg_db, docs=ls_db_docs, number_updates=num_revs, delay=0.1, auth=sg_session)
client.update_docs(url=ls_url, db=ls_db, docs=ls_db_docs, number_updates=num_revs, delay=0.1)
# Start replication ls_db <- sg_db
repl_two = client.start_replication(
url=ls_url,
continuous=True,
from_url=sg_admin_url, from_db=sg_db,
to_db=ls_db
)
client.wait_for_replication_status_idle(url=ls_url, replication_id=repl_one)
client.wait_for_replication_status_idle(url=ls_url, replication_id=repl_two)
client.compact_database(url=ls_url, db=ls_db)
# LiteServ should only have 20 revisions due to built in client revs limit
client.verify_revs_num_for_docs(url=ls_url, db=ls_db, docs=ls_db_docs, expected_revs_per_doc=20)
# Sync Gateway should have 100 revisions due to the specified revs_limit in the sg config and possible conflict winners from the liteserv db
client.verify_max_revs_num_for_docs(url=sg_url, db=sg_db, docs=ls_db_docs, expected_max_number_revs_per_doc=100, auth=sg_session)
client.delete_conflicts(url=ls_url, db=ls_db, docs=ls_db_docs)
expected_generation = num_revs + 1
client.verify_docs_rev_generations(url=ls_url, db=ls_db, docs=ls_db_docs, expected_generation=expected_generation)
client.verify_docs_rev_generations(url=sg_url, db=sg_db, docs=ls_db_docs, expected_generation=expected_generation, auth=sg_session)
client.delete_docs(url=ls_url, db=ls_db, docs=ls_db_docs)
client.verify_docs_deleted(url=ls_url, db=ls_db, docs=ls_db_docs)
client.verify_docs_deleted(url=sg_admin_url, db=sg_db, docs=ls_db_docs)
ls_db_docs = client.add_docs(url=ls_url, db=ls_db, number=num_docs, id_prefix=ls_db, channels=sg_user_channels)
assert len(ls_db_docs) == 10
expected_revs = num_revs + 20 + 2
client.update_docs(url=ls_url, db=ls_db, docs=ls_db_docs, delay=0.1, number_updates=num_revs)
client.verify_max_revs_num_for_docs(url=ls_url, db=ls_db, docs=ls_db_docs, expected_max_number_revs_per_doc=expected_revs)
expected_generation = (num_revs * 2) + 3
client.verify_docs_rev_generations(url=ls_url, db=ls_db, docs=ls_db_docs, expected_generation=expected_generation)
client.compact_database(url=ls_url, db=ls_db)
client.verify_revs_num_for_docs(url=ls_url, db=ls_db, docs=ls_db_docs, expected_revs_per_doc=20)
client.stop_replication(
url=ls_url,
continuous=True,
from_db=ls_db,
to_url=sg_admin_url, to_db=sg_db
)
client.stop_replication(
url=ls_url,
continuous=True,
from_url=sg_admin_url, from_db=sg_db,
to_db=ls_db
)
client.wait_for_no_replications(url=ls_url)
client.delete_conflicts(url=ls_url, db=ls_db, docs=ls_db_docs)
client.delete_conflicts(url=sg_url, db=sg_db, docs=ls_db_docs, auth=sg_session)
client.delete_docs(url=ls_url, db=ls_db, docs=ls_db_docs)
# Start push pull and verify that all docs are deleted
# Start replication ls_db -> sg_db
repl_one = client.start_replication(
url=ls_url,
continuous=True,
from_db=ls_db,
to_url=sg_admin_url, to_db=sg_db
)
# Start replication ls_db <- sg_db
repl_two = client.start_replication(
url=ls_url,
continuous=True,
from_url=sg_admin_url, from_db=sg_db,
to_db=ls_db
)
client.verify_docs_deleted(url=ls_url, db=ls_db, docs=ls_db_docs)
client.verify_docs_deleted(url=sg_admin_url, db=sg_db, docs=ls_db_docs)
def test_non_winning_revisions(params_from_base_test_setup, sg_conf_name):
""" Add non-winning revisions to the revision tree and ensure
that the changes feed returns the correct revisions
Steps:
- Add a doc
- Add 5 revs
- changes, assert rev starts with "6-" from 0, store "last_seq_1"
- Create a conflict off first revision ("2-foo") (POST docs, new_edits == false)
- changes, assert rev starts with "6-" from "last_seq_1", store "last_seq_2"
- changes, assert rev starts with "6-" from 0
- Add a "3-foo" rev with rev "2-foo" as parent
- changes, assert rev starts with "6-" from "last_seq_2", store "last_seq_3"
- changes, assert rev starts with "6-" from 0
- add tombstone rev as child of "6-" i.e. issue delete on ("6-")
- changes, assert rev starts with "3-foo" from "last_seq_3"
- changes, assert rev starts with "3-foo" from 0
"""
cluster_config = params_from_base_test_setup["cluster_config"]
topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
no_conflicts_enabled = params_from_base_test_setup["no_conflicts_enabled"]
if no_conflicts_enabled:
pytest.skip(
'--no-conflicts is enabled, this test needs to create conflicts, so skipping the test'
)
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
c = cluster.Cluster(cluster_config)
c.reset(sg_conf)
client = MobileRestClient()
seth_user_info = userinfo.UserInfo(name="seth",
password="******",
channels=["NATGEO"],
roles=[])
seth_auth = client.create_user(url=sg_admin_url,
db=sg_db,
name=seth_user_info.name,
password=seth_user_info.password,
channels=seth_user_info.channels)
test_doc_body = document.create_doc(doc_id="test_doc",
channels=seth_user_info.channels)
rev_gen_1_doc = client.add_doc(url=sg_url,
db=sg_db,
doc=test_doc_body,
auth=seth_auth)
rev_gen_6_doc = client.update_doc(url=sg_url,
db=sg_db,
doc_id=rev_gen_1_doc["id"],
number_updates=5,
auth=seth_auth)
assert rev_gen_6_doc["rev"].startswith("6-")
# Get changes until rev generation 6 document shows up
start = time.time()
last_seq = 0
while True:
if time.time() - start > keywords.constants.CLIENT_REQUEST_TIMEOUT:
raise keywords.exceptions.TimeoutError(
"Wait for Replication Status Idle: TIMEOUT")
changes_1 = client.get_changes(url=sg_url,
db=sg_db,
since=last_seq,
auth=seth_auth,
skip_user_docs=True)
last_seq = changes_1["last_seq"]
# break when expected rev shows up in changes feed
if changes_1["results"] and changes_1["results"][0]["changes"][0][
"rev"].startswith("6-"):
break
assert len(changes_1["results"]) == 1
assert changes_1["results"][0]["id"] == "test_doc"
assert changes_1["results"][0]["changes"][0]["rev"].startswith("6-")
# Create a conflict off of rev one
rev_gen_2_doc_conflict = client.add_conflict(
url=sg_url,
db=sg_db,
doc_id=rev_gen_1_doc["id"],
parent_revisions=rev_gen_1_doc["rev"],
new_revision="2-foo",
auth=seth_auth)
assert rev_gen_2_doc_conflict["id"] == "test_doc"
assert rev_gen_2_doc_conflict["rev"] == "2-foo"
# Issue changes since changes_1 last_seq above
changes_2 = client.get_changes(url=sg_url,
db=sg_db,
since=changes_1["last_seq"],
auth=seth_auth)
assert len(changes_2["results"]) == 1
assert changes_2["results"][0]["id"] == "test_doc"
assert changes_2["results"][0]["changes"][0]["rev"].startswith("6-")
# Issue changes since 0, strip user doc and make sure the doc is still the '6-' rev
changes_from_0_one = client.get_changes(url=sg_url,
db=sg_db,
since=0,
auth=seth_auth,
skip_user_docs=True)
assert len(changes_from_0_one["results"]) == 1
assert changes_from_0_one["results"][0]["id"] == "test_doc"
assert changes_from_0_one["results"][0]["changes"][0]["rev"].startswith(
"6-")
# Create a 3-foo rev with 2-foo as the parent
rev_gen_3_doc_conflict = client.add_conflict(
url=sg_url,
db=sg_db,
doc_id=rev_gen_2_doc_conflict["id"],
parent_revisions=rev_gen_2_doc_conflict["rev"],
new_revision="3-foo",
auth=seth_auth)
assert rev_gen_3_doc_conflict["id"] == "test_doc"
assert rev_gen_3_doc_conflict["rev"] == "3-foo"
# Issue changes since changes_2 last_seq above
changes_3 = client.get_changes(url=sg_url,
db=sg_db,
since=changes_2["last_seq"],
auth=seth_auth)
assert len(changes_3["results"]) == 1
assert changes_3["results"][0]["id"] == "test_doc"
assert changes_3["results"][0]["changes"][0]["rev"].startswith("6-")
# Issue changes since 0, strip user doc and make sure the doc is still the '6-' rev
changes_from_0_two = client.get_changes(url=sg_url,
db=sg_db,
since=0,
auth=seth_auth,
skip_user_docs=True)
assert len(changes_from_0_two["results"]) == 1
assert changes_from_0_two["results"][0]["id"] == "test_doc"
assert changes_from_0_two["results"][0]["changes"][0]["rev"].startswith(
"6-")
# Delete test_doc at rev 6-*
client.delete_doc(url=sg_url,
db=sg_db,
doc_id=rev_gen_6_doc["id"],
rev=rev_gen_6_doc["rev"],
auth=seth_auth)
# Issue changes since changes_3 last_seq above
changes_4 = client.get_changes(url=sg_url,
db=sg_db,
since=changes_3["last_seq"],
auth=seth_auth)
assert len(changes_4["results"]) == 1
assert changes_4["results"][0]["id"] == "test_doc"
assert changes_4["results"][0]["changes"][0]["rev"] == "3-foo"
# Issue a oneshot changes since changes_4 last_seq and assert no results are returned
changes_5 = client.get_changes(url=sg_url,
db=sg_db,
since=changes_4["last_seq"],
feed="normal",
auth=seth_auth)
assert len(changes_5["results"]) == 0
def test_take_down_bring_up_sg_accel_validate_cbgt(params_from_base_test_setup,
sg_conf):
"""
Scenario 1
Start with 3 sg_accels
Take down 2 sg_accels (block until down -- poll port if needed)
Doc adds with uuids (~30 sec for cbgt to reshard)
polling loop: wait for all docs to come back over changes feed
Call validate pindex with correct number of accels
Scenario 2 (Continuation)
When bringing up, you'd have to poll the cbgt_cfg until you get expected number of nodes,
then you could validate the pindex with 2 accels
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_dcp_reshard_single_sg_accel_goes_down_and_up'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_util = ClusterKeywords()
topology = cluster_util.get_cluster_topology(cluster_conf)
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
client = MobileRestClient()
doc_pusher_user_info = userinfo.UserInfo("doc_pusher",
"pass",
channels=["A"],
roles=[])
doc_pusher_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=doc_pusher_user_info.name,
password=doc_pusher_user_info.password,
channels=doc_pusher_user_info.channels)
log_info("Shutting down sg_accels: [{}, {}]".format(
cluster.sg_accels[1], cluster.sg_accels[2]))
# Shutdown two accel nodes in parallel
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as ex:
sg_accel_down_task_1 = ex.submit(cluster.sg_accels[1].stop)
sg_accel_down_task_2 = ex.submit(cluster.sg_accels[2].stop)
assert sg_accel_down_task_1.result() == 0
assert sg_accel_down_task_2.result() == 0
log_info("Finished taking nodes down!")
# It should take some time ~30 for cbgt to pick up failing nodes and reshard the pindexes. During
# this add a 1000 docs a start a longpoll changes loop to see if those docs make to to the changes feed
# If the reshard is successful they will show up at somepoint after. If not, the docs will fail to show up.
doc_pusher_docs = client.add_docs(url=sg_url,
db=sg_db,
number=1000,
id_prefix=None,
auth=doc_pusher_auth,
channels=doc_pusher_user_info.channels)
assert len(doc_pusher_docs) == 1000
client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=doc_pusher_docs,
auth=doc_pusher_auth,
polling_interval=5)
# The pindexes should be reshared at this point since all of the changes have shown up
assert cluster.validate_cbgt_pindex_distribution(num_running_sg_accels=1)
log_info("Start sg_accels: [{}, {}]".format(cluster.sg_accels[1],
cluster.sg_accels[2]))
# Start two accel nodes in parallel
status = cluster.sg_accels[1].start(sg_conf)
assert status == 0
# Poll on pIndex reshard after bring 2 accel nodes back
assert cluster.validate_cbgt_pindex_distribution_retry(
num_running_sg_accels=2)
status = cluster.sg_accels[2].start(sg_conf)
assert status == 0
# Poll on pIndex reshard after bring 2 accel nodes back
assert cluster.validate_cbgt_pindex_distribution_retry(
num_running_sg_accels=3)
def test_view_backfill_for_deletes(params_from_base_test_setup, sg_conf_name,
validate_changes_before_restart):
"""
Scenario:
1. Write a bunch of docs
2. Delete 1/2
3. Restart Sync Gateway
4. Issue _changes, assert view backfills docs and delete notifications
"""
num_docs = 1000
sg_db = 'db'
cluster_conf = params_from_base_test_setup['cluster_config']
cluster_topology = params_from_base_test_setup['cluster_topology']
mode = params_from_base_test_setup['mode']
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
sg_admin_url = cluster_topology['sync_gateways'][0]['admin']
sg_url = cluster_topology['sync_gateways'][0]['public']
cbs_url = cluster_topology['couchbase_servers'][0]
log_info('sg_conf: {}'.format(sg_conf))
log_info('sg_admin_url: {}'.format(sg_admin_url))
log_info('sg_url: {}'.format(sg_url))
log_info('cbs_url: {}'.format(cbs_url))
log_info('validate_changes_before_restart: {}'.format(
validate_changes_before_restart))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
# Create clients
sg_client = MobileRestClient()
# Create user / session
seth_user_info = UserInfo(name='seth',
password='******',
channels=['NASA', 'NATGEO'],
roles=[])
sg_client.create_user(url=sg_admin_url,
db=sg_db,
name=seth_user_info.name,
password=seth_user_info.password,
channels=seth_user_info.channels)
seth_auth = sg_client.create_session(url=sg_admin_url,
db=sg_db,
name=seth_user_info.name,
password=seth_user_info.password)
# Add 'num_docs' to Sync Gateway
doc_bodies = document.create_docs('test_doc',
number=num_docs,
channels=seth_user_info.channels)
bulk_resp = sg_client.add_bulk_docs(url=sg_url,
db=sg_db,
docs=doc_bodies,
auth=seth_auth)
assert len(bulk_resp) == num_docs
# Delete half of the docs randomly
deleted_docs = []
for _ in range(num_docs / 2):
random_doc = random.choice(bulk_resp)
deleted_doc = sg_client.delete_doc(url=sg_url,
db=sg_db,
doc_id=random_doc['id'],
rev=random_doc['rev'],
auth=seth_auth)
deleted_docs.append(deleted_doc)
bulk_resp.remove(random_doc)
log_info('Number of docs deleted: {}'.format(len(deleted_docs)))
all_docs = bulk_resp + deleted_docs
log_info('Number of docs to look for in changes: {}'.format(len(all_docs)))
# This test will check changes before and after SG restart if
# validate_changes_before_restart == True
# If it is not set to True, only build the changes after restart
if validate_changes_before_restart:
# Verify deletions and inital docs show up in changes feed
sg_client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=all_docs,
auth=seth_auth)
changes = sg_client.get_changes(url=sg_url,
db=sg_db,
since=0,
auth=seth_auth)
# All docs should show up + _user doc
assert len(changes['results']) == num_docs + 1
deleted_doc_ids = [doc['id'] for doc in deleted_docs]
assert len(deleted_doc_ids) == num_docs / 2
deleted_docs_in_changes = [
change['id'] for change in changes['results']
if 'deleted' in change and change['deleted']
]
assert len(deleted_docs_in_changes) == num_docs / 2
# All deleted docs should show up in the changes feed
for doc_id in deleted_docs_in_changes:
assert doc_id in deleted_doc_ids
deleted_doc_ids.remove(doc_id)
assert len(deleted_doc_ids) == 0
# Restart Sync Gateway
sg_controller = SyncGateway()
sg_controller.stop_sync_gateways(url=sg_url, cluster_config=cluster_conf)
sg_controller.start_sync_gateways(url=sg_url,
cluster_config=cluster_conf,
config=sg_conf)
# Verify deletions and inital docs show up in changes feed
sg_client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=all_docs,
auth=seth_auth)
changes = sg_client.get_changes(url=sg_url,
db=sg_db,
since=0,
auth=seth_auth)
# All docs should show up + _user doc
assert len(changes['results']) == num_docs + 1
deleted_doc_ids = [doc['id'] for doc in deleted_docs]
assert len(deleted_doc_ids) == num_docs / 2
deleted_docs_in_changes = [
change['id'] for change in changes['results']
if 'deleted' in change and change['deleted']
]
assert len(deleted_docs_in_changes) == num_docs / 2
# All deleted docs should show up in th changes feed
for doc_id in deleted_docs_in_changes:
assert doc_id in deleted_doc_ids
deleted_doc_ids.remove(doc_id)
assert len(deleted_doc_ids) == 0
def test_detect_stale_channel_index(params_from_base_test_setup, sg_conf):
"""
1. Bring up single Sync Gateway node, backed by Couchbase Server with 3 accels indexing
2. Configure such that the primary bucket and the channel index bucket are different (which is the norm)
3. Add 1000 documents
4. Shutdown Sync Gateway
5. Delete / create the primary bucket ('data-bucket'), but do not touch the channel index bucket
6. Start Sync Gateway
7. Assert that sync_gateway fails to start due to stale channel index
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_detect_stale_channel_index'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_util = ClusterKeywords()
topology = cluster_util.get_cluster_topology(cluster_conf)
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
cb_server_url = topology["couchbase_servers"][0]
sg_db = "db"
num_docs = 1000
cb_server = couchbaseserver.CouchbaseServer(url=cb_server_url)
client = MobileRestClient()
# Create doc pusher user
doc_pusher_user_info = userinfo.UserInfo(name="doc_pusher",
password="******",
channels=["NASA"],
roles=[])
doc_pusher_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=doc_pusher_user_info.name,
password=doc_pusher_user_info.password,
channels=doc_pusher_user_info.channels)
# Add some docs to Sync Gateway to cause indexing
docs = document.create_docs(None,
number=num_docs,
channels=doc_pusher_user_info.channels)
pushed_docs = client.add_bulk_docs(url=sg_url,
db=sg_db,
docs=docs,
auth=doc_pusher_auth)
assert len(pushed_docs) == num_docs
# Shut down sync_gateway
sg_util = SyncGateway()
sg_util.stop_sync_gateways(cluster_config=cluster_conf, url=sg_url)
# Delete server bucket
cb_server.delete_bucket(name="data-bucket")
# Create server bucket
ram_per_bucket_mb = cb_server.get_ram_per_bucket(num_buckets=2)
cb_server.create_bucket(name="data-bucket", ram_quota_mb=ram_per_bucket_mb)
# Start sync_gateway and assert that a Provisioning error is raised due to detecting stale index
with pytest.raises(exceptions.ProvisioningError):
sg_util.start_sync_gateways(cluster_config=cluster_conf,
url=sg_url,
config=sg_conf)
# TODO: To make this check even more accurate, could
# run remote ssh command "systemctl status sync_gateway.service" and look for
# regex pattern: Main PID: 7185 (code=exited, status=2)
# Delete index bucket and recreate it
cb_server.delete_bucket(name="index-bucket")
cb_server.create_bucket(name="index-bucket",
ram_quota_mb=ram_per_bucket_mb)
# Start sync gateway, should succeed now
sg_util.start_sync_gateways(cluster_config=cluster_conf,
url=sg_url,
config=sg_conf)
def test_winning_conflict_branch_revisions(params_from_base_test_setup, sg_conf_name):
""" Add winning conflict revisions to the revision tree and ensure
that the changes feed returns the correct revisions
Steps:
- Create a doc ('test_doc')
- Add 5 revs to 'test_doc'
- POST _changes, assert rev starts with "6-" from 0, store "last_seq_1"
- Create a conflict off first revision ("2-foo") (POST docs, new_edits == false)
- Append 5 revisions to the conflicting branch
(3-foo with 2-foo as parent, 4-foo with 3-foo as parent ... 7-foo with 6-foo as parent)
- GET 'test_doc' and verify that the rev is '7-foo'
- POST _changes, assert returns 7-foo
"""
cluster_config = params_from_base_test_setup["cluster_config"]
topology = params_from_base_test_setup["cluster_topology"]
mode = params_from_base_test_setup["mode"]
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
c = cluster.Cluster(cluster_config)
c.reset(sg_conf)
client = MobileRestClient()
seth_user_info = userinfo.UserInfo(
name="seth",
password="******",
channels=["NATGEO"],
roles=[]
)
seth_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=seth_user_info.name,
password=seth_user_info.password,
channels=seth_user_info.channels
)
test_doc_body = document.create_doc(doc_id="test_doc", channels=seth_user_info.channels)
rev_gen_1_doc = client.add_doc(url=sg_url, db=sg_db, doc=test_doc_body, auth=seth_auth)
rev_gen_6_doc = client.update_doc(url=sg_url, db=sg_db, doc_id=rev_gen_1_doc["id"], number_updates=5, auth=seth_auth)
assert rev_gen_6_doc["rev"].startswith("6-")
# Wait until doc shows up in changes feed
last_seq = 0
start = time.time()
while True:
if time.time() - start > keywords.constants.CLIENT_REQUEST_TIMEOUT:
raise keywords.exceptions.TimeoutError("Wait for Replication Status Idle: TIMEOUT")
changes_1 = client.get_changes(url=sg_url, db=sg_db, since=last_seq, auth=seth_auth, skip_user_docs=True)
last_seq = changes_1["last_seq"]
if len(changes_1["results"]) > 0 and changes_1["results"][0]["changes"][0]["rev"].startswith("6-"):
break
assert len(changes_1["results"]) == 1
assert changes_1["results"][0]["id"] == "test_doc"
assert changes_1["results"][0]["changes"][0]["rev"].startswith("6-")
# Create a conflict off of rev one
rev_gen_1_doc_conflict = client.add_conflict(
url=sg_url,
db=sg_db,
doc_id=rev_gen_1_doc["id"],
parent_revisions=rev_gen_1_doc["rev"],
new_revision="2-foo",
auth=seth_auth
)
# Update the conflicting branch 5x
rev_gen = 3
for _ in range(5):
rev_gen_1_doc_conflict = client.add_conflict(
url=sg_url,
db=sg_db,
doc_id=rev_gen_1_doc["id"],
parent_revisions=rev_gen_1_doc_conflict["rev"],
new_revision="{}-foo".format(rev_gen),
auth=seth_auth
)
rev_gen += 1
# Wait until doc shows up in changes feed from last_seq from where last changes loop from above left off
start = time.time()
while True:
if time.time() - start > keywords.constants.CLIENT_REQUEST_TIMEOUT:
raise keywords.exceptions.TimeoutError("Wait for Replication Status Idle: TIMEOUT")
changes_2 = client.get_changes(url=sg_url, db=sg_db, since=last_seq, auth=seth_auth)
last_seq = changes_2["last_seq"]
if len(changes_2["results"]) > 0:
break
# Verify that the the "7-foo" rev is return on the changes feed
assert len(changes_2["results"]) == 1
assert changes_2["results"][0]["id"] == "test_doc"
assert changes_2["results"][0]["changes"][0]["rev"] == "7-foo"
