def test_rolling_ttl_remove_expirary(params_from_base_test_setup, sg_conf_name): """ 1. PUT /db/doc1 via SG with property "_exp":3 2. Once per second for 10 seconds, update /db/doc1 with a new revision (also with "_exp":3) 3. Update /db/doc1 with a revision with no expiry 3. Get /db/doc1. Assert response is 200 """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) cluster_helper = ClusterKeywords() topology = cluster_helper.get_cluster_topology(cluster_config) cluster_helper.reset_cluster( cluster_config=cluster_config, sync_gateway_config=sg_conf ) cbs_url = topology["couchbase_servers"][0] sg_url = topology["sync_gateways"][0]["public"] sg_url_admin = topology["sync_gateways"][0]["admin"] log_info("Running 'test_rolling_ttl_remove_expirary'") log_info("cbs_url: {}".format(cbs_url)) log_info("sg_url: {}".format(sg_url)) log_info("sg_url_admin: {}".format(sg_url_admin)) sg_db = "db" sg_user_name = "sg_user" sg_user_password = "******" sg_user_channels = ["NBC", "ABC"] client = MobileRestClient() client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels) sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name) doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=3, channels=sg_user_channels) doc_exp_10_body = document.create_doc(doc_id="exp_10", expiry=10, channels=sg_user_channels) doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session) doc_exp_10 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_10_body, auth=sg_user_session) client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=10, expiry=3, delay=1, auth=sg_user_session) client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=1, auth=sg_user_session) # If expiry was not removed in the last update, this would expire doc_exp_3 time.sleep(5) # doc_exp_3 should still be around due to removal of expiry doc_exp_3 = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session) assert doc_exp_3["_id"] == "exp_3" # doc_exp_10 should be expired due to the updates (10s) + sleep (5s) with pytest.raises(HTTPError) as he: client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_10["id"], auth=sg_user_session) assert he.value[0].startswith("404 Client Error: Not Found for url:")
def test_rolling_ttl_expires(params_from_base_test_setup, sg_conf_name): """ 1. PUT /db/doc1 via SG with property "_exp":3 2. Update /db/doc1 10 times with a new revision (also with "_exp":3) 3. Wait 5 seconds 4. Get /db/doc1. Assert response is 200 """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) cluster_helper = ClusterKeywords() topology = cluster_helper.get_cluster_topology(cluster_config) cluster_helper.reset_cluster( cluster_config=cluster_config, sync_gateway_config=sg_conf ) cbs_url = topology["couchbase_servers"][0] sg_url = topology["sync_gateways"][0]["public"] sg_url_admin = topology["sync_gateways"][0]["admin"] log_info("Running 'test_rolling_ttl_expires'") log_info("cbs_url: {}".format(cbs_url)) log_info("sg_url: {}".format(sg_url)) log_info("sg_url_admin: {}".format(sg_url_admin)) sg_db = "db" sg_user_name = "sg_user" sg_user_password = "******" sg_user_channels = ["NBC", "ABC"] client = MobileRestClient() client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels) sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name) doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=3, channels=sg_user_channels) doc_exp_10_body = document.create_doc(doc_id="exp_10", expiry=10, channels=sg_user_channels) doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session) doc_exp_10 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_10_body, auth=sg_user_session) client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=10, expiry=3, auth=sg_user_session) # Sleep should allow doc_exp_3 to expire, but still be in the window to get doc_exp_10 time.sleep(5) # doc_exp_3 should be expired with pytest.raises(HTTPError) as he: client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session) assert he.value[0].startswith("404 Client Error: Not Found for url:") # doc_exp_10 should be available still doc_exp_10_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_10["id"], auth=sg_user_session) assert doc_exp_10_result["_id"] == "exp_10"
def test_removing_expiry(params_from_base_test_setup, sg_conf_name): """ 1. PUT /db/doc1 via SG with property "_exp":3 2. Update /db/doc1 with a new revision with no expiry value 3. After 10 updates, update /db/doc1 with a revision with no expiry """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) cluster_helper = ClusterKeywords() topology = cluster_helper.get_cluster_topology(cluster_config) cluster_helper.reset_cluster( cluster_config=cluster_config, sync_gateway_config=sg_conf ) cbs_url = topology["couchbase_servers"][0] sg_url = topology["sync_gateways"][0]["public"] sg_url_admin = topology["sync_gateways"][0]["admin"] log_info("Running 'test_removing_expiry'") log_info("cbs_url: {}".format(cbs_url)) log_info("sg_url: {}".format(sg_url)) log_info("sg_url_admin: {}".format(sg_url_admin)) sg_db = "db" sg_user_name = "sg_user" sg_user_password = "******" sg_user_channels = ["NBC", "ABC"] client = MobileRestClient() client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels) sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name) doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=3, channels=sg_user_channels) doc_exp_10_body = document.create_doc(doc_id="exp_10", expiry=10, channels=sg_user_channels) doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session) doc_exp_10 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_10_body, auth=sg_user_session) doc_exp_3_updated = client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=10, auth=sg_user_session) # Sleep should allow an expiry to happen on doc_exp_3 if it had not been removed. # Expected behavior is that the doc_exp_3 will still be around due to the removal of the expiry time.sleep(5) # doc_exp_3 should no longer have an expiry and should not raise an exception doc_exp_3_updated_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3_updated["id"], auth=sg_user_session) assert doc_exp_3_updated_result["_id"] == "exp_3" # doc_exp_10 should be available still and should not raise an exception doc_exp_10_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_10["id"], auth=sg_user_session) assert doc_exp_10_result["_id"] == "exp_10"
def test_revs_cache_size(params_from_base_test_setup, sg_conf_name, num_of_docs): """ @summary Test for no-conflicts with rev_cache size Test case link : https://docs.google.com/spreadsheets/d/1YwI_gCeoBebQKBybkzoAEoXSc0XLReszDA-mPFQapgk/edit#gid=0 covered #18 Steps: Note : the sg config have rev_cache_size as 1000 , make sure number of docs is less than 1000 to have the test work with expected behavior 1. Add docs to SG. 2. Get the docs 3. Verify number of rev_cache_hits is same as number of docs if rev_cache_size is more than number of docs. """ # Setup cluster_config = params_from_base_test_setup["cluster_config"] topology = params_from_base_test_setup["cluster_topology"] mode = params_from_base_test_setup["mode"] sg_url = topology["sync_gateways"][0]["public"] sg_admin_url = topology["sync_gateways"][0]["admin"] sg_db = "db" retrieved_docs = num_of_docs / 2 sync_gateway_version = params_from_base_test_setup["sync_gateway_version"] if sync_gateway_version < "2.0": pytest.skip('It does not work with sg < 2.0 , so skipping the test') sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) c = cluster.Cluster(cluster_config) c.reset(sg_conf) sg_client = MobileRestClient() channels = ["no-conflicts"] sg_client.create_user(url=sg_admin_url, db=sg_db, name='autotest', password='******', channels=channels) autouser_session = sg_client.create_session(url=sg_admin_url, db=sg_db, name='autotest', password='******') # end of Set up # 2. Add docs to SG. sgdoc_bodies = document.create_docs(doc_id_prefix="sg_docs", number=num_of_docs, channels=channels) sg_docs = sg_client.add_bulk_docs(url=sg_url, db=sg_db, docs=sgdoc_bodies, auth=autouser_session) assert len(sgdoc_bodies) == num_of_docs # 3. Get all docs for i in range(retrieved_docs): doc = sg_docs[i] sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc["id"], auth=autouser_session) # 4. Verify there are number of hits should be same as retrieved docs exp_vars = sg_client.get_expvars(url=sg_admin_url) revision_cache_hits = exp_vars["syncGateway_stats"]["revisionCache_hits"] revision_cache_misses = exp_vars["syncGateway_stats"]["revisionCache_misses"] assert revision_cache_hits == retrieved_docs, "Revision Cache hits did not hit with expected number {}".format(num_of_docs) assert revision_cache_misses == 0, "Revision Cache misses is not 0"
def test_numeric_expiry_as_ttl(params_from_base_test_setup, sg_conf_name): """ 1. PUT /db/doc1 via SG with property "_exp":3 PUT /db/doc2 via SG with property "_exp":10 2. Wait five seconds 3. Get /db/doc1. Assert response is 404 Get /db/doc2. Assert response is 200 """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] xattrs_enabled = params_from_base_test_setup['xattrs_enabled'] sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) cluster_helper = ClusterKeywords() topology = cluster_helper.get_cluster_topology(cluster_config) cluster_helper.reset_cluster(cluster_config=cluster_config, sync_gateway_config=sg_conf) cbs_url = topology["couchbase_servers"][0] sg_url = topology["sync_gateways"][0]["public"] sg_url_admin = topology["sync_gateways"][0]["admin"] log_info("Running 'test_numeric_expiry_as_ttl'") log_info("cbs_url: {}".format(cbs_url)) log_info("sg_url: {}".format(sg_url)) log_info("sg_url_admin: {}".format(sg_url_admin)) sg_db = "db" sg_user_name = "sg_user" sg_user_password = "******" sg_user_channels = ["NBC", "ABC"] bucket_name = "data-bucket" cbs_ip = host_for_url(cbs_url) sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name), password='******') client = MobileRestClient() client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels) sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name) doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=3, channels=sg_user_channels) doc_exp_10_body = document.create_doc(doc_id="exp_10", expiry=10, channels=sg_user_channels) doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session) doc_exp_10 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_10_body, auth=sg_user_session) # Sleep should allow doc_exp_3 to expire, but still be in the window to get doc_exp_10 time.sleep(5) # doc_exp_3 should be expired with pytest.raises(HTTPError) as he: client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session) # In XATTR mode, the expiry results in a tombstone # In Doc Meta mode, the expiry results in a purge if xattrs_enabled: assert he.value[0].startswith("403 Client Error: Forbidden for url:") else: assert he.value[0].startswith("404 Client Error: Not Found for url:") verify_doc_deletion_on_server(doc_id=doc_exp_3["id"], sdk_client=sdk_client, sg_client=client, sg_admin_url=sg_url_admin, sg_db=sg_db, xattrs_enabled=xattrs_enabled) # doc_exp_10 should be available still doc_exp_10_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_10["id"], auth=sg_user_session) assert doc_exp_10_result["_id"] == "exp_10"
def test_string_expiry_as_iso_8601_date(params_from_base_test_setup, sg_conf_name): """ 1. Calculate (server time + 3 seconds) as ISO-8601 date (e.g. 2016-01-01T00:00:00.000+00:00) 2. PUT /db/doc1 via SG with property "_exp":"[date]" PUT /db/doc2 via SG with property "_exp":"2026-01-01T00:00:00.000+00:00" 3. Wait five seconds 4. Get /db/doc1. Assert response is 404 Get /db/doc2. Assert response is 20 """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] xattrs_enabled = params_from_base_test_setup['xattrs_enabled'] sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) cluster_helper = ClusterKeywords() topology = cluster_helper.get_cluster_topology(cluster_config) cluster_helper.reset_cluster(cluster_config=cluster_config, sync_gateway_config=sg_conf) cbs_url = topology["couchbase_servers"][0] sg_url = topology["sync_gateways"][0]["public"] sg_url_admin = topology["sync_gateways"][0]["admin"] log_info("Running 'test_string_expiry_as_ISO_8601_Date'") log_info("cbs_url: {}".format(cbs_url)) log_info("sg_url: {}".format(sg_url)) log_info("sg_url_admin: {}".format(sg_url_admin)) sg_db = "db" sg_user_name = "sg_user" sg_user_password = "******" sg_user_channels = ["NBC", "ABC"] bucket_name = "data-bucket" cbs_ip = host_for_url(cbs_url) sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name), password='******') client = MobileRestClient() client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels) sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name) time_util = Time() iso_datetime = time_util.get_iso_datetime(delta=3) doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=iso_datetime, channels=sg_user_channels) doc_exp_years_body = document.create_doc( doc_id="exp_years", expiry="2026-01-01T00:00:00.000+00:00", channels=sg_user_channels) doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session) doc_exp_years = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_years_body, auth=sg_user_session) # Sleep should allow doc_exp_3 to expire time.sleep(10) # doc_exp_3 should be expired with pytest.raises(HTTPError) as he: client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session) # In XATTR mode, the expiry results in a tombstone # In Doc Meta mode, the expiry results in a purge if xattrs_enabled: assert he.value[0].startswith("403 Client Error: Forbidden for url:") else: assert he.value[0].startswith("404 Client Error: Not Found for url:") verify_doc_deletion_on_server(doc_id=doc_exp_3["id"], sdk_client=sdk_client, sg_client=client, sg_admin_url=sg_url_admin, sg_db=sg_db, xattrs_enabled=xattrs_enabled) # doc_exp_years should be available still doc_exp_years_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_years["id"], auth=sg_user_session) assert doc_exp_years_result["_id"] == "exp_years"
def test_string_expiry_as_unix_date(params_from_base_test_setup, sg_conf_name): """ 1. Calculate (server time + 3 seconds) as unix time (i.e. Epoch time, e.g. 1466465122) 2. PUT /db/doc1 via SG with property "_exp":"[unix time]" PUT /db/doc2 via SG with property "_exp":"1767225600" (Jan 1 2026) Note: the maximum epoch time supported by CBS is maxUint32, or Sun 07 Feb 2106, in case you want to move it out further than 2026. 3. Wait five seconds 4. Get /db/doc1. Assert response is 404 Get /db/doc2. Assert response is 200 """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] xattrs_enabled = params_from_base_test_setup['xattrs_enabled'] sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) cluster_helper = ClusterKeywords() topology = cluster_helper.get_cluster_topology(cluster_config) cluster_helper.reset_cluster(cluster_config=cluster_config, sync_gateway_config=sg_conf) cbs_url = topology["couchbase_servers"][0] sg_url = topology["sync_gateways"][0]["public"] sg_url_admin = topology["sync_gateways"][0]["admin"] log_info("Running 'test_string_expiry_as_unix_date'") log_info("cbs_url: {}".format(cbs_url)) log_info("sg_url: {}".format(sg_url)) log_info("sg_url_admin: {}".format(sg_url_admin)) sg_db = "db" sg_user_name = "sg_user" sg_user_password = "******" sg_user_channels = ["NBC", "ABC"] bucket_name = "data-bucket" cbs_ip = host_for_url(cbs_url) sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name), password='******') client = MobileRestClient() client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels) sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name) time_util = Time() unix_time_3s_ahead = time_util.get_unix_timestamp(delta=3) # Convert unix timestamp to string unix_time_3s_ahead_string = str(unix_time_3s_ahead) # Using string representation for unix time doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=unix_time_3s_ahead_string, channels=sg_user_channels) doc_exp_years_body = document.create_doc(doc_id="exp_years", expiry="1767225600", channels=sg_user_channels) doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session) doc_exp_years = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_years_body, auth=sg_user_session) # Sleep should allow doc_exp_3 to expire time.sleep(10) # doc_exp_3 should be expired with pytest.raises(HTTPError) as he: client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session) # In XATTR mode, the expiry results in a tombstone # In Doc Meta mode, the expiry results in a purge if xattrs_enabled: assert he.value[0].startswith("403 Client Error: Forbidden for url:") else: assert he.value[0].startswith("404 Client Error: Not Found for url:") verify_doc_deletion_on_server(doc_id=doc_exp_3["id"], sdk_client=sdk_client, sg_client=client, sg_admin_url=sg_url_admin, sg_db=sg_db, xattrs_enabled=xattrs_enabled) # doc_exp_years should be available still doc_exp_years_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_years["id"], auth=sg_user_session) assert doc_exp_years_result["_id"] == "exp_years"
def test_longpoll_awaken_channels(params_from_base_test_setup, sg_conf_name): cluster_conf = params_from_base_test_setup["cluster_config"] cluster_topology = params_from_base_test_setup["cluster_topology"] mode = params_from_base_test_setup["mode"] sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) sg_admin_url = cluster_topology["sync_gateways"][0]["admin"] sg_url = cluster_topology["sync_gateways"][0]["public"] log_info("sg_conf: {}".format(sg_conf)) log_info("sg_admin_url: {}".format(sg_admin_url)) log_info("sg_url: {}".format(sg_url)) cluster = Cluster(config=cluster_conf) cluster.reset(sg_config_path=sg_conf) adam_user_info = userinfo.UserInfo(name="adam", password="******", channels=["NBC", "ABC"], roles=[]) traun_user_info = userinfo.UserInfo(name="traun", password="******", channels=[], roles=[]) andy_user_info = userinfo.UserInfo(name="andy", password="******", channels=[], roles=[]) sg_db = "db" doc_id = "adam_doc_0" client = MobileRestClient() adam_auth = client.create_user(url=sg_admin_url, db=sg_db, name=adam_user_info.name, password=adam_user_info.password, channels=adam_user_info.channels) traun_auth = client.create_user(url=sg_admin_url, db=sg_db, name=traun_user_info.name, password=traun_user_info.password, channels=traun_user_info.channels) andy_auth = client.create_user(url=sg_admin_url, db=sg_db, name=andy_user_info.name, password=andy_user_info.password, channels=andy_user_info.channels) ############################################################ # changes feed wakes with Channel Access via Admin API ############################################################ # Get starting sequence of docs, use the last seq to progress past any _user docs. adam_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=adam_auth) traun_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=traun_auth) andy_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=andy_auth) with concurrent.futures.ProcessPoolExecutor() as ex: # Start changes feed for 3 users adam_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=adam_changes["last_seq"], timeout=10, auth=adam_auth) traun_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=traun_changes["last_seq"], timeout=10, auth=traun_auth) andy_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=andy_changes["last_seq"], timeout=10, auth=andy_auth) # Wait for changes feed to notice there are no changes and enter wait. 2 seconds should be more than enough time.sleep(2) # Make sure the changes future is still running and has not exited due to any new changes, the feed should be caught up # and waiting assert not adam_changes_task.done() assert not traun_changes_task.done() assert not andy_changes_task.done() # Add add a doc for adam with "NBC" and "ABC" channels # Add one doc, this should wake up the changes feed adam_add_docs_task = ex.submit(client.add_docs, url=sg_url, db=sg_db, number=1, id_prefix="adam_doc", auth=adam_auth, channels=adam_user_info.channels) # Wait for docs adds to complete adam_docs = adam_add_docs_task.result() assert len(adam_docs) == 1 # Assert that the changes feed woke up and that the doc change was propagated adam_changes = adam_changes_task.result() assert len(adam_changes["results"]) == 1 assert adam_changes["results"][0]["id"] == doc_id # Verify that the changes feed is still listening for Traun and Andy assert not traun_changes_task.done() assert not andy_changes_task.done() # Update the traun and andy to have one of adam's channels update_traun_user_task = ex.submit(client.update_user, url=sg_admin_url, db=sg_db, name=traun_user_info.name, password=traun_user_info.password, channels=["NBC"]) traun_auth = update_traun_user_task.result() update_andy_user_task = ex.submit(client.update_user, url=sg_admin_url, db=sg_db, name=andy_user_info.name, password=andy_user_info.password, channels=["ABC"]) andy_auth = update_andy_user_task.result() # Make sure changes feed wakes up and contains at least one change, 2 may be possible if the _user doc is included # Make sure the first change is 'adam_doc' traun_changes = traun_changes_task.result() assert 1 <= len(traun_changes["results"]) <= 2 assert traun_changes["results"][0]["id"] == "adam_doc_0" or traun_changes["results"][0]["id"] == "_user/traun" andy_changes = andy_changes_task.result() assert 1 <= len(andy_changes["results"]) <= 2 assert andy_changes["results"][0]["id"] == "adam_doc_0" or andy_changes["results"][0]["id"] == "_user/andy" # Block until user docs are seen client.verify_doc_id_in_changes(url=sg_url, db=sg_db, expected_doc_id="_user/adam", auth=adam_auth) client.verify_doc_id_in_changes(url=sg_url, db=sg_db, expected_doc_id="_user/traun", auth=traun_auth) client.verify_doc_id_in_changes(url=sg_url, db=sg_db, expected_doc_id="_user/andy", auth=andy_auth) # Make sure that adams doc shows up in changes due to the fact that the changes feed may be woken up with a _user doc above client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=adam_docs, auth=adam_auth) client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=adam_docs, auth=traun_auth) client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=adam_docs, auth=andy_auth) ############################################################ # changes feed wakes with Channel Removal via Sync function ############################################################ # Get latest last_seq for next test section adam_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=adam_auth) traun_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=traun_auth) andy_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=andy_auth) with concurrent.futures.ProcessPoolExecutor() as ex: # Start changes feed for 3 users from latest last_seq adam_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=adam_changes["last_seq"], timeout=10, auth=adam_auth) traun_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=traun_changes["last_seq"], timeout=10, auth=traun_auth) andy_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=andy_changes["last_seq"], timeout=10, auth=andy_auth) # Wait for changes feed to notice there are no changes and enter wait. 2 seconds should be more than enough time.sleep(2) # Make sure the changes future is still running and has not exited due to any new changes, the feed should be caught up # and waiting assert not adam_changes_task.done() assert not traun_changes_task.done() assert not andy_changes_task.done() # Remove the channels property from the doc client.update_doc(url=sg_url, db=sg_db, doc_id=doc_id, auth=traun_auth, channels=[]) # All three changes feeds should wake up and return one result adam_changes = adam_changes_task.result() assert len(adam_changes["results"]) == 1 assert adam_changes["results"][0]["removed"] == ["ABC", "NBC"] traun_changes = traun_changes_task.result() assert len(traun_changes["results"]) == 1 assert traun_changes["results"][0]["removed"] == ["NBC"] andy_changes = andy_changes_task.result() assert len(andy_changes["results"]) == 1 assert andy_changes["results"][0]["removed"] == ["ABC"] # Verify that users no longer can access the doc for user_auth in [adam_auth, traun_auth, andy_auth]: with pytest.raises(requests.exceptions.HTTPError) as excinfo: client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id, auth=user_auth) assert "403 Client Error: Forbidden for url:" in excinfo.value.message ############################################################ # changes feed wakes with Channel Grant via Sync function ############################################################ # Get latest last_seq for next test section adam_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=adam_auth) traun_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=traun_auth) andy_changes = client.get_changes(url=sg_url, db=sg_db, since=0, feed="normal", auth=andy_auth) admin_auth = client.create_user(url=sg_admin_url, db=sg_db, name="admin", password="******", channels=["admin"]) channel_grant_doc_id = "channel_grant_with_doc_intially" # Add another doc with no channels channel_grant_doc_body = document.create_doc(doc_id=channel_grant_doc_id, channels=["admin"]) client.add_doc(url=sg_url, db=sg_db, doc=channel_grant_doc_body, auth=admin_auth) with concurrent.futures.ProcessPoolExecutor() as ex: # Start changes feed for 3 users from latest last_seq adam_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=adam_changes["last_seq"], timeout=10, auth=adam_auth) traun_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=traun_changes["last_seq"], timeout=10, auth=traun_auth) andy_changes_task = ex.submit(client.get_changes, url=sg_url, db=sg_db, since=andy_changes["last_seq"], timeout=10, auth=andy_auth) # Wait for changes feed to notice there are no changes and enter wait. 2 seconds should be more than enough time.sleep(2) # Make sure the changes future is still running and has not exited due to any new changes, the feed should be caught up # and waiting assert not adam_changes_task.done() assert not traun_changes_task.done() assert not andy_changes_task.done() # update the grant doc to have channel for all users update_task = ex.submit(client.update_doc, url=sg_url, db=sg_db, doc_id=channel_grant_doc_id, auth=admin_auth, channels=["admin", "ABC", "NBC"]) updated_doc = update_task.result() assert updated_doc["rev"].startswith("2-") # Verify that access grant wakes up changes feed for adam, traun, and Andy adam_changes = adam_changes_task.result() assert len(adam_changes["results"]) == 1 assert adam_changes["results"][0]["id"] == "channel_grant_with_doc_intially" assert adam_changes["results"][0]["changes"][0]["rev"].startswith("2-") traun_changes = traun_changes_task.result() assert len(traun_changes["results"]) == 1 assert traun_changes["results"][0]["id"] == "channel_grant_with_doc_intially" assert traun_changes["results"][0]["changes"][0]["rev"].startswith("2-") andy_changes = andy_changes_task.result() assert len(andy_changes["results"]) == 1 assert andy_changes["results"][0]["id"] == "channel_grant_with_doc_intially" assert andy_changes["results"][0]["changes"][0]["rev"].startswith("2-")
def test_auto_prune_listener_keeps_conflicts_sanity(setup_client_syncgateway_test): """" 1. Create db on LiteServ and add docs 2. Create db on sync_gateway and add docs with the same id 3. Create one shot push / pull replication 4. Update LiteServ 50 times 5. Assert that pruned conflict is still present 6. Delete the current revision and check that a GET returns the old conflict as the current rev """ cluster_config = setup_client_syncgateway_test["cluster_config"] ls_url = setup_client_syncgateway_test["ls_url"] sg_url = setup_client_syncgateway_test["sg_url"] sg_admin_url = setup_client_syncgateway_test["sg_admin_url"] client = MobileRestClient() sg_helper = SyncGateway() sg_helper.start_sync_gateway( cluster_config=cluster_config, url=sg_url, config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS) ) log_info("Running 'test_auto_prune_listener_keeps_conflicts_sanity' ...") log_info("ls_url: {}".format(ls_url)) log_info("sg_url: {}".format(sg_url)) log_info("sg_admin_url: {}".format(sg_admin_url)) num_docs = 1 num_revs = 100 sg_db = "db" ls_db = "ls_db" sg_user_name = "sg_user" sg_user_channels = ["NBC"] client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels) sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name) ls_db = client.create_database(url=ls_url, name=ls_db) # Create docs with same prefix to create conflicts when the dbs complete 1 shot replication ls_db_docs = client.add_docs(url=ls_url, db=ls_db, number=num_docs, id_prefix="doc", channels=sg_user_channels) assert len(ls_db_docs) == num_docs sg_db_docs = client.add_docs( url=sg_url, db=sg_db, number=num_docs, id_prefix="doc", channels=sg_user_channels, auth=sg_session ) assert len(sg_db_docs) == num_docs # Setup one shot pull replication and wait for idle. client.start_replication(url=ls_url, continuous=False, from_url=sg_admin_url, from_db=sg_db, to_db=ls_db) client.wait_for_no_replications(url=ls_url) # There should now be a conflict on the client conflicting_revs = client.get_conflict_revs(url=ls_url, db=ls_db, doc=ls_db_docs[0]) # Get the doc with conflict rev client.get_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"], rev=conflicting_revs[0]) # Update doc past revs limit and make sure conflict is still available updated_doc = client.update_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"], number_updates=num_revs) client.get_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"], rev=conflicting_revs[0]) # Delete doc and ensure that the conflict is now the current rev client.delete_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"], rev=updated_doc["rev"]) current_doc = client.get_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"]) assert current_doc["_rev"] == conflicting_revs[0]
def update_docs_task(users, user_type, user_index, sg_url, sg_db, docs_per_user_per_update): user_name = '{}_{}'.format(user_type, user_index) # Get a random value to determin the update method # ~ 90% ops bulk, 10% ops single rand = random.random() if rand <= 0.90: update_method = 'bulk_docs' else: update_method = 'put' sg_client = MobileRestClient() # Get a random user current_user_auth = users[user_name]['auth'] current_user_doc_ids = list(users[user_name]['doc_ids']) # Get a random subset of docs to update user_docs_subset_to_update = [] for _ in range(docs_per_user_per_update): random_doc_id = random.choice(current_user_doc_ids) user_docs_subset_to_update.append(random_doc_id) current_user_doc_ids.remove(random_doc_id) log_info('Updating {} docs, method ({}) number of updates: {} ({})'.format( len(user_docs_subset_to_update), update_method, users[user_name]['updates'], user_name)) # Update the user's docs if update_method == 'bulk_docs': # Get docs for that user user_docs, errors = sg_client.get_bulk_docs( url=sg_url, db=sg_db, doc_ids=user_docs_subset_to_update, auth=current_user_auth) assert len(errors) == 0 # Update the 'updates' property for doc in user_docs: doc['updates'] += 1 # Add the docs via build_docs sg_client.add_bulk_docs(url=sg_url, db=sg_db, docs=user_docs, auth=current_user_auth) else: # Do a single GET / PUT for each of the user docs for doc_id in user_docs_subset_to_update: doc = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id, auth=current_user_auth) doc['updates'] += 1 sg_client.put_doc(url=sg_url, db=sg_db, doc_id=doc_id, doc_body=doc, rev=doc['_rev'], auth=current_user_auth) return user_name
def test_auto_prune_listener_keeps_conflicts_sanity( setup_client_syncgateway_test): """" 1. Create db on LiteServ and add docs 2. Create db on sync_gateway and add docs with the same id 3. Create one shot push / pull replication 4. Update LiteServ 50 times 5. Assert that pruned conflict is still present 6. Delete the current revision and check that a GET returns the old conflict as the current rev """ cluster_config = setup_client_syncgateway_test["cluster_config"] sg_mode = setup_client_syncgateway_test["sg_mode"] ls_url = setup_client_syncgateway_test["ls_url"] sg_url = setup_client_syncgateway_test["sg_url"] sg_admin_url = setup_client_syncgateway_test["sg_admin_url"] client = MobileRestClient() sg_config = sync_gateway_config_path_for_mode( "listener_tests/listener_tests", sg_mode) c = cluster.Cluster(config=cluster_config) c.reset(sg_config_path=sg_config) log_info("Running 'test_auto_prune_listener_keeps_conflicts_sanity' ...") log_info("ls_url: {}".format(ls_url)) log_info("sg_url: {}".format(sg_url)) log_info("sg_admin_url: {}".format(sg_admin_url)) num_docs = 1 num_revs = 100 sg_db = "db" ls_db = "ls_db" sg_user_name = "sg_user" sg_user_channels = ["NBC"] client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels) sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name) ls_db = client.create_database(url=ls_url, name=ls_db) # Create docs with same prefix to create conflicts when the dbs complete 1 shot replication ls_db_docs = client.add_docs(url=ls_url, db=ls_db, number=num_docs, id_prefix="doc", channels=sg_user_channels) assert len(ls_db_docs) == num_docs sg_db_docs = client.add_docs(url=sg_url, db=sg_db, number=num_docs, id_prefix="doc", channels=sg_user_channels, auth=sg_session) assert len(sg_db_docs) == num_docs # Wait for changes to be available on Sync Gateway client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=sg_db_docs, auth=sg_session, polling_interval=1) # Setup one shot pull replication and wait for idle. client.start_replication(url=ls_url, continuous=False, from_url=sg_admin_url, from_db=sg_db, to_db=ls_db) client.wait_for_no_replications(url=ls_url) # There should now be a conflict on the client conflicting_revs = client.get_conflict_revs(url=ls_url, db=ls_db, doc=ls_db_docs[0]) # Get the doc with conflict rev client.get_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"], rev=conflicting_revs[0]) # Update doc past revs limit and make sure conflict is still available updated_doc = client.update_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"], number_updates=num_revs) client.get_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"], rev=conflicting_revs[0]) # Delete doc and ensure that the conflict is now the current rev client.delete_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"], rev=updated_doc["rev"]) current_doc = client.get_doc(url=ls_url, db=ls_db, doc_id=ls_db_docs[0]["id"]) assert current_doc["_rev"] == conflicting_revs[0]
def test_verify_open_revs_with_revs_limit_push_conflict(setup_client_syncgateway_test): """Test replication from multiple client dbs to one sync_gateway db https://github.com/couchbase/couchbase-lite-ios/issues/1277 """ cluster_config = setup_client_syncgateway_test["cluster_config"] ls_url = setup_client_syncgateway_test["ls_url"] sg_url = setup_client_syncgateway_test["sg_url"] sg_admin_url = setup_client_syncgateway_test["sg_admin_url"] num_docs = 100 num_revs = 20 sg_db = "db" sg_user_name = "sg_user" sg_helper = SyncGateway() sg_helper.start_sync_gateway( cluster_config=cluster_config, url=sg_url, config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS) ) log_info("Running 'test_verify_open_revs_with_revs_limit_push_conflict'") log_info("ls_url: {}".format(ls_url)) log_info("sg_admin_url: {}".format(sg_admin_url)) log_info("sg_url: {}".format(sg_url)) log_info("num_docs: {}".format(num_docs)) log_info("num_revs: {}".format(num_revs)) client = MobileRestClient() # Test the endpoint, listener does not support users but should have a default response client.get_session(url=ls_url) sg_user_channels = ["NBC"] client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels) sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name) ls_db = client.create_database(url=ls_url, name="ls_db") ls_db_docs = client.add_docs(url=ls_url, db=ls_db, number=num_docs, id_prefix="ls_db", channels=sg_user_channels) assert len(ls_db_docs) == num_docs # Start replication ls_db -> sg_db repl_one = client.start_replication( url=ls_url, continuous=True, from_db=ls_db, to_url=sg_admin_url, to_db=sg_db ) client.verify_docs_present(url=sg_admin_url, db=sg_db, expected_docs=ls_db_docs) client.update_docs(url=sg_url, db=sg_db, docs=ls_db_docs, number_updates=num_revs, auth=sg_session) sg_current_doc = client.get_doc(url=sg_url, db=sg_db, doc_id="ls_db_2", auth=sg_session) client.update_docs(url=ls_url, db=ls_db, docs=ls_db_docs, number_updates=num_revs) ls_current_doc = client.get_doc(url=ls_url, db=ls_db, doc_id="ls_db_2") client.wait_for_replication_status_idle(url=ls_url, replication_id=repl_one) client.verify_doc_rev_generation(url=ls_url, db=ls_db, doc_id=ls_current_doc["_id"], expected_generation=21) client.verify_doc_rev_generation(url=sg_url, db=sg_db, doc_id=sg_current_doc["_id"], expected_generation=21, auth=sg_session) expected_ls_revs = [ls_current_doc["_rev"]] client.verify_open_revs(url=ls_url, db=ls_db, doc_id=ls_current_doc["_id"], expected_open_revs=expected_ls_revs) expected_sg_revs = [ls_current_doc["_rev"], sg_current_doc["_rev"]] client.verify_open_revs(url=sg_admin_url, db=sg_db, doc_id=sg_current_doc["_id"], expected_open_revs=expected_sg_revs)
def test_string_expiry_as_iso_8601_date(params_from_base_test_setup, sg_conf_name): """ 1. Calculate (server time + 3 seconds) as ISO-8601 date (e.g. 2016-01-01T00:00:00.000+00:00) 2. PUT /db/doc1 via SG with property "_exp":"[date]" PUT /db/doc2 via SG with property "_exp":"2026-01-01T00:00:00.000+00:00" 3. Wait five seconds 4. Get /db/doc1. Assert response is 404 Get /db/doc2. Assert response is 20 """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) cluster_helper = ClusterKeywords() topology = cluster_helper.get_cluster_topology(cluster_config) cluster_helper.reset_cluster( cluster_config=cluster_config, sync_gateway_config=sg_conf ) cbs_url = topology["couchbase_servers"][0] sg_url = topology["sync_gateways"][0]["public"] sg_url_admin = topology["sync_gateways"][0]["admin"] log_info("Running 'test_string_expiry_as_ISO_8601_Date'") log_info("cbs_url: {}".format(cbs_url)) log_info("sg_url: {}".format(sg_url)) log_info("sg_url_admin: {}".format(sg_url_admin)) sg_db = "db" sg_user_name = "sg_user" sg_user_password = "******" sg_user_channels = ["NBC", "ABC"] client = MobileRestClient() client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels) sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name) time_util = Time() iso_datetime = time_util.get_iso_datetime(delta=3) doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=iso_datetime, channels=sg_user_channels) doc_exp_years_body = document.create_doc(doc_id="exp_years", expiry="2026-01-01T00:00:00.000+00:00", channels=sg_user_channels) doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session) doc_exp_years = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_years_body, auth=sg_user_session) # Sleep should allow doc_exp_3 to expire time.sleep(10) # doc_exp_3 should be expired with pytest.raises(HTTPError) as he: client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session) assert he.value[0].startswith("404 Client Error: Not Found for url:") # doc_exp_years should be available still doc_exp_years_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_years["id"], auth=sg_user_session) assert doc_exp_years_result["_id"] == "exp_years"
def test_string_expiry_as_unix_date(params_from_base_test_setup, sg_conf_name): """ 1. Calculate (server time + 3 seconds) as unix time (i.e. Epoch time, e.g. 1466465122) 2. PUT /db/doc1 via SG with property "_exp":"[unix time]" PUT /db/doc2 via SG with property "_exp":"1767225600" (Jan 1 2026) Note: the maximum epoch time supported by CBS is maxUint32, or Sun 07 Feb 2106, in case you want to move it out further than 2026. 3. Wait five seconds 4. Get /db/doc1. Assert response is 404 Get /db/doc2. Assert response is 200 """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) cluster_helper = ClusterKeywords() topology = cluster_helper.get_cluster_topology(cluster_config) cluster_helper.reset_cluster( cluster_config=cluster_config, sync_gateway_config=sg_conf ) cbs_url = topology["couchbase_servers"][0] sg_url = topology["sync_gateways"][0]["public"] sg_url_admin = topology["sync_gateways"][0]["admin"] log_info("Running 'test_string_expiry_as_unix_date'") log_info("cbs_url: {}".format(cbs_url)) log_info("sg_url: {}".format(sg_url)) log_info("sg_url_admin: {}".format(sg_url_admin)) sg_db = "db" sg_user_name = "sg_user" sg_user_password = "******" sg_user_channels = ["NBC", "ABC"] client = MobileRestClient() client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels) sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name) time_util = Time() unix_time_3s_ahead = time_util.get_unix_timestamp(delta=3) # Convert unix timestamp to string unix_time_3s_ahead_string = str(unix_time_3s_ahead) # Using string representation for unix time doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=unix_time_3s_ahead_string, channels=sg_user_channels) doc_exp_years_body = document.create_doc(doc_id="exp_years", expiry="1767225600", channels=sg_user_channels) doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session) doc_exp_years = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_years_body, auth=sg_user_session) # Sleep should allow doc_exp_3 to expire time.sleep(10) # doc_exp_3 should be expired with pytest.raises(HTTPError) as he: client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session) assert he.value[0].startswith("404 Client Error: Not Found for url:") # doc_exp_years should be available still doc_exp_years_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_years["id"], auth=sg_user_session) assert doc_exp_years_result["_id"] == "exp_years"
def test_stale_revision_should_not_be_in_the_index(setup_client_syncgateway_test): """original ticket: https://github.com/couchbase/couchbase-lite-android/issues/855 scenario: 1. Running sync_gateway 2. Create database and starts both push and pull replicators through client REST API 3. Create two or more views through client REST API 4. Add doc, and verify doc is index with current revision through client REST API 5. Make sure document is pushed to sync gateway through sync gateway REST API 6. Update doc with sync gateway (not client side) through sync gateway REST API 7. Make sure updated document is pull replicated to client through client REST API 8. Make sure updated document is indexed through client REST API 9. Make sure stale revision is deleted from index. through client REST API 10. Pass criteria """ cluster_config = setup_client_syncgateway_test["cluster_config"] sg_mode = setup_client_syncgateway_test["sg_mode"] ls_url = setup_client_syncgateway_test["ls_url"] sg_url = setup_client_syncgateway_test["sg_url"] sg_admin_url = setup_client_syncgateway_test["sg_admin_url"] num_docs = 10 num_revs = 100 d_doc_name = "dd" sg_db = "db" sg_user_name = "sg_user" sg_config = sync_gateway_config_path_for_mode("listener_tests/listener_tests", sg_mode) c = cluster.Cluster(config=cluster_config) c.reset(sg_config_path=sg_config) log_info("Running 'test_stale_revision_should_not_be_in_the_index'") log_info("ls_url: {}".format(ls_url)) log_info("sg_admin_url: {}".format(sg_admin_url)) log_info("sg_url: {}".format(sg_url)) log_info("num_docs: {}".format(num_docs)) log_info("num_revs: {}".format(num_revs)) client = MobileRestClient() sg_user_channels = ["NBC"] client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels) sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name) view = """{ "language" : "javascript", "views" : { "content_view" : { "map" : "function(doc, meta) { if (doc.content) { emit(doc._id, doc._rev); } }" }, "update_view" : { "map" : "function(doc, meta) { emit(doc.updates, null); }" } } }""" ls_db = client.create_database(url=ls_url, name="ls_db") # Setup continuous push / pull replication from ls_db1 to sg_db client.start_replication( url=ls_url, continuous=True, from_db=ls_db, to_url=sg_admin_url, to_db=sg_db ) client.start_replication( url=ls_url, continuous=True, from_url=sg_admin_url, from_db=sg_db, to_db=ls_db ) design_doc_id = client.add_design_doc(url=ls_url, db=ls_db, name=d_doc_name, doc=view) client.get_doc(url=ls_url, db=ls_db, doc_id=design_doc_id) doc_body = document.create_doc(doc_id="doc_1", content={"hi": "I should be in the view"}, channels=sg_user_channels) log_info(doc_body) doc_body_2 = document.create_doc(doc_id="doc_2", channels=sg_user_channels) doc = client.add_doc(url=ls_url, db=ls_db, doc=doc_body) doc_2 = client.add_doc(url=ls_url, db=ls_db, doc=doc_body_2) content_view_rows = client.get_view(url=ls_url, db=ls_db, design_doc_name=d_doc_name, view_name="content_view") client.verify_view_row_num(view_response=content_view_rows, expected_num_rows=1) update_view_rows = client.get_view(url=ls_url, db=ls_db, design_doc_name=d_doc_name, view_name="update_view") client.verify_view_row_num(view_response=update_view_rows, expected_num_rows=2) expected_docs_list = [doc, doc_2] client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=expected_docs_list, auth=sg_session) updated_doc = client.update_doc(url=sg_url, db=sg_db, doc_id=doc["id"], number_updates=10, auth=sg_session) client.verify_docs_present(url=ls_url, db=ls_db, expected_docs=updated_doc) content_view_rows_2 = client.get_view(url=ls_url, db=ls_db, design_doc_name=d_doc_name, view_name="content_view") client.verify_view_row_num(view_response=content_view_rows_2, expected_num_rows=1) client.verify_view_contains_keys(view_response=content_view_rows_2, keys=doc["id"]) client.verify_view_contains_values(view_response=content_view_rows_2, values=updated_doc["rev"])
def test_mobile_opt_in(params_from_base_test_setup, sg_conf_name): """ Scenario: Enable mobile opt in sync function in sync-gateway configuration file - Check xattrs/mobile-opt-in_cc or di json files - 8 cases covered - doc : https://docs.google.com/document/d/1XxLIBsjuj_UxTTJs4Iu7C7uZdos8ZEzeckrVc17y3sw/edit - #1 Create doc via sdk with mobile opt in and verify doc is imported - #2 Create doc via sdk with mobile opt out and verify doc is not imported - #3 Create doc via sg with mobile opt in and update via sdk and verify doc is imported - #4 Create doc via sg with mobile opt out and update via sdk and verify doc is not imported - Try to update same doc via sg and verify 409 conflict error is thrown - Create a doc with same doc id and verify doc is created successfully - #5 Create doc via sg with mobile opt out and update via sdk which created no revisions - Now do sdk create with mobile opt in should import case #5 - #6 Create doc via sg with mobile opt out and update via sdk with opt in - Verify type is overrided and doc is imported - #7 Create doc via sg with mobile opt in and update via sdk with opt out - Verify type is overrided and doc is not imported - #8 Disable import in the sg config and have mobile opt in function Create doc via sdk with mobile property and verify sg update succeeds - #9 Same config as #8 and have mobile opt in function in config Create doc via sdk without mobile property and create new doc via sg with same doc id and verify it succeeds """ bucket_name = 'data-bucket' sg_db = 'db' cluster_conf = params_from_base_test_setup['cluster_config'] cluster_topology = params_from_base_test_setup['cluster_topology'] mode = params_from_base_test_setup['mode'] xattrs_enabled = params_from_base_test_setup['xattrs_enabled'] # This test should only run when using xattr meta storage if not xattrs_enabled: pytest.skip('XATTR tests require --xattrs flag') sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) sg_admin_url = cluster_topology['sync_gateways'][0]['admin'] sg_url = cluster_topology['sync_gateways'][0]['public'] cbs_url = cluster_topology['couchbase_servers'][0] log_info('sg_conf: {}'.format(sg_conf)) log_info('sg_admin_url: {}'.format(sg_admin_url)) log_info('sg_url: {}'.format(sg_url)) log_info('cbs_url: {}'.format(cbs_url)) cluster = Cluster(config=cluster_conf) cluster.reset(sg_config_path=sg_conf) # Create clients sg_client = MobileRestClient() cbs_ip = host_for_url(cbs_url) sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name), password='******', timeout=SDK_TIMEOUT) # Create user / session auto_user_info = UserInfo(name='autotest', password='******', channels=['mobileOptIn'], roles=[]) sg_client.create_user( url=sg_admin_url, db=sg_db, name=auto_user_info.name, password=auto_user_info.password, channels=auto_user_info.channels ) test_auth_session = sg_client.create_session( url=sg_admin_url, db=sg_db, name=auto_user_info.name, password=auto_user_info.password ) def update_mobile_prop(): return { 'updates': 0, 'type': 'mobile', } def update_non_mobile_prop(): return { 'updates': 0, 'test': 'true', 'type': 'mobile opt out', } # Create first doc via SDK with type mobile. Case #1 doc_id1 = 'mobile_opt_in_sdk_doc' doc = document.create_doc(doc_id=doc_id1, channels=['mobileOptIn'], prop_generator=update_mobile_prop) sdk_client.upsert(doc_id1, doc) sg_get_doc1 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id1, auth=test_auth_session) assert sg_get_doc1['_rev'].startswith('1-') and sg_get_doc1['_id'] == doc_id1 # Additional coverage for case #1 sg_client.update_doc(url=sg_url, db=sg_db, doc_id=doc_id1, number_updates=1, auth=test_auth_session) sg_get_doc1 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id1, auth=test_auth_session) assert sg_get_doc1['_rev'].startswith('2-') and sg_get_doc1['_id'] == doc_id1 # Create second doc via SDK with type non mobile. Case #2 doc_id2 = 'mobile_opt_out_sdk_doc' doc = document.create_doc(doc_id=doc_id2, channels=['mobileOptIn'], prop_generator=update_non_mobile_prop) sdk_client.upsert(doc_id2, doc) with pytest.raises(HTTPError) as he: sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id2, auth=test_auth_session) log_info(he.value) assert he.value.message.startswith('404 Client Error: Not Found for url:') # Create third sg doc with mobile opt in and update via sdk. Case #3 doc_id3 = 'mobile_opt_in_sg_doc' doc_body = document.create_doc(doc_id=doc_id3, channels=['mobileOptIn'], prop_generator=update_mobile_prop) doc = sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session) sg_get_doc3 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id3, auth=test_auth_session) sg_get_doc3["updated_sdk_via_sg"] = "1" sdk_client.upsert(doc_id3, sg_get_doc3) sg_get_doc3 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id3, auth=test_auth_session) assert sg_get_doc3['_rev'].startswith('2-') and sg_get_doc3['_id'] == doc_id3 log_info("sg get doc3 is {}".format(sg_get_doc3)) # Create fourth sg doc with mobile opt out and update via sdk. Case #4 and case #8 doc_id4 = 'mobile_opt_out_sg_doc' doc_body = document.create_doc(doc_id=doc_id4, channels=['mobileOptIn'], prop_generator=update_non_mobile_prop) doc = sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session) # update vis SDK sg_get_doc4 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id4, auth=test_auth_session) rev = sg_get_doc4['_rev'] sg_get_doc4["updated_sdk_via_sg"] = "1" sdk_client.upsert(doc_id4, sg_get_doc4) with pytest.raises(HTTPError) as he: sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id4, auth=test_auth_session) log_info(he.value) assert he.value.message.startswith('404 Client Error: Not Found for url:') # update via SG with pytest.raises(HTTPError) as he: sg_client.put_doc(url=sg_url, db=sg_db, doc_id=doc_id4, doc_body={'sg_rewrite': 'True'}, rev=rev, auth=test_auth_session) log_info(he.value) assert he.value.message.startswith('409 Client Error: Conflict for url:') # Create same doc again to verify there is not existing key error covers case #8 doc_body = document.create_doc(doc_id=doc_id4, channels=['mobileOptIn'], prop_generator=update_non_mobile_prop) sg_get_doc4_1 = sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session) log_info("4th doc after recreate vis sg is {}".format(sg_get_doc4_1)) assert sg_get_doc4_1['rev'].startswith('1-') and sg_get_doc4_1['id'] == doc_id4 # Create Fifth sg doc with mobile opt in and delete doc which created no revisions i.e tombstone doc # Now do sdk create with mobile opt in should import case #5 doc_id5 = 'mobile_sdk_recreate_no_activerev' doc_body = document.create_doc(doc_id=doc_id5, channels=['mobileOptIn'], prop_generator=update_mobile_prop) doc = sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session) rev = doc['rev'] sg_client.delete_doc(url=sg_url, db=sg_db, doc_id=doc_id5, rev=rev, auth=test_auth_session) # At this point no active revisions for this doc, so now update via sdk with mobile opt in should be successful # in getting doc doc = document.create_doc(doc_id=doc_id5, channels=['mobileOptIn'], prop_generator=update_mobile_prop) sdk_client.upsert(doc_id5, doc) sg_get_doc5 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id5, auth=test_auth_session) log_info("sg get doc 5 is {}".format(sg_get_doc5)) assert sg_get_doc5['_rev'].startswith('1-') and sg_get_doc5['_id'] == doc_id5 # Create sixth sg doc with mobile opt out and update via sdk with opt in doc_id6 = 'mobileoptout_sg_doc_sdkupdate_optin' doc_body = document.create_doc(doc_id=doc_id6, channels=['mobileOptIn'], prop_generator=update_non_mobile_prop) doc = sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session) sg_get_doc6 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id6, auth=test_auth_session) log_info("Sg sixth doc is {}".format(sg_get_doc6)) sg_get_doc6["type"] = "mobile" sdk_client.upsert(doc_id6, sg_get_doc6) sg_get_doc6 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id6, auth=test_auth_session) assert sg_get_doc6['_rev'].startswith('2-') and sg_get_doc6['_id'] == doc_id6 # Create seventh sg doc with mobile opt in and update via sdk with opt out doc_id7 = 'mobileoptin_sg_doc_sdkupdate_optout' doc_body = document.create_doc(doc_id=doc_id7, channels=['mobileOptIn'], prop_generator=update_mobile_prop) doc = sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session) sg_get_doc7 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id7, auth=test_auth_session) log_info("Sg sixth doc is {}".format(sg_get_doc7)) sg_get_doc7["type"] = "mobile opt out" sdk_client.upsert(doc_id7, sg_get_doc7) with pytest.raises(HTTPError) as he: sg_get_doc7 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id7, auth=test_auth_session) log_info(he.value) assert he.value.message.startswith('404 Client Error: Not Found for url:') # TODO : verify _changes that it shows tombstone revisions -> it will happen on 2.0 # Create eighth sdk doc with import disabled and add mobile property and update via sg. Case #7 sg_conf_name = "xattrs/mobile_opt_in_no_import" sg_no_import_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) sg_util = SyncGateway() sg_util.start_sync_gateways(cluster_config=cluster_conf, url=sg_url, config=sg_no_import_conf) doc_id8 = 'mobile_opt_in_sg_rewrite_with_importdisabled' doc_body = document.create_doc(doc_id=doc_id8, channels=['mobileOptIn'], prop_generator=update_mobile_prop) sdk_client.upsert(doc_id8, doc_body) with pytest.raises(HTTPError) as he: sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session) log_info(he.value) assert he.value.message.startswith('409 Client Error: Conflict for url:') sg_client.update_doc(url=sg_url, db=sg_db, doc_id=doc_id8, number_updates=1, auth=test_auth_session) sg_get_doc8 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id8, auth=test_auth_session) assert sg_get_doc8['_rev'].startswith('2-') and sg_get_doc8['_id'] == doc_id8 # Create ninth sdk doc with import disabled and add mobile property and update via sg. Case #8 doc_id9 = 'mobile_opt_out_sg_rewrite_with_importdisabled' doc_body = document.create_doc(doc_id=doc_id9, channels=['mobileOptIn'], prop_generator=update_non_mobile_prop) sdk_client.upsert(doc_id9, doc_body) sg_client.add_doc(url=sg_url, db=sg_db, doc=doc_body, auth=test_auth_session) # sg_client.update_doc(url=sg_url, db=sg_db, doc_id=doc_id8, number_updates=1, auth=test_auth_session) sg_get_doc9 = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id9, auth=test_auth_session) assert sg_get_doc9['_rev'].startswith('1-') and sg_get_doc9['_id'] == doc_id9
def test_stale_revision_should_not_be_in_the_index(setup_client_syncgateway_test): """original ticket: https://github.com/couchbase/couchbase-lite-android/issues/855 scenario: 1. Running sync_gateway 2. Create database and starts both push and pull replicators through client REST API 3. Create two or more views through client REST API 4. Add doc, and verify doc is index with current revision through client REST API 5. Make sure document is pushed to sync gateway through sync gateway REST API 6. Update doc with sync gateway (not client side) through sync gateway REST API 7. Make sure updated document is pull replicated to client through client REST API 8. Make sure updated document is indexed through client REST API 9. Make sure stale revision is deleted from index. through client REST API 10. Pass criteria """ cluster_config = setup_client_syncgateway_test["cluster_config"] ls_url = setup_client_syncgateway_test["ls_url"] sg_url = setup_client_syncgateway_test["sg_url"] sg_admin_url = setup_client_syncgateway_test["sg_admin_url"] num_docs = 10 num_revs = 100 d_doc_name = "dd" sg_db = "db" sg_user_name = "sg_user" sg_helper = SyncGateway() sg_helper.start_sync_gateway( cluster_config=cluster_config, url=sg_url, config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS) ) log_info("Running 'test_stale_revision_should_not_be_in_the_index'") log_info("ls_url: {}".format(ls_url)) log_info("sg_admin_url: {}".format(sg_admin_url)) log_info("sg_url: {}".format(sg_url)) log_info("num_docs: {}".format(num_docs)) log_info("num_revs: {}".format(num_revs)) client = MobileRestClient() sg_user_channels = ["NBC"] client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password="******", channels=sg_user_channels) sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name) view = """{ "language" : "javascript", "views" : { "content_view" : { "map" : "function(doc, meta) { if (doc.content) { emit(doc._id, doc._rev); } }" }, "update_view" : { "map" : "function(doc, meta) { emit(doc.updates, null); }" } } }""" ls_db = client.create_database(url=ls_url, name="ls_db") # Setup continuous push / pull replication from ls_db1 to sg_db client.start_replication( url=ls_url, continuous=True, from_db=ls_db, to_url=sg_admin_url, to_db=sg_db ) client.start_replication( url=ls_url, continuous=True, from_url=sg_admin_url, from_db=sg_db, to_db=ls_db ) design_doc_id = client.add_design_doc(url=ls_url, db=ls_db, name=d_doc_name, doc=view) client.get_doc(url=ls_url, db=ls_db, doc_id=design_doc_id) doc_body = document.create_doc(doc_id="doc_1", content={"hi": "I should be in the view"}, channels=sg_user_channels) log_info(doc_body) doc_body_2 = document.create_doc(doc_id="doc_2", channels=sg_user_channels) doc = client.add_doc(url=ls_url, db=ls_db, doc=doc_body) doc_2 = client.add_doc(url=ls_url, db=ls_db, doc=doc_body_2) content_view_rows = client.get_view(url=ls_url, db=ls_db, design_doc_id=design_doc_id, view_name="content_view") client.verify_view_row_num(view_response=content_view_rows, expected_num_rows=1) update_view_rows = client.get_view(url=ls_url, db=ls_db, design_doc_id=design_doc_id, view_name="update_view") client.verify_view_row_num(view_response=update_view_rows, expected_num_rows=2) expected_docs_list = [doc, doc_2] client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=expected_docs_list, auth=sg_session) updated_doc = client.update_doc(url=sg_url, db=sg_db, doc_id=doc["id"], number_updates=10, auth=sg_session) client.verify_docs_present(url=ls_url, db=ls_db, expected_docs=updated_doc) content_view_rows_2 = client.get_view(url=ls_url, db=ls_db, design_doc_id=design_doc_id, view_name="content_view") client.verify_view_row_num(view_response=content_view_rows_2, expected_num_rows=1) client.verify_view_contains_keys(view_response=content_view_rows_2, keys=doc["id"]) client.verify_view_contains_values(view_response=content_view_rows_2, values=updated_doc["rev"])
def test_removing_expiry(params_from_base_test_setup, sg_conf_name): """ 1. PUT /db/doc1 via SG with property "_exp":3 2. Update /db/doc1 with a new revision with no expiry value 3. After 10 updates, update /db/doc1 with a revision with no expiry """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) cluster_helper = ClusterKeywords() topology = cluster_helper.get_cluster_topology(cluster_config) cluster_helper.reset_cluster(cluster_config=cluster_config, sync_gateway_config=sg_conf) cbs_url = topology["couchbase_servers"][0] sg_url = topology["sync_gateways"][0]["public"] sg_url_admin = topology["sync_gateways"][0]["admin"] log_info("Running 'test_removing_expiry'") log_info("cbs_url: {}".format(cbs_url)) log_info("sg_url: {}".format(sg_url)) log_info("sg_url_admin: {}".format(sg_url_admin)) sg_db = "db" sg_user_name = "sg_user" sg_user_password = "******" sg_user_channels = ["NBC", "ABC"] client = MobileRestClient() client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels) sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name) doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=3, channels=sg_user_channels) doc_exp_10_body = document.create_doc(doc_id="exp_10", expiry=10, channels=sg_user_channels) doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session) doc_exp_10 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_10_body, auth=sg_user_session) doc_exp_3_updated = client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=10, auth=sg_user_session) # Sleep should allow an expiry to happen on doc_exp_3 if it had not been removed. # Expected behavior is that the doc_exp_3 will still be around due to the removal of the expiry time.sleep(5) # doc_exp_3 should no longer have an expiry and should not raise an exception doc_exp_3_updated_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3_updated["id"], auth=sg_user_session) assert doc_exp_3_updated_result["_id"] == "exp_3" # doc_exp_10 should be available still and should not raise an exception doc_exp_10_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_10["id"], auth=sg_user_session) assert doc_exp_10_result["_id"] == "exp_10"
def test_rolling_ttl_remove_expirary(params_from_base_test_setup, sg_conf_name): """ 1. PUT /db/doc1 via SG with property "_exp":3 2. Once per second for 10 seconds, update /db/doc1 with a new revision (also with "_exp":3) 3. Update /db/doc1 with a revision with no expiry 3. Get /db/doc1. Assert response is 200 """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] xattrs_enabled = params_from_base_test_setup['xattrs_enabled'] sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) cluster_helper = ClusterKeywords() topology = cluster_helper.get_cluster_topology(cluster_config) cluster_helper.reset_cluster(cluster_config=cluster_config, sync_gateway_config=sg_conf) cbs_url = topology["couchbase_servers"][0] sg_url = topology["sync_gateways"][0]["public"] sg_url_admin = topology["sync_gateways"][0]["admin"] log_info("Running 'test_rolling_ttl_remove_expirary'") log_info("cbs_url: {}".format(cbs_url)) log_info("sg_url: {}".format(sg_url)) log_info("sg_url_admin: {}".format(sg_url_admin)) sg_db = "db" sg_user_name = "sg_user" sg_user_password = "******" sg_user_channels = ["NBC", "ABC"] bucket_name = "data-bucket" cbs_ip = host_for_url(cbs_url) sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name), password='******') client = MobileRestClient() client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels) sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name) doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=3, channels=sg_user_channels) doc_exp_10_body = document.create_doc(doc_id="exp_10", expiry=10, channels=sg_user_channels) doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session) doc_exp_10 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_10_body, auth=sg_user_session) client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=10, expiry=3, delay=1, auth=sg_user_session) client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=1, auth=sg_user_session) # If expiry was not removed in the last update, this would expire doc_exp_3 time.sleep(5) # doc_exp_3 should still be around due to removal of expiry doc_exp_3 = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session) assert doc_exp_3["_id"] == "exp_3" # doc_exp_10 should be expired due to the updates (10s) + sleep (5s) with pytest.raises(HTTPError) as he: client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_10["id"], auth=sg_user_session) # In XATTR mode, the expiry results in a tombstone # In Doc Meta mode, the expiry results in a purge if xattrs_enabled: assert he.value[0].startswith("403 Client Error: Forbidden for url:") else: assert he.value[0].startswith("404 Client Error: Not Found for url:") verify_doc_deletion_on_server(doc_id=doc_exp_10["id"], sdk_client=sdk_client, sg_client=client, sg_admin_url=sg_url_admin, sg_db=sg_db, xattrs_enabled=xattrs_enabled)
def test_document_resurrection(params_from_base_test_setup, sg_conf_name, deletion_type): """ Scenarios: Doc meta mode / tombstone - Create docs (set A) via Sync Gateway - Delete docs (set A) via Sync Gateway - Verify docs (set A) are deleted via Sync Gateway - Create docs (set A) via Sync Gateway - Verify revs (set A) are generation 3 via Sync Gateway Doc meta mode / purge - Create docs (set A) via Sync Gateway - Purge docs (set A) via Sync Gateway - Verify docs (set A) are deleted via Sync Gateway - Create docs (set A) via Sync Gateway - Verify revs (set A) are generation 1 via Sync Gateway XATTRs / tombstone - Create docs (set A) via Sync Gateway - Create docs (set B) via SDK - Delete SDK docs (set B) via Sync Gateway - Delete SG docs (set A) via SDK - Verify docs (set B) are deleted via Sync Gateway - Verify docs (set B) are deleted via SDK - Verify docs (set A) are deleted via Sync Gateway - Verify docs (set A) are deleted via SDK - Create docs (set A) via Sync Gateway - Create docs (set B) via SDK - Verify revs (set A, B) are generation 3 via Sync Gateway XATTRs / purge - Create docs (set A) via Sync Gateway - Create docs (set B) via SDK - Purge SDK docs (set B) via Sync Gateway - Delete SG docs (set A) via SDK - Verify docs (set B) are deleted via Sync Gateway - Verify docs (set B) are deleted via SDK - Verify docs (set A) are deleted via Sync Gateway - Verify docs (set A) are deleted via SDK - Create docs (set A) via Sync Gateway - Create docs (set B) via SDK - Verify revs (set A, B) are generation 1 via Sync Gateway """ cluster_conf = params_from_base_test_setup['cluster_config'] cluster_topology = params_from_base_test_setup['cluster_topology'] mode = params_from_base_test_setup['mode'] xattrs_enabled = params_from_base_test_setup['xattrs_enabled'] cbs_url = cluster_topology['couchbase_servers'][0] sg_admin_url = cluster_topology['sync_gateways'][0]['admin'] sg_url = cluster_topology['sync_gateways'][0]['public'] bucket_name = 'data-bucket' sg_db = 'db' cbs_host = host_for_url(cbs_url) num_docs_per_client = 10 # Reset cluster sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) cluster = Cluster(config=cluster_conf) cluster.reset(sg_config_path=sg_conf) # Initialize clients sg_client = MobileRestClient() sdk_client = Bucket('couchbase://{}/{}'.format(cbs_host, bucket_name), password='******') # Create Sync Gateway user sg_user_channels = ['NASA', 'NATGEO'] sg_client.create_user(url=sg_admin_url, db=sg_db, name='seth', password='******', channels=sg_user_channels) sg_user_auth = sg_client.create_session(url=sg_admin_url, db=sg_db, name='seth', password='******') # Create / Add docs from SG sg_doc_bodies = document.create_docs( doc_id_prefix='sg_doc', number=num_docs_per_client, content={'foo': 'bar'}, channels=sg_user_channels, attachments_generator=attachment.generate_2_png_10_10 ) sg_doc_ids = [doc['_id'] for doc in sg_doc_bodies] sg_bulk_docs_resp = sg_client.add_bulk_docs(url=sg_url, db=sg_db, docs=sg_doc_bodies, auth=sg_user_auth) assert len(sg_bulk_docs_resp) == num_docs_per_client all_doc_ids = sg_doc_ids assert len(all_doc_ids) == num_docs_per_client if xattrs_enabled: # Create / Add docs from sdk log_info('Adding docs via SDK') sdk_doc_bodies = document.create_docs( doc_id_prefix='sdk_doc', number=num_docs_per_client, content={'foo': 'bar'}, channels=sg_user_channels, ) sdk_docs = {doc['_id']: doc for doc in sdk_doc_bodies} sdk_doc_ids = [doc['_id'] for doc in sdk_doc_bodies] log_info('Creating SDK docs') sdk_client.upsert_multi(sdk_docs) all_doc_ids = sg_doc_ids + sdk_doc_ids assert len(all_doc_ids) == num_docs_per_client * 2 if deletion_type == 'tombstone': # Set the target docs. # Doc meta mode: Delete Sync Gateway docs via Sync Gateway # XATTR mode: Delete SDK docs via Sync Gateway sg_doc_ids_to_delete = sg_doc_ids if xattrs_enabled: sg_doc_ids_to_delete = sdk_doc_ids # SG delete target docs for doc_id in sg_doc_ids_to_delete: doc = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id, auth=sg_user_auth) deleted = sg_client.delete_doc(url=sg_url, db=sg_db, doc_id=doc_id, rev=doc['_rev'], auth=sg_user_auth) log_info(deleted) if xattrs_enabled: log_info('Deleting SG docs via SDK') sdk_client.remove_multi(sg_doc_ids) elif deletion_type == 'purge': # SG Purge all docs all_docs, errors = sg_client.get_bulk_docs(url=sg_url, db=sg_db, doc_ids=all_doc_ids, auth=sg_user_auth) if xattrs_enabled: assert len(all_docs) == num_docs_per_client * 2 assert len(errors) == 0 else: assert len(all_docs) == num_docs_per_client assert len(errors) == 0 log_info('Purging docs via Sync Gateway') sg_client.purge_docs(url=sg_admin_url, db=sg_db, docs=all_docs) else: raise ValueError('Invalid test parameters') # Verify deletes via Sync Gateway deleted_docs_to_verify = sg_doc_ids assert len(deleted_docs_to_verify) == num_docs_per_client # If running is xattr mode, make sure to verify SG + SDK docs if xattrs_enabled: deleted_docs_to_verify = sg_doc_ids + sdk_doc_ids assert len(deleted_docs_to_verify) == num_docs_per_client * 2 if xattrs_enabled and deletion_type == 'tombstone': # Verify SDK + SG docs are deleted from Sync Gateway verify_sg_deletes(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth) # Verify SDK + SG docs are deleted from SDK verify_sdk_deletes(sdk_client, deleted_docs_to_verify) elif xattrs_enabled and deletion_type == 'purge': # Verify SDK + SG docs are purged from Sync Gateway verify_sg_purges(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth) # Verify SDK + SG docs are deleted from SDK verify_sdk_deletes(sdk_client, deleted_docs_to_verify) elif not xattrs_enabled and deletion_type == 'tombstone': # Doc meta: Verify SG docs are all deleted via SG verify_sg_deletes(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth) elif not xattrs_enabled and deletion_type == 'purge': # Doc meta: Verify SG docs are all deleted via SG verify_sg_purges(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth) else: raise ValueError('Invalid test parameters') # Recreate deleted docs from Sync Gateway sg_bulk_docs_resp = sg_client.add_bulk_docs(url=sg_url, db=sg_db, docs=sg_doc_bodies, auth=sg_user_auth) assert len(sg_bulk_docs_resp) == num_docs_per_client if xattrs_enabled: log_info('Recreating SDK docs') # Recreate deleted docs from SDK sdk_client.upsert_multi(sdk_docs) # Get docs via Sync Gateway doc_ids_to_get = sg_doc_ids if xattrs_enabled: doc_ids_to_get = sg_doc_ids + sdk_doc_ids docs, errors = sg_client.get_bulk_docs( url=sg_url, db=sg_db, doc_ids=doc_ids_to_get, auth=sg_user_auth, validate=False ) if xattrs_enabled: assert len(docs) == num_docs_per_client * 2 assert len(errors) == 0 else: assert len(docs) == num_docs_per_client assert len(errors) == 0 if xattrs_enabled: # Get SDK docs and makes sure all docs were recreated all_docs_from_sdk = sdk_client.get_multi(doc_ids_to_get) assert len(all_docs_from_sdk) == num_docs_per_client * 2 log_info('Found: {} recreated docs via SDK'.format(len(all_docs_from_sdk))) # Make sure we are able to get recreated docs via SDK doc_ids_to_get_scratch = list(doc_ids_to_get) assert len(doc_ids_to_get_scratch) == num_docs_per_client * 2 for doc_id in all_docs_from_sdk: doc_ids_to_get_scratch.remove(doc_id) assert len(doc_ids_to_get_scratch) == 0 # Make sure we are able to get recreated docs via SDK doc_ids_to_get_scratch = list(doc_ids_to_get) if xattrs_enabled: # SG + SDK docs assert len(doc_ids_to_get_scratch) == num_docs_per_client * 2 else: # SG docs assert len(doc_ids_to_get_scratch) == num_docs_per_client for doc in docs: # Verify expected document revisions if xattrs_enabled: if deletion_type == 'purge': # SG purges SG docs and recreates them, expecting 1- rev # SDK removes SDK docs and recreates them, expecting 1- rev assert doc['_rev'].startswith('1-') else: # SG tombstones SG docs and recreates them, expecting 3- rev # SDK removes SDK docs and recreates them, expecting 1- rev if doc['_id'].startswith('sg_'): assert doc['_rev'].startswith('3-') else: assert doc['_rev'].startswith('1-') else: if deletion_type == 'purge': # SG purges SG docs and recreates them, expecting 1- rev assert doc['_rev'].startswith('1-') else: # SG tombstones SG docs and recreates them, expecting 3- rev assert doc['_rev'].startswith('3-') doc_ids_to_get_scratch.remove(doc['_id']) # Make sure all docs were found assert len(doc_ids_to_get_scratch) == 0
def test_deleted_docs_from_changes_active_only(params_from_base_test_setup, sg_conf_name): """ https://github.com/couchbase/sync_gateway/issues/2955 1. Create a document 2. Delete the document 3. Restart Sync Gateway (to force rebuild of cache from view) 4. Issue an active_only=true changes request 5. Issue an active_only=false changes request The deleted document was not being included in the result set in step 5. """ cluster_config = params_from_base_test_setup["cluster_config"] topology = params_from_base_test_setup["cluster_topology"] sg_admin_url = topology["sync_gateways"][0]["admin"] sg_db = "db" num_docs = 10 client = MobileRestClient() # Add doc to SG added_doc = client.add_docs(url=sg_admin_url, db=sg_db, number=num_docs, id_prefix="test_changes") # Delete 1 doc doc_id = added_doc[0]["id"] log_info("Deleting {}".format(doc_id)) doc = client.get_doc(url=sg_admin_url, db=sg_db, doc_id=doc_id) doc_rev = doc['_rev'] client.delete_doc(sg_admin_url, sg_db, doc_id, doc_rev) # Restart SG sg_obj = SyncGateway() sg_obj.restart_sync_gateways(cluster_config) # Changes request with active_only=true session = Session() request_url = "{}/{}/_changes?active_only=true".format(sg_admin_url, sg_db) log_info("Issuing changes request {}".format(request_url)) resp = session.get(request_url) resp.raise_for_status() resp_obj = resp.json() log_info( "Checking that the deleted doc is not included in the active_only=true changes request" ) for d in resp_obj["results"]: assert doc_id not in d # Changes request with active_only=false request_url = "{}/{}/_changes?active_only=false".format( sg_admin_url, sg_db) log_info("Issuing changes request {}".format(request_url)) resp = session.get(request_url) resp.raise_for_status() resp_obj = resp.json() doc_found = False for d in resp_obj["results"]: if doc_id != d["id"]: continue else: assert doc_id == d["id"] assert d["deleted"] doc_found = True break log_info( "Checking that the deleted doc is included in the active_only=false changes request" ) assert doc_found
def test_migrate_conflicts_delete_last_rev(params_from_base_test_setup, sg_conf_name, num_of_docs): """ @summary Migrate conflicts to no-conflicts mode and delete last revision and verify revisions exists in open revisions Test case link : https://docs.google.com/spreadsheets/d/1YwI_gCeoBebQKBybkzoAEoXSc0XLReszDA-mPFQapgk/edit#gid=0 covered #19 Steps: 1. Start sg with default(i.e allow_conflicts=true) 2. Add docs to SG. 3. Update docs few times . 4. Create a conflicts and verify it is successful. 5. Modify sg config by enabling allow_conflicts to false 6. restart sg. 7. Delete doc by revision of current active open revision 8. Verify all revisions in history exists in open revisions which got at step 4. """ # Setup cluster_config = params_from_base_test_setup["cluster_config"] topology = params_from_base_test_setup["cluster_topology"] no_conflicts_enabled = params_from_base_test_setup["no_conflicts_enabled"] mode = params_from_base_test_setup["mode"] sg_url = topology["sync_gateways"][0]["public"] sg_admin_url = topology["sync_gateways"][0]["admin"] sync_gateway_version = params_from_base_test_setup["sync_gateway_version"] sg_db = "db" if no_conflicts_enabled or sync_gateway_version < "2.0": pytest.skip('--no-conflicts is enabled and does not work with sg < 2.0 , so skipping the test') sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) # 1. Start sg with default(i.e allow_conflicts=true) c = cluster.Cluster(cluster_config) c.reset(sg_conf) sg_client = MobileRestClient() channels = ["no-conflicts"] sg_client.create_user(url=sg_admin_url, db=sg_db, name='autotest', password='******', channels=channels) autouser_session = sg_client.create_session(url=sg_admin_url, db=sg_db, name='autotest', password='******') # end of Set up # 2. Add docs to SG. sgdoc_bodies = document.create_docs(doc_id_prefix="sg_docs", number=num_of_docs, channels=channels) sg_docs = sg_client.add_bulk_docs(url=sg_url, db=sg_db, docs=sgdoc_bodies, auth=autouser_session) assert len(sgdoc_bodies) == num_of_docs # 3. Update the docs few times prev_revs = [] for i in xrange(5): update_sg_docs = sg_client.update_docs(url=sg_url, db=sg_db, docs=sg_docs, number_updates=1, delay=None, auth=autouser_session, channels=channels) rev = update_sg_docs[0]['rev'].split('-')[1] prev_revs.append(rev) # 4. Create a conflicts and verify it is successful. for doc in sg_docs: conflicted_rev = sg_client.add_conflict(url=sg_url, db=sg_db, doc_id=doc["id"], parent_revisions=doc["rev"], new_revision="2-foo", auth=autouser_session) assert conflicted_rev["rev"] == "2-foo" for doc in sg_docs: num_of_open_revs = sg_client.get_open_revs_ids(url=sg_url, db=sg_db, doc_id=doc["id"], rev="2-foo", auth=autouser_session) time.sleep(5) # 5. Enable allow_conflicts = false in SG config and 6. restart sg revs_limit = 2 temp_cluster_config = copy_to_temp_conf(cluster_config, mode) persist_cluster_config_environment_prop(temp_cluster_config, 'no_conflicts_enabled', "True", property_name_check=False) persist_cluster_config_environment_prop(temp_cluster_config, 'revs_limit', revs_limit, property_name_check=False) status = c.sync_gateways[0].restart(config=sg_conf, cluster_config=temp_cluster_config) assert status == 0, "Syncgateway did not start after no conflicts is enabled" sg_client.update_docs(url=sg_url, db=sg_db, docs=sg_docs, number_updates=1, auth=autouser_session, channels=channels) # 6. Delete doc by revision of current active open revision for doc in sg_docs: num_of_revs = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc["id"], auth=autouser_session) sg_client.delete_doc(url=sg_url, db=sg_db, doc_id=doc["id"], rev=num_of_revs["_rev"], auth=autouser_session) # 7.Verify all revisions in history exists in open revisions which got at step 4. for doc in sg_docs: num_of_revs_history = sg_client.get_revs_num_in_history(url=sg_url, db=sg_db, doc_id=doc["id"], auth=autouser_session) for rev in num_of_revs_history: assert rev in num_of_open_revs, "Expected revision does not exist in revision history "