def test_openidconnect_invalid_scope(params_from_base_test_setup, sg_conf_name):
"""Try to discover the authenticate endpoint URL with a test provider that has an
invalid scope, and expect an error"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_invalid_scope'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
try:
discover_authenticate_endpoint(sg_url, sg_db, "testinvalidscope")
except HTTPError:
log_info("got expected HTTPError trying to get the authenticate endpoint")
# ok we got an exception, which is expected since we are using an invalid scope
return
raise Exception("Expected HTTPError since we are using invalid scope")
def test_openidconnect_no_session(params_from_base_test_setup, sg_conf_name):
"""Authenticate with a test openid provider that is configured to NOT add a Set-Cookie header"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_no_session'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
# multipart/form data content
formdata = {
'username': ('', 'testuser'),
'authenticated': ('', 'Return a valid authorization code for this user')
}
authenticate_url = discover_authenticate_url(sg_url, sg_db, "testnosessions")
# Make the request to _oidc_testing
response = requests.post(authenticate_url, files=formdata)
log_r(response)
assert "Set-Cookie" not in response.headers
def test_log_rotation_negative(params_from_base_test_setup, sg_conf_name):
"""Test log rotation with negative values for:
"maxsize": -1,
"maxage": -30,
"maxbackups": -2
SG shouldn't start
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_conf)
sg_one_url = cluster_hosts["sync_gateways"][0]["public"]
# read sample sg_conf
data = load_sync_gateway_config(sg_conf, mode,
cluster_hosts["couchbase_servers"][0],
xattrs_enabled, cluster_conf)
# set negative values for rotation section
data['logging']["default"]["rotation"] = {
"maxsize": -1,
"maxage": -30,
"maxbackups": -2,
"localtime": True
}
# create temp config file in the same folder as sg_conf
temp_conf = "/".join(sg_conf.split('/')[:-2]) + '/temp_conf.json'
with open(temp_conf, 'w') as fp:
json.dump(data, fp)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
sg_helper = SyncGateway()
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
try:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=temp_conf)
except ProvisioningError:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=sg_conf)
# Remove generated conf file
os.remove(temp_conf)
return
# Remove generated conf file
os.remove(temp_conf)
pytest.fail("SG shouldn't be started!!!!")
def test_openidconnect_oidc_challenge_invalid_provider_name(params_from_base_test_setup, sg_conf_name):
"""
If oidc_challenge is called with an invalid provider name, it should not return
an Www-Authenticate header
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_oidc_challenge_invalid_provider_name'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
# make a request to the _oidc_challenge endpoint
oidc_challenge_url = "{}/{}/_oidc_challenge?provider={}".format(sg_url, sg_db, "bogusprovider")
response = requests.get(oidc_challenge_url)
log_info("response.headers: {}".format(response.headers))
assert "Www-Authenticate" not in response.headers
assert response.status_code == 400
def install_couchbase_server(cluster_config, couchbase_server_config):
log_info(cluster_config)
log_info(couchbase_server_config)
ansible_runner = AnsibleRunner(cluster_config)
log_info(">>> Installing Couchbase Server")
# Install Server
server_baseurl, server_package_name = couchbase_server_config.get_baseurl_package()
status = ansible_runner.run_ansible_playbook(
"install-couchbase-server-package.yml",
extra_vars={
"couchbase_server_package_base_url": server_baseurl,
"couchbase_server_package_name": server_package_name
}
)
if status != 0:
raise ProvisioningError("Failed to install Couchbase Server")
# Wait for server to be in 'healthy state'
print(">>> Waiting for server to be in 'healthy' state")
cluster_keywords = ClusterKeywords()
cluster_topology = cluster_keywords.get_cluster_topology(cluster_config)
server_url = cluster_topology["couchbase_servers"][0]
cb_server = CouchbaseServer(server_url)
cb_server.wait_for_ready_state()
def params_from_base_suite_setup(request):
log_info("Setting up 'params_from_base_suite_setup' ...")
server_version = request.config.getoption("--server-version")
sync_gateway_version = request.config.getoption("--sync-gateway-version")
mode = request.config.getoption("--mode")
skip_provisioning = request.config.getoption("--skip-provisioning")
log_info("server_version: {}".format(server_version))
log_info("sync_gateway_version: {}".format(sync_gateway_version))
log_info("mode: {}".format(mode))
log_info("skip_provisioning: {}".format(skip_provisioning))
# Make sure mode for sync_gateway is supported ('cc' or 'di')
validate_sync_gateway_mode(mode)
# use base_cc cluster config if mode is "cc" or base_di cluster config if more is "di"
cluster_config = "{}/multiple_sync_gateways_{}".format(constants.CLUSTER_CONFIGS_DIR, mode)
sg_config = sync_gateway_config_path_for_mode("sync_gateway_default_functional_tests", mode)
# Skip provisioning if user specifies '--skip-provisoning'
if not skip_provisioning:
cluster_helper = ClusterKeywords()
cluster_helper.provision_cluster(
cluster_config=cluster_config,
server_version=server_version,
sync_gateway_version=sync_gateway_version,
sync_gateway_config=sg_config
)
yield {"cluster_config": cluster_config, "mode": mode}
log_info("Tearing down 'params_from_base_suite_setup' ...")
def create_server_buckets(cluster_config, sync_gateway_config):
# get the couchbase server url
cluster_helper = ClusterKeywords()
cluster_topology = cluster_helper.get_cluster_topology(cluster_config)
# Handle the case of resources/cluster_configs/1sg, where we are targeting a
# sync_gateway without a backing server
if len(cluster_topology["couchbase_servers"]) == 0:
log_info(
"The cluster_config: {} does not have a couchbase server. Skipping bucket creation!!"
.format(cluster_config))
return
couchbase_server_url = cluster_topology["couchbase_servers"][0]
# delete existing buckets
cb_server = CouchbaseServer(couchbase_server_url)
cb_server.delete_buckets()
# find bucket names from sg config
bucket_names = get_buckets_from_sync_gateway_config(
sync_gateway_config.config_path)
# create couchbase server buckets
cb_server.create_buckets(bucket_names)
def install_couchbase_server(cluster_config,
couchbase_server_config,
cbs_platform="centos7"):
log_info(cluster_config)
log_info(couchbase_server_config)
ansible_runner = AnsibleRunner(cluster_config)
cluster_keywords = ClusterKeywords()
cluster_topology = cluster_keywords.get_cluster_topology(cluster_config)
server_url = cluster_topology["couchbase_servers"][0]
cb_server = CouchbaseServer(server_url)
log_info(">>> Installing Couchbase Server")
# Install Server
server_baseurl, server_package_name = couchbase_server_config.get_baseurl_package(
cb_server, cbs_platform)
status = ansible_runner.run_ansible_playbook(
"install-couchbase-server-package.yml",
extra_vars={
"couchbase_server_package_base_url": server_baseurl,
"couchbase_server_package_name": server_package_name
})
if status != 0:
raise ProvisioningError("Failed to install Couchbase Server")
# Wait for server to be in 'healthy state'
print(">>> Waiting for server to be in 'healthy' state")
cb_server.wait_for_ready_state()
def test_rolling_ttl_remove_expirary(params_from_base_test_setup, sg_conf_name):
"""
1. PUT /db/doc1 via SG with property "_exp":3
2. Once per second for 10 seconds, update /db/doc1 with a new revision (also with "_exp":3)
3. Update /db/doc1 with a revision with no expiry
3. Get /db/doc1. Assert response is 200
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_rolling_ttl_remove_expirary'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=3, channels=sg_user_channels)
doc_exp_10_body = document.create_doc(doc_id="exp_10", expiry=10, channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session)
doc_exp_10 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_10_body, auth=sg_user_session)
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=10, expiry=3, delay=1, auth=sg_user_session)
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=1, auth=sg_user_session)
# If expiry was not removed in the last update, this would expire doc_exp_3
time.sleep(5)
# doc_exp_3 should still be around due to removal of expiry
doc_exp_3 = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session)
assert doc_exp_3["_id"] == "exp_3"
# doc_exp_10 should be expired due to the updates (10s) + sleep (5s)
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_10["id"], auth=sg_user_session)
assert he.value[0].startswith("404 Client Error: Not Found for url:")
def test_rolling_ttl_expires(params_from_base_test_setup, sg_conf_name):
"""
1. PUT /db/doc1 via SG with property "_exp":3
2. Update /db/doc1 10 times with a new revision (also with "_exp":3)
3. Wait 5 seconds
4. Get /db/doc1. Assert response is 200
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_rolling_ttl_expires'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=3, channels=sg_user_channels)
doc_exp_10_body = document.create_doc(doc_id="exp_10", expiry=10, channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session)
doc_exp_10 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_10_body, auth=sg_user_session)
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=10, expiry=3, auth=sg_user_session)
# Sleep should allow doc_exp_3 to expire, but still be in the window to get doc_exp_10
time.sleep(5)
# doc_exp_3 should be expired
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session)
assert he.value[0].startswith("404 Client Error: Not Found for url:")
# doc_exp_10 should be available still
doc_exp_10_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_10["id"], auth=sg_user_session)
assert doc_exp_10_result["_id"] == "exp_10"
def test_writing_attachment_to_couchbase_server(params_from_base_test_setup, sg_conf_name):
"""
1. Start sync_gateway with sync function that rejects all writes:
function(doc, oldDoc) {
throw({forbidden:"No writes!"});
}
2. Create a doc with attachment
3. Use CBS sdk to see if attachment doc exists. Doc ID will look like _sync:att:sha1-Kq5sNclPz7QV2+lfQIuc6R7oRu0= (where the suffix is the digest)
4. Assert att doc does not exist
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(cluster_config, sg_conf)
topology = cluster_helper.get_cluster_topology(cluster_config)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
sg_db = "db"
bucket = "data-bucket"
log_info("Running 'test_writing_attachment_to_couchbase_server'")
log_info("Using cbs_url: {}".format(cbs_url))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_url_admin: {}".format(sg_url_admin))
log_info("Using sg_db: {}".format(sg_db))
log_info("Using bucket: {}".format(bucket))
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
docs = client.add_docs(url=sg_url, db=sg_db, number=100, id_prefix=sg_db, channels=sg_user_channels, auth=sg_user_session)
assert len(docs) == 100
# Create doc with attachment and push to sync_gateway
doc_with_att = document.create_doc(doc_id="att_doc", content={"sample_key": "sample_val"}, attachment_name="sample_text.txt", channels=sg_user_channels)
client.add_doc(url=sg_url, db=sg_db, doc=doc_with_att, auth=sg_user_session)
server = CouchbaseServer(cbs_url)
# Assert that the attachment doc gets written to couchbase server
server_att_docs = server.get_server_docs_with_prefix(bucket=bucket, prefix="_sync:att:")
num_att_docs = len(server_att_docs)
assert num_att_docs == 1
def test_removing_expiry(params_from_base_test_setup, sg_conf_name):
"""
1. PUT /db/doc1 via SG with property "_exp":3
2. Update /db/doc1 with a new revision with no expiry value
3. After 10 updates, update /db/doc1 with a revision with no expiry
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_removing_expiry'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=3, channels=sg_user_channels)
doc_exp_10_body = document.create_doc(doc_id="exp_10", expiry=10, channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session)
doc_exp_10 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_10_body, auth=sg_user_session)
doc_exp_3_updated = client.update_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], number_updates=10, auth=sg_user_session)
# Sleep should allow an expiry to happen on doc_exp_3 if it had not been removed.
# Expected behavior is that the doc_exp_3 will still be around due to the removal of the expiry
time.sleep(5)
# doc_exp_3 should no longer have an expiry and should not raise an exception
doc_exp_3_updated_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3_updated["id"], auth=sg_user_session)
assert doc_exp_3_updated_result["_id"] == "exp_3"
# doc_exp_10 should be available still and should not raise an exception
doc_exp_10_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_10["id"], auth=sg_user_session)
assert doc_exp_10_result["_id"] == "exp_10"
def setup_client_syncgateway_test(request, setup_client_syncgateway_suite):
"""Test setup fixture for client sync_gateway tests"""
log_info("Setting up client sync_gateway test ...")
liteserv = setup_client_syncgateway_suite["liteserv"]
cluster_config = setup_client_syncgateway_suite["cluster_config"]
xattrs_enabled = setup_client_syncgateway_suite["xattrs_enabled"]
device_enabled = setup_client_syncgateway_suite["device_enabled"]
liteserv_platform = setup_client_syncgateway_suite["liteserv_platform"]
liteserv_version = setup_client_syncgateway_suite["liteserv_version"]
test_name = request.node.name
if request.config.getoption("--liteserv-platform") == "macosx" and \
str(request.config.getoption("--liteserv-version")).startswith("1.3.1") and \
str(test_name).startswith("test_longpoll_changes_termination"):
pytest.skip("test_longpoll_changes_termination tests are known to fail on macosx with 1.3.1 CBL")
client = MobileRestClient()
# Start LiteServ and delete any databases
log_info("Starting LiteServ...")
if device_enabled and liteserv_platform == "ios":
ls_url = liteserv.start_device("{}/logs/{}-{}-{}.txt".format(RESULTS_DIR, type(liteserv).__name__, test_name, datetime.datetime.now()))
else:
ls_url = liteserv.start("{}/logs/{}-{}-{}.txt".format(RESULTS_DIR, type(liteserv).__name__, test_name, datetime.datetime.now()))
client.delete_databases(ls_url)
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_config=cluster_config)
sg_url = cluster_hosts["sync_gateways"][0]["public"]
sg_admin_url = cluster_hosts["sync_gateways"][0]["admin"]
# Yield values to test case via fixture argument
yield {
"cluster_config": cluster_config,
"sg_mode": setup_client_syncgateway_suite["sg_mode"],
"ls_url": ls_url,
"sg_url": sg_url,
"sg_admin_url": sg_admin_url,
"xattrs_enabled": xattrs_enabled,
"liteserv": liteserv,
"liteserv_platform": liteserv_platform,
"device_enabled": device_enabled,
"liteserv_version": liteserv_version
}
log_info("Tearing down test")
client.delete_databases(ls_url)
liteserv.stop()
# if the test failed pull logs
if request.node.rep_call.failed:
logging_helper = Logging()
logging_helper.fetch_and_analyze_logs(cluster_config=cluster_config, test_name=test_name)
def test_openidconnect_large_scope(params_from_base_test_setup, sg_conf_name):
"""Authenticate against a test provider config that only has a larger scope than the default,
and make sure things like the nickname are returned in the jwt token returned back"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_large_scope'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
# multipart/form data content
formdata = {
'username': ('', 'testuser'),
'authenticated': ('', 'Return a valid authorization code for this user')
}
# get the authenticate endpoint and query params, should look something like:
# authenticate?client_id=sync_gateway&redirect_uri= ...
authenticate_endpoint = discover_authenticate_endpoint(sg_url, sg_db, "testlargescope")
# build the full url
url = "{}/{}/_oidc_testing/{}".format(
sg_url,
sg_db,
authenticate_endpoint
)
# Make the request to _oidc_testing
response = requests.post(url, files=formdata)
log_r(response)
# extract the token from the response
response_json = response.json()
id_token = response_json["id_token"]
# {u'iss': u'http://localhost:4984/db/_oidc_testing', u'iat': 1466050188, u'aud': u'sync_gateway', u'exp': 1466053788, u'sub': u'testuser'}
decoded_id_token = jwt.decode(id_token, verify=False)
log_info("decoded_id_token: {}".format(decoded_id_token))
assert "nickname" in decoded_id_token.keys()
def setup_client_2sgs_suite(request):
"""Suite setup fixture for client sync_gateway tests"""
log_info("Setting up client sync_gateway suite ...")
liteserv_platform = request.config.getoption("--liteserv-platform")
liteserv_version = request.config.getoption("--liteserv-version")
liteserv_storage_engine = request.config.getoption("--liteserv-storage-engine")
sync_gateway_version = request.config.getoption("--sync-gateway-version")
ls = LiteServ()
log_info("Downloading LiteServ One ...")
# Download LiteServ One
ls.download_liteserv(
platform=liteserv_platform,
version=liteserv_version,
storage_engine=liteserv_storage_engine
)
ls_cluster_target = None
if liteserv_platform == "net-win":
ls_cluster_target = "resources/cluster_configs/windows"
# Install LiteServ
ls.install_liteserv(
platform=liteserv_platform,
version=liteserv_version,
storage_engine=liteserv_storage_engine,
cluster_config=ls_cluster_target
)
cluster_helper = ClusterKeywords()
cluster_helper.set_cluster_config("2sgs")
cluster_config = os.environ["CLUSTER_CONFIG"]
clean_cluster(cluster_config=cluster_config)
log_info("Installing sync_gateway")
sg_helper = SyncGateway()
sg_helper.install_sync_gateway(
cluster_config=cluster_config,
sync_gateway_version=sync_gateway_version,
sync_gateway_config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS)
)
# Wait at the yeild until tests referencing this suite setup have run,
# Then execute the teardown
yield
log_info("Tearing down suite ...")
cluster_helper.unset_cluster_config()
def test_load_balance_sanity(params_from_base_test_setup):
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_util = ClusterKeywords()
cluster_util.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf_path
)
topology = cluster_util.get_cluster_topology(cluster_config)
admin_sg_one = topology["sync_gateways"][0]["admin"]
lb_url = "{}:4984".format(topology["load_balancers"][0])
sg_db = "db"
num_docs = 1000
sg_user_name = "seth"
sg_user_password = "******"
channels = ["ABC", "CBS"]
client = MobileRestClient()
user = client.create_user(admin_sg_one, sg_db, sg_user_name, sg_user_password, channels=channels)
session = client.create_session(admin_sg_one, sg_db, sg_user_name)
log_info(user)
log_info(session)
log_info("Adding docs to the load balancer ...")
ct = ChangesTracker(url=lb_url, db=sg_db, auth=session)
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
log_info("Starting ...")
ct_task = executor.submit(ct.start)
log_info("Adding docs ...")
docs = client.add_docs(lb_url, sg_db, num_docs, "test_doc", channels=channels, auth=session)
assert len(docs) == num_docs
log_info("Adding docs done")
wait_for_changes = executor.submit(ct.wait_until, docs)
if wait_for_changes.result():
log_info("Stopping ...")
log_info("Found all docs ...")
executor.submit(ct.stop)
ct_task.result()
else:
executor.submit(ct.stop)
ct_task.result()
raise Exception("Could not find all changes in feed before timeout!!")
def test_load_balance_sanity(params_from_base_test_setup):
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_util = ClusterKeywords()
cluster_util.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf_path
)
topology = cluster_util.get_cluster_topology(cluster_config)
admin_sg_one = topology["sync_gateways"][0]["admin"]
lb_url = topology["load_balancers"][0]
sg_db = "db"
num_docs = 1000
sg_user_name = "seth"
sg_user_password = "******"
channels = ["ABC", "CBS"]
client = MobileRestClient()
user = client.create_user(admin_sg_one, sg_db, sg_user_name, sg_user_password, channels=channels)
session = client.create_session(admin_sg_one, sg_db, sg_user_name)
log_info(user)
log_info(session)
log_info("Adding docs to the load balancer ...")
ct = ChangesTracker(url=lb_url, db=sg_db, auth=session)
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
log_info("Starting ...")
ct_task = executor.submit(ct.start)
log_info("Adding docs ...")
docs = client.add_docs(lb_url, sg_db, num_docs, "test_doc", channels=channels, auth=session)
assert len(docs) == num_docs
log_info("Adding docs done")
wait_for_changes = executor.submit(ct.wait_until, docs)
if wait_for_changes.result():
log_info("Stopping ...")
log_info("Found all docs ...")
executor.submit(ct.stop)
ct_task.result()
else:
executor.submit(ct.stop)
ct_task.result()
raise Exception("Could not find all changes in feed before timeout!!")
def test_setting_expiry_in_bulk_docs(params_from_base_test_setup, sg_conf_name):
"""
1. PUT /db/_bulk_docs with 10 documents. Set the "_exp":3 on 5 of these documents
2. Wait five seconds
3. POST /db/_bulk_get for the 10 documents. Validate that only the 5 non-expiring documents are returned
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_setting_expiry_in_bulk_docs'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
doc_exp_3_bodies = document.create_docs(doc_id_prefix="exp_3", number=5, expiry=3, channels=sg_user_channels)
doc_exp_10_bodies = document.create_docs(doc_id_prefix="exp_10", number=5, expiry=10, channels=sg_user_channels)
bulk_bodies = doc_exp_3_bodies + doc_exp_10_bodies
bulk_docs = client.add_bulk_docs(url=sg_url, db=sg_db, docs=bulk_bodies, auth=sg_user_session)
# Allow exp_3 docs to expire
time.sleep(5)
bulk_get_docs = client.get_bulk_docs(url=sg_url, db=sg_db, docs=bulk_docs, auth=sg_user_session)
expected_ids = ["exp_10_0", "exp_10_1", "exp_10_2", "exp_10_3", "exp_10_4"]
expected_missing_ids = ["exp_3_0", "exp_3_1", "exp_3_2", "exp_3_3", "exp_3_4"]
client.verify_doc_ids_found_in_response(response=bulk_get_docs, expected_doc_ids=expected_ids)
client.verify_doc_ids_not_found_in_response(response=bulk_get_docs, expected_missing_doc_ids=expected_missing_ids)
def test_log_rotation_invalid_path(params_from_base_test_setup, sg_conf_name):
"""Test to check that SG is not started with invalid logFilePath.
OS specific case. SG should check if path correct on startup
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_conf)
sg_one_url = cluster_hosts["sync_gateways"][0]["public"]
# read sample sg_conf
data = load_sync_gateway_config(sg_conf, mode,
cluster_hosts["couchbase_servers"][0],
xattrs_enabled, cluster_conf)
# set non existing logFilePath
data['logging']["default"]["logFilePath"] = "/12345/1231/131231.log"
# create temp config file in the same folder as sg_conf
temp_conf = "/".join(sg_conf.split('/')[:-2]) + '/temp_conf.json'
with open(temp_conf, 'w') as fp:
json.dump(data, fp)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
sg_helper = SyncGateway()
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
try:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=temp_conf)
except ProvisioningError:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=sg_conf)
# Remove generated conf file
os.remove(temp_conf)
return
# Remove generated conf file
os.remove(temp_conf)
pytest.fail("SG shouldn't be started!!!!")
def test_log_logLevel_invalid(params_from_base_test_setup, sg_conf_name):
"""Run SG with non existing logLevel value
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_conf)
sg_one_url = cluster_hosts["sync_gateways"][0]["public"]
# read sample sg_conf
data = load_sync_gateway_config(sg_conf, mode,
cluster_hosts["couchbase_servers"][0],
xattrs_enabled, cluster_conf)
# 'debugFake' invalid value for logLevel
data['logging']["default"]["logLevel"] = "debugFake"
temp_conf = "/".join(sg_conf.split('/')[:-2]) + '/temp_conf.json'
# create temp config file in the same folder as sg_conf
with open(temp_conf, 'w') as fp:
json.dump(data, fp)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
sg_helper = SyncGateway()
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
try:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=temp_conf)
except ProvisioningError:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=sg_conf)
# Remove generated conf file
os.remove(temp_conf)
return
# Remove generated conf file
os.remove(temp_conf)
pytest.fail("SG shouldn't be started!!!!")
def setup_client_syncgateway_suite(request):
"""Suite setup fixture for client sync_gateway tests"""
log_info("Setting up client sync_gateway suite ...")
liteserv_platform = request.config.getoption("--liteserv-platform")
liteserv_version = request.config.getoption("--liteserv-version")
liteserv_host = request.config.getoption("--liteserv-host")
liteserv_port = request.config.getoption("--liteserv-port")
liteserv_storage_engine = request.config.getoption("--liteserv-storage-engine")
sync_gateway_version = request.config.getoption("--sync-gateway-version")
liteserv = LiteServFactory.create(
platform=liteserv_platform,
version_build=liteserv_version,
host=liteserv_host,
port=liteserv_port,
storage_engine=liteserv_storage_engine,
)
log_info("Downloading LiteServ ...")
# Download LiteServ
liteserv.download()
# Install LiteServ
liteserv.install()
cluster_helper = ClusterKeywords()
cluster_helper.set_cluster_config("1sg")
cluster_config = os.environ["CLUSTER_CONFIG"]
clean_cluster(cluster_config=cluster_config)
log_info("Installing sync_gateway")
sg_helper = SyncGateway()
sg_helper.install_sync_gateway(
cluster_config=cluster_config,
sync_gateway_version=sync_gateway_version,
sync_gateway_config="{}/walrus.json".format(SYNC_GATEWAY_CONFIGS),
)
# Wait at the yeild until tests referencing this suite setup have run,
# Then execute the teardown
yield liteserv
log_info("Tearing down suite ...")
cluster_helper.unset_cluster_config()
liteserv.remove()
def test_log_logKeys_string(params_from_base_test_setup, sg_conf_name):
"""Negative test to verify that we are not able start SG when
logKeys is string
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
# read sample sg_conf
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_conf)
sg_one_url = cluster_hosts["sync_gateways"][0]["public"]
data = load_sync_gateway_config(sg_conf, mode,
cluster_hosts["couchbase_servers"][0],
xattrs_enabled, cluster_conf)
# set logKeys as string in config file
data['logging']["default"]["logKeys"] = "http"
# create temp config file in the same folder as sg_conf
temp_conf = "/".join(sg_conf.split('/')[:-2]) + '/temp_conf.json'
with open(temp_conf, 'w') as fp:
json.dump(data, fp)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
sg_helper = SyncGateway()
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
try:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=temp_conf)
except ProvisioningError:
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=sg_conf)
# Remove generated conf file
os.remove(temp_conf)
return
# Remove generated conf file
os.remove(temp_conf)
pytest.fail("SG shouldn't be started!!!!")
def install_nginx(cluster_config):
"""
Deploys nginx to nodes with the load_balancer tag
1. Get the sync_gateway endpoints from the cluster configuration
2. Use the endpoints to render the nginx config (resources/nginx_configs/nginx.conf)
to distribute load across the running sync_gateways.
i.e. If your 'cluster_config' has 2 sync_gateways, nginx will be setup to forward
requests to both of the sync_gateways using a weighted round robin distribution.
If you have 3, it will split the load between 3, etc ...
3. Deploy the config and install nginx on load_balancer nodes
4. Start the nginx service
"""
cluster = ClusterKeywords()
# Set lb_enable to False to get the actual SG IPs for nginx.conf
topology = cluster.get_cluster_topology(cluster_config, lb_enable=False)
# Get sync_gateway enpoints from cluster_config
# and build a string of upstream server definitions
# upstream sync_gateway {
# server 192.168.33.11:4984;
# server 192.168.33.12:4984;
# }
upstream_definition = ""
upstream_definition_admin = ""
for sg in topology["sync_gateways"]:
# string http:// to adhere to expected format for nginx.conf
ip_port = sg["public"].replace("http://", "")
ip_port_admin = sg["admin"].replace("http://", "")
upstream_definition += "server {};\n".format(ip_port)
upstream_definition_admin += "server {};\n".format(ip_port_admin)
log_info("Upstream definition: {}".format(upstream_definition))
log_info("Upstream definition admin: {}".format(upstream_definition_admin))
ansible_runner = AnsibleRunner(cluster_config)
status = ansible_runner.run_ansible_playbook(
"install-nginx.yml",
extra_vars={
"upstream_sync_gatways": upstream_definition,
"upstream_sync_gatways_admin": upstream_definition_admin
})
assert status == 0, "Failed to install nginx!"
def test_openidconnect_negative_token_expiry(params_from_base_test_setup, sg_conf_name):
"""Create a token with a negative expiry time and expect that authentication
is not possible"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_negative_token_expiry'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
token_expiry_seconds = -5
# multipart/form data content
formdata = {
'username': ('', 'testuser'),
'authenticated': ('', 'Return a valid authorization code for this user'),
'tokenttl': ('', "{}".format(token_expiry_seconds)),
}
# get the authenticate endpoint and query params, should look something like:
# authenticate?client_id=sync_gateway&redirect_uri= ...
authenticate_endpoint = discover_authenticate_endpoint(sg_url, sg_db, DEFAULT_PROVIDER)
# build the full url
url = "{}/{}/_oidc_testing/{}".format(
sg_url,
sg_db,
authenticate_endpoint
)
response = requests.post(url, files=formdata)
assert response.status_code == 500
def test_openidconnect_no_session(params_from_base_test_setup, sg_conf_name):
"""Authenticate with a test openid provider that is configured to NOT add a Set-Cookie header"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_no_session'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
# multipart/form data content
formdata = {
'username': ('', 'testuser'),
'authenticated': ('', 'Return a valid authorization code for this user')
}
authenticate_url = discover_authenticate_url(sg_url, sg_db, "testnosessions")
# Make the request to _oidc_testing
response = requests.post(authenticate_url, files=formdata)
log_r(response)
assert "Set-Cookie" not in response.headers
def test_log_nondefault_logKeys_set(params_from_base_test_setup, sg_conf_name):
"""Test to verify non default logKeys with any invalid area.
SG should work even with non existing logging area
(positive case)
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
# read sample sg_conf
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(cluster_conf)
sg_one_url = cluster_hosts["sync_gateways"][0]["public"]
data = load_sync_gateway_config(sg_conf, mode,
cluster_hosts["couchbase_servers"][0],
xattrs_enabled, cluster_conf)
# "FAKE" not valid area in logging
data['logging']["default"]["logKeys"] = ["HTTP", "FAKE"]
# create temp config file in the same folder as sg_conf
temp_conf = "/".join(sg_conf.split('/')[:-2]) + '/temp_conf.json'
with open(temp_conf, 'w') as fp:
json.dump(data, fp)
# Stop sync_gateways
log_info(">>> Stopping sync_gateway")
sg_helper = SyncGateway()
sg_helper.stop_sync_gateways(cluster_config=cluster_conf, url=sg_one_url)
# Start sync_gateways
sg_helper.start_sync_gateways(cluster_config=cluster_conf,
url=sg_one_url,
config=temp_conf)
# Remove generated conf file
os.remove(temp_conf)
def test_openidconnect_invalid_scope(params_from_base_test_setup, sg_conf_name):
"""Try to discover the authenticate endpoint URL with a test provider that has an
invalid scope, and expect an error"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_invalid_scope'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
try:
discover_authenticate_endpoint(sg_url, sg_db, "testinvalidscope")
except HTTPError:
log_info("got expected HTTPError trying to get the authenticate endpoint")
# ok we got an exception, which is expected since we are using an invalid scope
return
raise Exception("Expected HTTPError since we are using invalid scope")
def test_openidconnect_oidc_challenge_invalid_provider_name(params_from_base_test_setup, sg_conf_name):
"""
If oidc_challenge is called with an invalid provider name, it should not return
an Www-Authenticate header
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_oidc_challenge_invalid_provider_name'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
# make a request to the _oidc_challenge endpoint
oidc_challenge_url = "{}/{}/_oidc_challenge?provider={}".format(sg_url, sg_db, "bogusprovider")
response = requests.get(oidc_challenge_url)
log_info("response.headers: {}".format(response.headers))
assert "Www-Authenticate" not in response.headers
assert response.status_code == 400
def setup_client_syncgateway_test(request, setup_client_syncgateway_suite):
"""Test setup fixture for client sync_gateway tests"""
log_info("Setting up client sync_gateway test ...")
liteserv = setup_client_syncgateway_suite["liteserv"]
cluster_config = setup_client_syncgateway_suite["cluster_config"]
xattrs_enabled = setup_client_syncgateway_suite["xattrs_enabled"]
test_name = request.node.name
client = MobileRestClient()
# Start LiteServ and delete any databases
ls_url = liteserv.start("{}/logs/{}-{}-{}.txt".format(
RESULTS_DIR,
type(liteserv).__name__, test_name, datetime.datetime.now()))
client.delete_databases(ls_url)
cluster_helper = ClusterKeywords()
cluster_hosts = cluster_helper.get_cluster_topology(
cluster_config=cluster_config)
sg_url = cluster_hosts["sync_gateways"][0]["public"]
sg_admin_url = cluster_hosts["sync_gateways"][0]["admin"]
# Yield values to test case via fixture argument
yield {
"cluster_config": cluster_config,
"sg_mode": setup_client_syncgateway_suite["sg_mode"],
"ls_url": ls_url,
"sg_url": sg_url,
"sg_admin_url": sg_admin_url,
"xattrs_enabled": xattrs_enabled
}
log_info("Tearing down test")
client.delete_databases(ls_url)
liteserv.stop()
# if the test failed pull logs
if request.node.rep_call.failed:
logging_helper = Logging()
logging_helper.fetch_and_analyze_logs(cluster_config=cluster_config,
test_name=test_name)
def setup_client_syncgateway_test(request, setup_client_syncgateway_suite):
"""Test setup fixture for client sync_gateway tests"""
log_info("Setting up client sync_gateway test ...")
liteserv = setup_client_syncgateway_suite
test_name = request.node.name
ls_url = liteserv.start(
"{}/logs/{}-{}-{}.txt".format(RESULTS_DIR, type(liteserv).__name__, test_name, datetime.datetime.now())
)
cluster_helper = ClusterKeywords()
sg_helper = SyncGateway()
cluster_hosts = cluster_helper.get_cluster_topology(os.environ["CLUSTER_CONFIG"])
sg_url = cluster_hosts["sync_gateways"][0]["public"]
sg_admin_url = cluster_hosts["sync_gateways"][0]["admin"]
sg_helper.stop_sync_gateway(cluster_config=os.environ["CLUSTER_CONFIG"], url=sg_url)
# Yield values to test case via fixture argument
yield {
"cluster_config": os.environ["CLUSTER_CONFIG"],
"ls_url": ls_url,
"sg_url": sg_url,
"sg_admin_url": sg_admin_url,
}
log_info("Tearing down test")
# Teardown test
client = MobileRestClient()
client.delete_databases(ls_url)
liteserv.stop()
sg_helper.stop_sync_gateway(cluster_config=os.environ["CLUSTER_CONFIG"], url=sg_url)
# if the test failed pull logs
if request.node.rep_call.failed:
logging_helper = Logging()
logging_helper.fetch_and_analyze_logs(cluster_config=os.environ["CLUSTER_CONFIG"], test_name=test_name)
def install_nginx(cluster_config):
"""
Deploys nginx to nodes with the load_balancer tag
1. Get the sync_gateway endpoints from the cluster configuration
2. Use the endpoints to render the nginx config (resources/nginx_configs/nginx.conf)
to distribute load across the running sync_gateways.
i.e. If your 'cluster_config' has 2 sync_gateways, nginx will be setup to forward
requests to both of the sync_gateways using a weighted round robin distribution.
If you have 3, it will split the load between 3, etc ...
3. Deploy the config and install nginx on load_balancer nodes
4. Start the nginx service
"""
cluster = ClusterKeywords()
topology = cluster.get_cluster_topology(cluster_config)
# Get sync_gateway enpoints from cluster_config
# and build a string of upstream server definitions
# upstream sync_gateway {
# server 192.168.33.11:4984;
# server 192.168.33.12:4984;
# }
upstream_definition = ""
for sg in topology["sync_gateways"]:
# string http:// to adhere to expected format for nginx.conf
ip_port = sg["public"].replace("http://", "")
upstream_definition += "server {};\n".format(ip_port)
log_info("Upstream definition: ")
log_info(upstream_definition)
ansible_runner = AnsibleRunner(cluster_config)
status = ansible_runner.run_ansible_playbook(
"install-nginx.yml",
extra_vars={
"upstream_sync_gatways": upstream_definition
}
)
assert status == 0, "Failed to install nginx!"
def test_openidconnect_notauthenticated(params_from_base_test_setup, sg_conf_name):
"""Simulate a failed authentication and make sure no session is created"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_notauthenticated'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
# get the authenticate endpoint and query params, should look something like:
# authenticate?client_id=sync_gateway&redirect_uri= ...
authenticate_endpoint = discover_authenticate_endpoint(sg_url, sg_db, DEFAULT_PROVIDER)
# build the full url
authenticate_endpoint_url = "{}/{}/_oidc_testing/{}".format(
sg_url,
sg_db,
authenticate_endpoint
)
# Make the request to _oidc_testing
formdata = {
'username': ('', 'testuser'),
'notauthenticated': ('', 'Return an authorization error for this user')
}
response = requests.post(authenticate_endpoint_url, files=formdata)
assert response.status_code == 401
def test_openidconnect_large_scope(params_from_base_test_setup, sg_conf_name):
"""Authenticate against a test provider config that only has a larger scope than the default,
and make sure things like the nickname are returned in the jwt token returned back"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_large_scope'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
# multipart/form data content
formdata = {
'username': ('', 'testuser'),
'authenticated': ('', 'Return a valid authorization code for this user')
}
# get the authenticate endpoint and query params, should look something like:
# authenticate?client_id=sync_gateway&redirect_uri= ...
authenticate_endpoint = discover_authenticate_endpoint(sg_url, sg_db, "testlargescope")
# build the full url
url = "{}/{}/_oidc_testing/{}".format(
sg_url,
sg_db,
authenticate_endpoint
)
# Make the request to _oidc_testing
response = requests.post(url, files=formdata)
log_r(response)
# extract the token from the response
response_json = response.json()
id_token = response_json["id_token"]
# {u'iss': u'http://localhost:4984/db/_oidc_testing', u'iat': 1466050188, u'aud': u'sync_gateway', u'exp': 1466053788, u'sub': u'testuser'}
decoded_id_token = jwt.decode(id_token, verify=False)
log_info("decoded_id_token: {}".format(decoded_id_token))
assert "nickname" in decoded_id_token.keys()
def create_server_buckets(cluster_config, sync_gateway_config):
# get the couchbase server url
cluster_helper = ClusterKeywords()
cluster_topology = cluster_helper.get_cluster_topology(cluster_config)
# Handle the case of resources/cluster_configs/1sg, where we are targeting a
# sync_gateway without a backing server
if len(cluster_topology["couchbase_servers"]) == 0:
log_info("The cluster_config: {} does not have a couchbase server. Skipping bucket creation!!".format(cluster_config))
return
couchbase_server_url = cluster_topology["couchbase_servers"][0]
# delete existing buckets
cb_server = CouchbaseServer(couchbase_server_url)
cb_server.delete_buckets()
# find bucket names from sg config
bucket_names = get_buckets_from_sync_gateway_config(sync_gateway_config.config_path)
# create couchbase server buckets
cb_server.create_buckets(bucket_names)
def params_from_base_suite_setup(request):
log_info("Setting up 'params_from_base_suite_setup' ...")
server_version = request.config.getoption("--server-version")
sync_gateway_version = request.config.getoption("--sync-gateway-version")
mode = request.config.getoption("--mode")
skip_provisioning = request.config.getoption("--skip-provisioning")
log_info("server_version: {}".format(server_version))
log_info("sync_gateway_version: {}".format(sync_gateway_version))
log_info("mode: {}".format(mode))
log_info("skip_provisioning: {}".format(skip_provisioning))
# Make sure mode for sync_gateway is supported ('cc' or 'di')
validate_sync_gateway_mode(mode)
# Skip these tests unless you are running in 'di' mode
if mode != "di":
pytest.skip("These tests should only run in with sg_accels")
# use multiple_sg_accels_di
cluster_config = "{}/multiple_sg_accels_di".format(keywords.constants.CLUSTER_CONFIGS_DIR)
sg_config = "{}/sync_gateway_default_functional_tests_di.json".format(SYNC_GATEWAY_CONFIGS)
# Skip provisioning if user specifies '--skip-provisoning'
if not skip_provisioning:
cluster_helper = ClusterKeywords()
cluster_helper.provision_cluster(
cluster_config=cluster_config,
server_version=server_version,
sync_gateway_version=sync_gateway_version,
sync_gateway_config=sg_config
)
yield {"cluster_config": cluster_config, "mode": mode}
log_info("Tearing down 'params_from_base_suite_setup' ...")
def test_openidconnect_negative_token_expiry(params_from_base_test_setup, sg_conf_name):
"""Create a token with a negative expiry time and expect that authentication
is not possible"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_negative_token_expiry'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
token_expiry_seconds = -5
# multipart/form data content
formdata = {
'username': ('', 'testuser'),
'authenticated': ('', 'Return a valid authorization code for this user'),
'tokenttl': ('', "{}".format(token_expiry_seconds)),
}
# get the authenticate endpoint and query params, should look something like:
# authenticate?client_id=sync_gateway&redirect_uri= ...
authenticate_endpoint = discover_authenticate_endpoint(sg_url, sg_db, DEFAULT_PROVIDER)
# build the full url
url = "{}/{}/_oidc_testing/{}".format(
sg_url,
sg_db,
authenticate_endpoint
)
response = requests.post(url, files=formdata)
assert response.status_code == 500
def test_openidconnect_notauthenticated(params_from_base_test_setup, sg_conf_name):
"""Simulate a failed authentication and make sure no session is created"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_notauthenticated'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
# get the authenticate endpoint and query params, should look something like:
# authenticate?client_id=sync_gateway&redirect_uri= ...
authenticate_endpoint = discover_authenticate_endpoint(sg_url, sg_db, DEFAULT_PROVIDER)
# build the full url
authenticate_endpoint_url = "{}/{}/_oidc_testing/{}".format(
sg_url,
sg_db,
authenticate_endpoint
)
# Make the request to _oidc_testing
formdata = {
'username': ('', 'testuser'),
'notauthenticated': ('', 'Return an authorization error for this user')
}
response = requests.post(authenticate_endpoint_url, files=formdata)
assert response.status_code == 401
def test_take_down_bring_up_sg_accel_validate_cbgt(params_from_base_test_setup,
sg_conf):
"""
Scenario 1
Start with 3 sg_accels
Take down 2 sg_accels (block until down -- poll port if needed)
Doc adds with uuids (~30 sec for cbgt to reshard)
polling loop: wait for all docs to come back over changes feed
Call validate pindex with correct number of accels
Scenario 2 (Continuation)
When bringing up, you'd have to poll the cbgt_cfg until you get expected number of nodes,
then you could validate the pindex with 2 accels
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_dcp_reshard_single_sg_accel_goes_down_and_up'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_util = ClusterKeywords()
topology = cluster_util.get_cluster_topology(cluster_conf)
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
client = MobileRestClient()
doc_pusher_user_info = userinfo.UserInfo("doc_pusher",
"pass",
channels=["A"],
roles=[])
doc_pusher_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=doc_pusher_user_info.name,
password=doc_pusher_user_info.password,
channels=doc_pusher_user_info.channels)
log_info("Shutting down sg_accels: [{}, {}]".format(
cluster.sg_accels[1], cluster.sg_accels[2]))
# Shutdown two accel nodes in parallel
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as ex:
sg_accel_down_task_1 = ex.submit(cluster.sg_accels[1].stop)
sg_accel_down_task_2 = ex.submit(cluster.sg_accels[2].stop)
assert sg_accel_down_task_1.result() == 0
assert sg_accel_down_task_2.result() == 0
log_info("Finished taking nodes down!")
# It should take some time ~30 for cbgt to pick up failing nodes and reshard the pindexes. During
# this add a 1000 docs a start a longpoll changes loop to see if those docs make to to the changes feed
# If the reshard is successful they will show up at somepoint after. If not, the docs will fail to show up.
doc_pusher_docs = client.add_docs(url=sg_url,
db=sg_db,
number=1000,
id_prefix=None,
auth=doc_pusher_auth,
channels=doc_pusher_user_info.channels)
assert len(doc_pusher_docs) == 1000
client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=doc_pusher_docs,
auth=doc_pusher_auth,
polling_interval=5)
# The pindexes should be reshared at this point since all of the changes have shown up
assert cluster.validate_cbgt_pindex_distribution(num_running_sg_accels=1)
log_info("Start sg_accels: [{}, {}]".format(cluster.sg_accels[1],
cluster.sg_accels[2]))
# Start two accel nodes in parallel
status = cluster.sg_accels[1].start(sg_conf)
assert status == 0
# Poll on pIndex reshard after bring 2 accel nodes back
assert cluster.validate_cbgt_pindex_distribution_retry(
num_running_sg_accels=2)
status = cluster.sg_accels[2].start(sg_conf)
assert status == 0
# Poll on pIndex reshard after bring 2 accel nodes back
assert cluster.validate_cbgt_pindex_distribution_retry(
num_running_sg_accels=3)
def test_openidconnect_public_session_endpoint(params_from_base_test_setup, sg_conf_name):
"""Create a new session from the OpenID Connect token returned by hitting
the public _session endpoint and make sure the response contains the Set-Cookie header."""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
sg_url = topology["sync_gateways"][0]["public"]
sg_db = "db"
log_info("Running 'test_openidconnect_public_session_endpoint'")
log_info("Using cluster_config: {}".format(cluster_config))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_db: {}".format(sg_db))
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
# multipart/form data content
formdata = {
'username': ('', 'testuser'),
'authenticated': ('', 'Return a valid authorization code for this user')
}
# get the authenticate endpoint and query params, should look something like:
# authenticate?client_id=sync_gateway&redirect_uri= ...
authenticate_endpoint = discover_authenticate_endpoint(sg_url, sg_db, DEFAULT_PROVIDER)
# build the full url
url = "{}/{}/_oidc_testing/{}".format(
sg_url,
sg_db,
authenticate_endpoint
)
# Make the request to _oidc_testing
response = requests.post(url, files=formdata)
log_r(response)
# extract the token from the response
response_json = response.json()
id_token = response_json["id_token"]
headers = {
"Authorization": "Bearer {}".format(id_token),
"Content-Type": "application/json"
}
url = "{}/{}/_session".format(
sg_url,
sg_db
)
response = requests.post(url, headers=headers)
assert "Set-Cookie" in response.headers.keys()
set_cookie_response = response.headers['Set-Cookie']
assert "SyncGatewaySession" in set_cookie_response
def test_attachment_revpos_when_ancestor_unavailable(params_from_base_test_setup, sg_conf_name):
"""
Creates a document with an attachment, then updates that document so that
the body of the revision that originally pushed the document is no
longer available. Add a new revision that's not a child of the
active revision, and validate that it's uploaded successfully.
Example:
1. Document is created with attachment at rev-1
2. Document is updated (strip digests and length, only put revpos & stub) multiple times on the server, goes to rev-4
3. Client attempts to add a new (conflicting) revision 2, with parent rev-1.
4. If the body of rev-1 is no longer available on the server (temporary backup of revision has expired, and is no longer stored
in the in-memory rev cache), we were throwing an error to client
because we couldn't verify based on the _attachments property in rev-1.
5. In this scenario, before returning error, we are now checking if the active revision has a common ancestor with the incoming revision.
If so, we can validate any revpos values equal to or earlier than the common ancestor against the active revision
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(cluster_config, sg_conf)
topology = cluster_helper.get_cluster_topology(cluster_config)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
sg_db = "db"
bucket = "data-bucket"
log_info("Running 'test_attachment_revpos_when_ancestor_unavailable'")
log_info("Using cbs_url: {}".format(cbs_url))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_url_admin: {}".format(sg_url_admin))
log_info("Using sg_db: {}".format(sg_db))
log_info("Using bucket: {}".format(bucket))
channels_list = ["ABC"]
client = MobileRestClient()
sg_util = SyncGateway()
cb_server = CouchbaseServer(cbs_url)
user1 = client.create_user(url=sg_url_admin, db=sg_db, name="user1", password="******", channels=channels_list)
doc_with_att = document.create_doc(doc_id="att_doc", content={"sample_key": "sample_val"}, attachment_name="sample_text.txt", channels=channels_list)
doc_gen_1 = client.add_doc(url=sg_url, db=sg_db, doc=doc_with_att, auth=user1)
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_gen_1["id"], number_updates=10, auth=user1)
# Clear cached rev doc bodys from server and cycle sync_gateway
sg_util.stop_sync_gateway(cluster_config=cluster_config, url=sg_url)
cb_server.delete_couchbase_server_cached_rev_bodies(bucket=bucket)
sg_util.start_sync_gateway(cluster_config=cluster_config, url=sg_url, config=sg_conf)
client.add_conflict(
url=sg_url, db=sg_db,
doc_id=doc_gen_1["id"],
parent_revisions=doc_gen_1["rev"],
new_revision="2-foo",
auth=user1
)
def setup_client_syncgateway_suite(request):
"""Suite setup fixture for client sync_gateway tests"""
log_info("Setting up client sync_gateway suite ...")
liteserv_platform = request.config.getoption("--liteserv-platform")
liteserv_version = request.config.getoption("--liteserv-version")
liteserv_host = request.config.getoption("--liteserv-host")
liteserv_port = request.config.getoption("--liteserv-port")
liteserv_storage_engine = request.config.getoption("--liteserv-storage-engine")
skip_provisioning = request.config.getoption("--skip-provisioning")
sync_gateway_version = request.config.getoption("--sync-gateway-version")
sync_gateway_mode = request.config.getoption("--sync-gateway-mode")
server_version = request.config.getoption("--server-version")
xattrs_enabled = request.config.getoption("--xattrs")
device_enabled = request.config.getoption("--device")
liteserv = LiteServFactory.create(platform=liteserv_platform,
version_build=liteserv_version,
host=liteserv_host,
port=liteserv_port,
storage_engine=liteserv_storage_engine)
if xattrs_enabled and version_is_binary(sync_gateway_version):
check_xattr_support(server_version, sync_gateway_version)
log_info("Downloading LiteServ ...")
# Download LiteServ
liteserv.download()
# Install LiteServ
if device_enabled and liteserv_platform == "ios":
liteserv.install_device()
else:
liteserv.install()
cluster_config = "{}/base_{}".format(CLUSTER_CONFIGS_DIR, sync_gateway_mode)
try:
server_version
except NameError:
log_info("Server version is not provided")
persist_cluster_config_environment_prop(cluster_config, 'server_version', "")
else:
log_info("Running test with server version {}".format(server_version))
persist_cluster_config_environment_prop(cluster_config, 'server_version', server_version)
try:
sync_gateway_version
except NameError:
log_info("Sync gateway version is not provided")
persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_version', "")
else:
log_info("Running test with sync_gateway version {}".format(sync_gateway_version))
persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_version', sync_gateway_version)
if xattrs_enabled:
log_info("Running test with xattrs for sync meta storage")
persist_cluster_config_environment_prop(cluster_config, 'xattrs_enabled', True)
else:
log_info("Using document storage for sync meta data")
persist_cluster_config_environment_prop(cluster_config, 'xattrs_enabled', False)
sg_config = sync_gateway_config_path_for_mode("listener_tests/listener_tests", sync_gateway_mode)
if not skip_provisioning:
log_info("Installing Sync Gateway + Couchbase Server + Accels ('di' only)")
cluster_utils = ClusterKeywords()
cluster_utils.provision_cluster(
cluster_config=cluster_config,
server_version=server_version,
sync_gateway_version=sync_gateway_version,
sync_gateway_config=sg_config
)
# Wait at the yeild until tests referencing this suite setup have run,
# Then execute the teardown
yield {
"liteserv": liteserv,
"cluster_config": cluster_config,
"sg_mode": sync_gateway_mode,
"xattrs_enabled": xattrs_enabled,
"device_enabled": device_enabled,
"liteserv_platform": liteserv_platform,
"liteserv_version": liteserv_version
}
log_info("Tearing down suite ...")
if not (device_enabled and liteserv_platform == "ios"):
liteserv.remove()
def test_string_expiry_as_unix_date(params_from_base_test_setup, sg_conf_name):
"""
1. Calculate (server time + 3 seconds) as unix time (i.e. Epoch time, e.g. 1466465122)
2. PUT /db/doc1 via SG with property "_exp":"[unix time]"
PUT /db/doc2 via SG with property "_exp":"1767225600" (Jan 1 2026) Note: the maximum epoch time supported by CBS is maxUint32, or Sun 07 Feb 2106, in case you want to move it out further than 2026.
3. Wait five seconds
4. Get /db/doc1. Assert response is 404
Get /db/doc2. Assert response is 200
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_string_expiry_as_unix_date'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
time_util = Time()
unix_time_3s_ahead = time_util.get_unix_timestamp(delta=3)
# Convert unix timestamp to string
unix_time_3s_ahead_string = str(unix_time_3s_ahead)
# Using string representation for unix time
doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=unix_time_3s_ahead_string, channels=sg_user_channels)
doc_exp_years_body = document.create_doc(doc_id="exp_years", expiry="1767225600", channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session)
doc_exp_years = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_years_body, auth=sg_user_session)
# Sleep should allow doc_exp_3 to expire
time.sleep(10)
# doc_exp_3 should be expired
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session)
assert he.value[0].startswith("404 Client Error: Not Found for url:")
# doc_exp_years should be available still
doc_exp_years_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_years["id"], auth=sg_user_session)
assert doc_exp_years_result["_id"] == "exp_years"
def test_removing_expiry(params_from_base_test_setup, sg_conf_name):
"""
1. PUT /db/doc1 via SG with property "_exp":3
2. Update /db/doc1 with a new revision with no expiry value
3. After 10 updates, update /db/doc1 with a revision with no expiry
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_removing_expiry'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
doc_exp_3_body = document.create_doc(doc_id="exp_3",
expiry=3,
channels=sg_user_channels)
doc_exp_10_body = document.create_doc(doc_id="exp_10",
expiry=10,
channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_3_body,
auth=sg_user_session)
doc_exp_10 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_10_body,
auth=sg_user_session)
doc_exp_3_updated = client.update_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3["id"],
number_updates=10,
auth=sg_user_session)
# Sleep should allow an expiry to happen on doc_exp_3 if it had not been removed.
# Expected behavior is that the doc_exp_3 will still be around due to the removal of the expiry
time.sleep(5)
# doc_exp_3 should no longer have an expiry and should not raise an exception
doc_exp_3_updated_result = client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3_updated["id"],
auth=sg_user_session)
assert doc_exp_3_updated_result["_id"] == "exp_3"
# doc_exp_10 should be available still and should not raise an exception
doc_exp_10_result = client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_10["id"],
auth=sg_user_session)
assert doc_exp_10_result["_id"] == "exp_10"
def test_numeric_expiry_as_ttl(params_from_base_test_setup, sg_conf_name):
"""
1. PUT /db/doc1 via SG with property "_exp":3
PUT /db/doc2 via SG with property "_exp":10
2. Wait five seconds
3. Get /db/doc1. Assert response is 404
Get /db/doc2. Assert response is 200
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_numeric_expiry_as_ttl'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
bucket_name = "data-bucket"
cbs_ip = host_for_url(cbs_url)
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name),
password='******')
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
doc_exp_3_body = document.create_doc(doc_id="exp_3",
expiry=3,
channels=sg_user_channels)
doc_exp_10_body = document.create_doc(doc_id="exp_10",
expiry=10,
channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_3_body,
auth=sg_user_session)
doc_exp_10 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_10_body,
auth=sg_user_session)
# Sleep should allow doc_exp_3 to expire, but still be in the window to get doc_exp_10
time.sleep(5)
# doc_exp_3 should be expired
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3["id"],
auth=sg_user_session)
# In XATTR mode, the expiry results in a tombstone
# In Doc Meta mode, the expiry results in a purge
if xattrs_enabled:
assert he.value[0].startswith("403 Client Error: Forbidden for url:")
else:
assert he.value[0].startswith("404 Client Error: Not Found for url:")
verify_doc_deletion_on_server(doc_id=doc_exp_3["id"],
sdk_client=sdk_client,
sg_client=client,
sg_admin_url=sg_url_admin,
sg_db=sg_db,
xattrs_enabled=xattrs_enabled)
# doc_exp_10 should be available still
doc_exp_10_result = client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_10["id"],
auth=sg_user_session)
assert doc_exp_10_result["_id"] == "exp_10"
def test_string_expiry_as_iso_8601_date(params_from_base_test_setup,
sg_conf_name):
"""
1. Calculate (server time + 3 seconds) as ISO-8601 date (e.g. 2016-01-01T00:00:00.000+00:00)
2. PUT /db/doc1 via SG with property "_exp":"[date]"
PUT /db/doc2 via SG with property "_exp":"2026-01-01T00:00:00.000+00:00"
3. Wait five seconds
4. Get /db/doc1. Assert response is 404
Get /db/doc2. Assert response is 20
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_string_expiry_as_ISO_8601_Date'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
bucket_name = "data-bucket"
cbs_ip = host_for_url(cbs_url)
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name),
password='******')
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
time_util = Time()
iso_datetime = time_util.get_iso_datetime(delta=3)
doc_exp_3_body = document.create_doc(doc_id="exp_3",
expiry=iso_datetime,
channels=sg_user_channels)
doc_exp_years_body = document.create_doc(
doc_id="exp_years",
expiry="2026-01-01T00:00:00.000+00:00",
channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_3_body,
auth=sg_user_session)
doc_exp_years = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_years_body,
auth=sg_user_session)
# Sleep should allow doc_exp_3 to expire
time.sleep(10)
# doc_exp_3 should be expired
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3["id"],
auth=sg_user_session)
# In XATTR mode, the expiry results in a tombstone
# In Doc Meta mode, the expiry results in a purge
if xattrs_enabled:
assert he.value[0].startswith("403 Client Error: Forbidden for url:")
else:
assert he.value[0].startswith("404 Client Error: Not Found for url:")
verify_doc_deletion_on_server(doc_id=doc_exp_3["id"],
sdk_client=sdk_client,
sg_client=client,
sg_admin_url=sg_url_admin,
sg_db=sg_db,
xattrs_enabled=xattrs_enabled)
# doc_exp_years should be available still
doc_exp_years_result = client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_years["id"],
auth=sg_user_session)
assert doc_exp_years_result["_id"] == "exp_years"
def test_string_expiry_as_unix_date(params_from_base_test_setup, sg_conf_name):
"""
1. Calculate (server time + 3 seconds) as unix time (i.e. Epoch time, e.g. 1466465122)
2. PUT /db/doc1 via SG with property "_exp":"[unix time]"
PUT /db/doc2 via SG with property "_exp":"1767225600" (Jan 1 2026) Note: the maximum epoch time supported by CBS is maxUint32, or Sun 07 Feb 2106, in case you want to move it out further than 2026.
3. Wait five seconds
4. Get /db/doc1. Assert response is 404
Get /db/doc2. Assert response is 200
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_string_expiry_as_unix_date'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
bucket_name = "data-bucket"
cbs_ip = host_for_url(cbs_url)
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name),
password='******')
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
time_util = Time()
unix_time_3s_ahead = time_util.get_unix_timestamp(delta=3)
# Convert unix timestamp to string
unix_time_3s_ahead_string = str(unix_time_3s_ahead)
# Using string representation for unix time
doc_exp_3_body = document.create_doc(doc_id="exp_3",
expiry=unix_time_3s_ahead_string,
channels=sg_user_channels)
doc_exp_years_body = document.create_doc(doc_id="exp_years",
expiry="1767225600",
channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_3_body,
auth=sg_user_session)
doc_exp_years = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_years_body,
auth=sg_user_session)
# Sleep should allow doc_exp_3 to expire
time.sleep(10)
# doc_exp_3 should be expired
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3["id"],
auth=sg_user_session)
# In XATTR mode, the expiry results in a tombstone
# In Doc Meta mode, the expiry results in a purge
if xattrs_enabled:
assert he.value[0].startswith("403 Client Error: Forbidden for url:")
else:
assert he.value[0].startswith("404 Client Error: Not Found for url:")
verify_doc_deletion_on_server(doc_id=doc_exp_3["id"],
sdk_client=sdk_client,
sg_client=client,
sg_admin_url=sg_url_admin,
sg_db=sg_db,
xattrs_enabled=xattrs_enabled)
# doc_exp_years should be available still
doc_exp_years_result = client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_years["id"],
auth=sg_user_session)
assert doc_exp_years_result["_id"] == "exp_years"
def test_rebalance_sanity(params_from_base_test_setup):
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
cluster_helper = ClusterKeywords()
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf_path)
topology = cluster_helper.get_cluster_topology(cluster_config)
admin_sg_one = topology["sync_gateways"][0]["admin"]
sg_one_url = topology["sync_gateways"][0]["public"]
cluster_servers = topology["couchbase_servers"]
cbs_one_url = cluster_servers[0]
cbs_two_url = cluster_servers[1]
log_info("Running: 'test_distributed_index_rebalance_sanity'")
log_info("cluster_config: {}".format(cluster_config))
log_info("admin_sg: {}".format(admin_sg_one))
log_info("sg_url: {}".format(sg_one_url))
log_info("cbs_one_url: {}".format(cbs_one_url))
log_info("cbs_two_url: {}".format(cbs_two_url))
sg_db = "db"
num_docs = 100
num_updates = 100
sg_user_name = "seth"
sg_user_password = "******"
channels = ["ABC", "CBS"]
client = MobileRestClient()
cb_server = CouchbaseServer(cbs_one_url)
server_to_remove = CouchbaseServer(cbs_two_url)
client.create_user(admin_sg_one, sg_db, sg_user_name, sg_user_password, channels=channels)
session = client.create_session(admin_sg_one, sg_db, sg_user_name)
with concurrent.futures.ThreadPoolExecutor(5) as executor:
# Add docs to sg
log_info("Adding docs to sync_gateway")
docs = client.add_docs(sg_one_url, sg_db, num_docs, "test_doc", channels=channels, auth=session)
assert len(docs) == num_docs
# Start updating docs and rebalance out one CBS node
log_info("Updating docs on sync_gateway")
update_docs_task = executor.submit(client.update_docs, sg_one_url, sg_db, docs, num_updates, auth=session)
# Run rebalance in background
cb_server.rebalance_out(cluster_servers, server_to_remove)
updated_docs = update_docs_task.result()
log_info(updated_docs)
# Verify docs / revisions present
client.verify_docs_present(sg_one_url, sg_db, updated_docs, auth=session)
# Verify docs revisions in changes feed
client.verify_docs_in_changes(sg_one_url, sg_db, updated_docs, auth=session)
# Rebalance Server back in to the pool
cb_server.add_node(server_to_remove)
cb_server.rebalance_in(cluster_servers, server_to_remove)
def test_take_down_bring_up_sg_accel_validate_cbgt(params_from_base_test_setup, sg_conf):
"""
Scenario 1
Start with 3 sg_accels
Take down 2 sg_accels (block until down -- poll port if needed)
Doc adds with uuids (~30 sec for cbgt to reshard)
polling loop: wait for all docs to come back over changes feed
Call validate pindex with correct number of accels
Scenario 2 (Continuation)
When bringing up, you'd have to poll the cbgt_cfg until you get expected number of nodes,
then you could validate the pindex with 2 accels
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_dcp_reshard_single_sg_accel_goes_down_and_up'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_util = ClusterKeywords()
topology = cluster_util.get_cluster_topology(cluster_conf)
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
client = MobileRestClient()
doc_pusher_user_info = userinfo.UserInfo("doc_pusher", "pass", channels=["A"], roles=[])
doc_pusher_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=doc_pusher_user_info.name,
password=doc_pusher_user_info.password,
channels=doc_pusher_user_info.channels
)
log_info("Shutting down sg_accels: [{}, {}]".format(cluster.sg_accels[1], cluster.sg_accels[2]))
# Shutdown two accel nodes in parallel
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as ex:
sg_accel_down_task_1 = ex.submit(cluster.sg_accels[1].stop)
sg_accel_down_task_2 = ex.submit(cluster.sg_accels[2].stop)
assert sg_accel_down_task_1.result() == 0
assert sg_accel_down_task_2.result() == 0
log_info("Finished taking nodes down!")
# It should take some time ~30 for cbgt to pick up failing nodes and reshard the pindexes. During
# this add a 1000 docs a start a longpoll changes loop to see if those docs make to to the changes feed
# If the reshard is successful they will show up at somepoint after. If not, the docs will fail to show up.
doc_pusher_docs = client.add_docs(
url=sg_url,
db=sg_db,
number=1000,
id_prefix=None,
auth=doc_pusher_auth,
channels=doc_pusher_user_info.channels
)
assert len(doc_pusher_docs) == 1000
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=doc_pusher_docs, auth=doc_pusher_auth, polling_interval=5)
# The pindexes should be reshared at this point since all of the changes have shown up
assert cluster.validate_cbgt_pindex_distribution(num_running_sg_accels=1)
log_info("Start sg_accels: [{}, {}]".format(cluster.sg_accels[1], cluster.sg_accels[2]))
# Start two accel nodes in parallel
status = cluster.sg_accels[1].start(sg_conf)
assert status == 0
# Poll on pIndex reshard after bring 2 accel nodes back
assert cluster.validate_cbgt_pindex_distribution_retry(num_running_sg_accels=2)
status = cluster.sg_accels[2].start(sg_conf)
assert status == 0
# Poll on pIndex reshard after bring 2 accel nodes back
assert cluster.validate_cbgt_pindex_distribution_retry(num_running_sg_accels=3)
def test_server_goes_down_sanity(params_from_base_test_setup):
"""
1. Start with a two node couchbase server cluster
2. Starting adding docs
3. Kill one of the server nodes and signal completion
4. Stop adding docs
5. Verify that that the expected docs are present and in the changes feed.
6. Start server again and add to cluster
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
cluster_helper = ClusterKeywords()
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf_path)
topology = cluster_helper.get_cluster_topology(cluster_config)
admin_sg = topology["sync_gateways"][0]["admin"]
sg_url = topology["sync_gateways"][0]["public"]
coucbase_servers = topology["couchbase_servers"]
cbs_one_url = coucbase_servers[0]
cbs_two_url = coucbase_servers[1]
log_info("Running: 'test_server_goes_down_sanity'")
log_info("cluster_config: {}".format(cluster_config))
log_info("admin_sg: {}".format(admin_sg))
log_info("sg_url: {}".format(sg_url))
log_info("cbs_one_url: {}".format(cbs_one_url))
log_info("cbs_two_url: {}".format(cbs_two_url))
sg_db = "db"
num_docs = 100
sg_user_name = "seth"
sg_user_password = "******"
channels = ["ABC", "CBS"]
client = MobileRestClient()
main_server = CouchbaseServer(cbs_one_url)
flakey_server = CouchbaseServer(cbs_two_url)
client.create_user(admin_sg, sg_db, sg_user_name, sg_user_password, channels=channels)
session = client.create_session(admin_sg, sg_db, sg_user_name)
# Stop second server
flakey_server.stop()
# Try to add 100 docs in a loop until all succeed, if the never do, fail with timeout
errors = num_docs
# Wait 30 seconds for auto failover
# (Minimum value suggested - http://docs.couchbase.com/admin/admin/Tasks/tasks-nodeFailover.html)
# + 15 seconds to add docs
timeout = 45
start = time.time()
successful_add = False
while not successful_add:
# Fail tests if all docs do not succeed before timeout
if (time.time() - start) > timeout:
# Bring server back up before failing the test
flakey_server.start()
main_server.rebalance_in(coucbase_servers, flakey_server)
raise TimeoutError("Failed to successfully put docs before timeout")
try:
docs = client.add_docs(url=sg_url, db=sg_db, number=num_docs, id_prefix=None, auth=session, channels=channels)
# If the above add doc does not throw, it was a successfull add.
successful_add = True
except requests.exceptions.HTTPError as he:
log_info("Failed to add docs: {}".format(he))
log_info("Seeing: {} errors".format(errors))
time.sleep(1)
assert len(docs) == 100
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=docs, auth=session)
try:
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=docs, auth=session, polling_interval=5)
except keywords.exceptions.TimeoutException:
# timeout verifying docs. Bring server back in to restore topology, then fail
# Failing due to https://github.com/couchbase/sync_gateway/issues/2197
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
raise keywords.exceptions.TimeoutException("Failed to get all changes")
# Test succeeded without timeout, bring server back into topology
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
# Make sure all docs were not added before server was
log_info("test_server_goes_down_sanity complete!")
def test_take_all_sgaccels_down(params_from_base_test_setup, sg_conf):
"""
Scenario that takes all sync_gateway accel nodes offline during doc load.
After bring the nodes back online during load, the reshard of the DCP feed is verified.
The changes feed is verified that all docs show up.
1. Start doc load (1000 doc)
2. Take all sg_accel nodes down in parallel
3. Verify node are down
4. Wait for doc adds to complete, store "doc_push_result_1"
5. Verify "doc_push_result_1" docs added
6. Start doc load (1000 docs)
7. Wait for 5. to complete, store "doc_push_result_2"
8. Verify "doc_push_result_2" docs added
9. Start another doc load (1000 docs)
10. Bring up nodes in parallel
11. poll on p-index reshard
12. Wait for 9. to complete, store "doc_push_result_3"
13. Verify "doc_push_result_3" docs added
14. Verify "doc_push_result_1" + "doc_push_result_2" + "doc_push_result_3" show up in _changes feed
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_dcp_reshard_single_sg_accel_goes_down_and_up'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_util = ClusterKeywords()
topology = cluster_util.get_cluster_topology(cluster_conf)
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
num_docs = 1000
client = MobileRestClient()
doc_pusher_user_info = userinfo.UserInfo("doc_pusher",
"pass",
channels=["A"],
roles=[])
doc_pusher_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=doc_pusher_user_info.name,
password=doc_pusher_user_info.password,
channels=doc_pusher_user_info.channels)
a_user_info = userinfo.UserInfo("a_user", "pass", channels=["A"], roles=[])
client.create_user(url=sg_admin_url,
db=sg_db,
name=a_user_info.name,
password=a_user_info.password,
channels=a_user_info.channels)
a_user_session = client.create_session(url=sg_admin_url,
db=sg_db,
name=a_user_info.name,
password=a_user_info.password)
# Shutdown all accel nodes in parallel
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as ex:
# Start adding docs
docs_1 = document.create_docs(None,
num_docs,
channels=doc_pusher_user_info.channels)
docs_1_task = ex.submit(client.add_bulk_docs,
url=sg_url,
db=sg_db,
docs=docs_1,
auth=doc_pusher_auth)
# Take down all access nodes
log_info("Shutting down sg_accels: [{}, {}, {}] ...".format(
cluster.sg_accels[0], cluster.sg_accels[1], cluster.sg_accels[2]))
sg_accel_down_task_1 = ex.submit(cluster.sg_accels[0].stop)
sg_accel_down_task_2 = ex.submit(cluster.sg_accels[1].stop)
sg_accel_down_task_3 = ex.submit(cluster.sg_accels[2].stop)
assert sg_accel_down_task_1.result() == 0
assert sg_accel_down_task_2.result() == 0
assert sg_accel_down_task_3.result() == 0
# Block until bulk_docs is complete
doc_push_result_1 = docs_1_task.result()
assert len(doc_push_result_1) == num_docs
client.verify_docs_present(url=sg_url,
db=sg_db,
expected_docs=doc_push_result_1,
auth=doc_pusher_auth)
# Load sync_gateway with another batch of docs while the sg_accel nodes are offline
docs_2_bodies = document.create_docs(
None, num_docs, channels=doc_pusher_user_info.channels)
docs_push_result_2 = client.add_bulk_docs(url=sg_url,
db=sg_db,
docs=docs_2_bodies,
auth=doc_pusher_auth)
assert len(docs_push_result_2) == num_docs
client.verify_docs_present(url=sg_url,
db=sg_db,
expected_docs=docs_push_result_2,
auth=doc_pusher_auth)
# Start loading Sync Gateway with another set of docs while bringing the sg_accel nodes online
docs_3 = document.create_docs(None,
num_docs,
channels=doc_pusher_user_info.channels)
docs_3_task = ex.submit(client.add_bulk_docs,
url=sg_url,
db=sg_db,
docs=docs_3,
auth=doc_pusher_auth)
# Bring all the sg_accel nodes back up
# Take down all access nodes
log_info("Starting sg_accels: [{}, {}, {}] ...".format(
cluster.sg_accels[0], cluster.sg_accels[1], cluster.sg_accels[2]))
sg_accel_up_task_1 = ex.submit(cluster.sg_accels[0].start, sg_conf)
sg_accel_up_task_2 = ex.submit(cluster.sg_accels[1].start, sg_conf)
sg_accel_up_task_3 = ex.submit(cluster.sg_accels[2].start, sg_conf)
assert sg_accel_up_task_1.result() == 0
assert sg_accel_up_task_2.result() == 0
assert sg_accel_up_task_3.result() == 0
# Wait for pindex to reshard correctly
assert cluster.validate_cbgt_pindex_distribution_retry(3)
# Block until second bulk_docs is complete
doc_push_result_3 = docs_3_task.result()
assert len(doc_push_result_3) == num_docs
client.verify_docs_present(url=sg_url,
db=sg_db,
expected_docs=doc_push_result_3,
auth=doc_pusher_auth)
# Combine the 3 push results and make sure the changes propagate to a_user
# a_user has access to the doc's channel.
log_info("Verifying all the changes show up for 'a_user' ...")
all_docs = doc_push_result_1 + docs_push_result_2 + doc_push_result_3
client.verify_docs_in_changes(url=sg_url,
db=sg_db,
expected_docs=all_docs,
auth=a_user_session,
polling_interval=2)
def test_detect_stale_channel_index(params_from_base_test_setup, sg_conf):
"""
1. Bring up single Sync Gateway node, backed by Couchbase Server with 3 accels indexing
2. Configure such that the primary bucket and the channel index bucket are different (which is the norm)
3. Add 1000 documents
4. Shutdown Sync Gateway
5. Delete / create the primary bucket ('data-bucket'), but do not touch the channel index bucket
6. Start Sync Gateway
7. Assert that sync_gateway fails to start due to stale channel index
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_detect_stale_channel_index'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_util = ClusterKeywords()
topology = cluster_util.get_cluster_topology(cluster_conf)
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
cb_server_url = topology["couchbase_servers"][0]
sg_db = "db"
num_docs = 1000
cb_server = couchbaseserver.CouchbaseServer(url=cb_server_url)
client = MobileRestClient()
# Create doc pusher user
doc_pusher_user_info = userinfo.UserInfo(name="doc_pusher",
password="******",
channels=["NASA"],
roles=[])
doc_pusher_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=doc_pusher_user_info.name,
password=doc_pusher_user_info.password,
channels=doc_pusher_user_info.channels)
# Add some docs to Sync Gateway to cause indexing
docs = document.create_docs(None,
number=num_docs,
channels=doc_pusher_user_info.channels)
pushed_docs = client.add_bulk_docs(url=sg_url,
db=sg_db,
docs=docs,
auth=doc_pusher_auth)
assert len(pushed_docs) == num_docs
# Shut down sync_gateway
sg_util = SyncGateway()
sg_util.stop_sync_gateways(cluster_config=cluster_conf, url=sg_url)
# Delete server bucket
cb_server.delete_bucket(name="data-bucket")
# Create server bucket
ram_per_bucket_mb = cb_server.get_ram_per_bucket(num_buckets=2)
cb_server.create_bucket(name="data-bucket", ram_quota_mb=ram_per_bucket_mb)
# Start sync_gateway and assert that a Provisioning error is raised due to detecting stale index
with pytest.raises(exceptions.ProvisioningError):
sg_util.start_sync_gateways(cluster_config=cluster_conf,
url=sg_url,
config=sg_conf)
# TODO: To make this check even more accurate, could
# run remote ssh command "systemctl status sync_gateway.service" and look for
# regex pattern: Main PID: 7185 (code=exited, status=2)
# Delete index bucket and recreate it
cb_server.delete_bucket(name="index-bucket")
cb_server.create_bucket(name="index-bucket",
ram_quota_mb=ram_per_bucket_mb)
# Start sync gateway, should succeed now
sg_util.start_sync_gateways(cluster_config=cluster_conf,
url=sg_url,
config=sg_conf)
def test_rolling_ttl_remove_expirary(params_from_base_test_setup,
sg_conf_name):
"""
1. PUT /db/doc1 via SG with property "_exp":3
2. Once per second for 10 seconds, update /db/doc1 with a new revision (also with "_exp":3)
3. Update /db/doc1 with a revision with no expiry
3. Get /db/doc1. Assert response is 200
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_rolling_ttl_remove_expirary'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
bucket_name = "data-bucket"
cbs_ip = host_for_url(cbs_url)
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name),
password='******')
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
doc_exp_3_body = document.create_doc(doc_id="exp_3",
expiry=3,
channels=sg_user_channels)
doc_exp_10_body = document.create_doc(doc_id="exp_10",
expiry=10,
channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_3_body,
auth=sg_user_session)
doc_exp_10 = client.add_doc(url=sg_url,
db=sg_db,
doc=doc_exp_10_body,
auth=sg_user_session)
client.update_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3["id"],
number_updates=10,
expiry=3,
delay=1,
auth=sg_user_session)
client.update_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3["id"],
number_updates=1,
auth=sg_user_session)
# If expiry was not removed in the last update, this would expire doc_exp_3
time.sleep(5)
# doc_exp_3 should still be around due to removal of expiry
doc_exp_3 = client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_3["id"],
auth=sg_user_session)
assert doc_exp_3["_id"] == "exp_3"
# doc_exp_10 should be expired due to the updates (10s) + sleep (5s)
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_exp_10["id"],
auth=sg_user_session)
# In XATTR mode, the expiry results in a tombstone
# In Doc Meta mode, the expiry results in a purge
if xattrs_enabled:
assert he.value[0].startswith("403 Client Error: Forbidden for url:")
else:
assert he.value[0].startswith("404 Client Error: Not Found for url:")
verify_doc_deletion_on_server(doc_id=doc_exp_10["id"],
sdk_client=sdk_client,
sg_client=client,
sg_admin_url=sg_url_admin,
sg_db=sg_db,
xattrs_enabled=xattrs_enabled)
def test_string_expiry_as_iso_8601_date(params_from_base_test_setup, sg_conf_name):
"""
1. Calculate (server time + 3 seconds) as ISO-8601 date (e.g. 2016-01-01T00:00:00.000+00:00)
2. PUT /db/doc1 via SG with property "_exp":"[date]"
PUT /db/doc2 via SG with property "_exp":"2026-01-01T00:00:00.000+00:00"
3. Wait five seconds
4. Get /db/doc1. Assert response is 404
Get /db/doc2. Assert response is 20
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_string_expiry_as_ISO_8601_Date'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
time_util = Time()
iso_datetime = time_util.get_iso_datetime(delta=3)
doc_exp_3_body = document.create_doc(doc_id="exp_3", expiry=iso_datetime, channels=sg_user_channels)
doc_exp_years_body = document.create_doc(doc_id="exp_years", expiry="2026-01-01T00:00:00.000+00:00", channels=sg_user_channels)
doc_exp_3 = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_3_body, auth=sg_user_session)
doc_exp_years = client.add_doc(url=sg_url, db=sg_db, doc=doc_exp_years_body, auth=sg_user_session)
# Sleep should allow doc_exp_3 to expire
time.sleep(10)
# doc_exp_3 should be expired
with pytest.raises(HTTPError) as he:
client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_3["id"], auth=sg_user_session)
assert he.value[0].startswith("404 Client Error: Not Found for url:")
# doc_exp_years should be available still
doc_exp_years_result = client.get_doc(url=sg_url, db=sg_db, doc_id=doc_exp_years["id"], auth=sg_user_session)
assert doc_exp_years_result["_id"] == "exp_years"
def test_setting_expiry_in_bulk_docs(params_from_base_test_setup,
sg_conf_name):
"""
1. PUT /db/_bulk_docs with 10 documents. Set the "_exp":3 on 5 of these documents
2. Wait five seconds
3. POST /db/_bulk_get for the 10 documents. Validate that only the 5 non-expiring documents are returned
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_setting_expiry_in_bulk_docs'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
bucket_name = "data-bucket"
cbs_ip = host_for_url(cbs_url)
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name),
password='******')
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
doc_exp_3_bodies = document.create_docs(doc_id_prefix="exp_3",
number=5,
expiry=3,
channels=sg_user_channels)
doc_exp_10_bodies = document.create_docs(doc_id_prefix="exp_10",
number=5,
expiry=10,
channels=sg_user_channels)
bulk_bodies = doc_exp_3_bodies + doc_exp_10_bodies
bulk_docs = client.add_bulk_docs(url=sg_url,
db=sg_db,
docs=bulk_bodies,
auth=sg_user_session)
# Allow exp_3 docs to expire
time.sleep(5)
bulk_docs_ids = [doc["id"] for doc in bulk_docs]
expected_ids = ["exp_10_0", "exp_10_1", "exp_10_2", "exp_10_3", "exp_10_4"]
expected_missing_ids = [
"exp_3_0", "exp_3_1", "exp_3_2", "exp_3_3", "exp_3_4"
]
bulk_get_docs, errors = client.get_bulk_docs(url=sg_url,
db=sg_db,
doc_ids=bulk_docs_ids,
auth=sg_user_session,
validate=False)
assert len(bulk_get_docs) == len(expected_ids)
assert len(errors) == len(expected_missing_ids)
bulk_get_doc_ids = [doc["_id"] for doc in bulk_get_docs]
error_ids = [doc["id"] for doc in errors]
assert bulk_get_doc_ids == expected_ids
assert error_ids == expected_missing_ids
client.verify_doc_ids_found_in_response(response=bulk_get_docs,
expected_doc_ids=expected_ids)
client.verify_doc_ids_not_found_in_response(
response=errors, expected_missing_doc_ids=expected_missing_ids)
for expired_doc in error_ids:
verify_doc_deletion_on_server(doc_id=expired_doc,
sdk_client=sdk_client,
sg_client=client,
sg_admin_url=sg_url_admin,
sg_db=sg_db,
xattrs_enabled=xattrs_enabled)
def test_take_all_sgaccels_down(params_from_base_test_setup, sg_conf):
"""
Scenario that takes all sync_gateway accel nodes offline during doc load.
After bring the nodes back online during load, the reshard of the DCP feed is verified.
The changes feed is verified that all docs show up.
1. Start doc load (1000 doc)
2. Take all sg_accel nodes down in parallel
3. Verify node are down
4. Wait for doc adds to complete, store "doc_push_result_1"
5. Verify "doc_push_result_1" docs added
6. Start doc load (1000 docs)
7. Wait for 5. to complete, store "doc_push_result_2"
8. Verify "doc_push_result_2" docs added
9. Start another doc load (1000 docs)
10. Bring up nodes in parallel
11. poll on p-index reshard
12. Wait for 9. to complete, store "doc_push_result_3"
13. Verify "doc_push_result_3" docs added
14. Verify "doc_push_result_1" + "doc_push_result_2" + "doc_push_result_3" show up in _changes feed
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_dcp_reshard_single_sg_accel_goes_down_and_up'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
cluster_util = ClusterKeywords()
topology = cluster_util.get_cluster_topology(cluster_conf)
sg_url = topology["sync_gateways"][0]["public"]
sg_admin_url = topology["sync_gateways"][0]["admin"]
sg_db = "db"
num_docs = 1000
client = MobileRestClient()
doc_pusher_user_info = userinfo.UserInfo("doc_pusher", "pass", channels=["A"], roles=[])
doc_pusher_auth = client.create_user(
url=sg_admin_url,
db=sg_db,
name=doc_pusher_user_info.name,
password=doc_pusher_user_info.password,
channels=doc_pusher_user_info.channels
)
a_user_info = userinfo.UserInfo("a_user", "pass", channels=["A"], roles=[])
client.create_user(
url=sg_admin_url,
db=sg_db,
name=a_user_info.name,
password=a_user_info.password,
channels=a_user_info.channels
)
a_user_session = client.create_session(
url=sg_admin_url,
db=sg_db,
name=a_user_info.name,
password=a_user_info.password
)
# Shutdown all accel nodes in parallel
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as ex:
# Start adding docs
docs_1 = document.create_docs(None, num_docs, channels=doc_pusher_user_info.channels)
docs_1_task = ex.submit(client.add_bulk_docs, url=sg_url, db=sg_db, docs=docs_1, auth=doc_pusher_auth)
# Take down all access nodes
log_info("Shutting down sg_accels: [{}, {}, {}] ...".format(
cluster.sg_accels[0],
cluster.sg_accels[1],
cluster.sg_accels[2]
))
sg_accel_down_task_1 = ex.submit(cluster.sg_accels[0].stop)
sg_accel_down_task_2 = ex.submit(cluster.sg_accels[1].stop)
sg_accel_down_task_3 = ex.submit(cluster.sg_accels[2].stop)
assert sg_accel_down_task_1.result() == 0
assert sg_accel_down_task_2.result() == 0
assert sg_accel_down_task_3.result() == 0
# Block until bulk_docs is complete
doc_push_result_1 = docs_1_task.result()
assert len(doc_push_result_1) == num_docs
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=doc_push_result_1, auth=doc_pusher_auth)
# Load sync_gateway with another batch of docs while the sg_accel nodes are offline
docs_2_bodies = document.create_docs(None, num_docs, channels=doc_pusher_user_info.channels)
docs_push_result_2 = client.add_bulk_docs(url=sg_url, db=sg_db, docs=docs_2_bodies, auth=doc_pusher_auth)
assert len(docs_push_result_2) == num_docs
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=docs_push_result_2, auth=doc_pusher_auth)
# Start loading Sync Gateway with another set of docs while bringing the sg_accel nodes online
docs_3 = document.create_docs(None, num_docs, channels=doc_pusher_user_info.channels)
docs_3_task = ex.submit(client.add_bulk_docs, url=sg_url, db=sg_db, docs=docs_3, auth=doc_pusher_auth)
# Bring all the sg_accel nodes back up
# Take down all access nodes
log_info("Starting sg_accels: [{}, {}, {}] ...".format(
cluster.sg_accels[0],
cluster.sg_accels[1],
cluster.sg_accels[2]
))
sg_accel_up_task_1 = ex.submit(cluster.sg_accels[0].start, sg_conf)
sg_accel_up_task_2 = ex.submit(cluster.sg_accels[1].start, sg_conf)
sg_accel_up_task_3 = ex.submit(cluster.sg_accels[2].start, sg_conf)
assert sg_accel_up_task_1.result() == 0
assert sg_accel_up_task_2.result() == 0
assert sg_accel_up_task_3.result() == 0
# Wait for pindex to reshard correctly
assert cluster.validate_cbgt_pindex_distribution_retry(3)
# Block until second bulk_docs is complete
doc_push_result_3 = docs_3_task.result()
assert len(doc_push_result_3) == num_docs
client.verify_docs_present(url=sg_url, db=sg_db, expected_docs=doc_push_result_3, auth=doc_pusher_auth)
# Combine the 3 push results and make sure the changes propagate to a_user
# a_user has access to the doc's channel.
log_info("Verifying all the changes show up for 'a_user' ...")
all_docs = doc_push_result_1 + docs_push_result_2 + doc_push_result_3
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=all_docs, auth=a_user_session, polling_interval=2)
def test_server_goes_down_rebuild_channels(params_from_base_test_setup):
"""
1. Start with a two node couchbase server cluster
2. Starting adding docs
3. Kill one of the server nodes and signal completion
4. Stop adding docs
5. Verify that that the expected docs are present and in the changes feed.
6. Start server again and add to cluster
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
cluster_helper = ClusterKeywords()
sg_conf_name = "sync_gateway_default_functional_tests"
sg_conf_path = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf_path)
topology = cluster_helper.get_cluster_topology(cluster_config)
admin_sg = topology["sync_gateways"][0]["admin"]
sg_url = topology["sync_gateways"][0]["public"]
coucbase_servers = topology["couchbase_servers"]
cbs_one_url = coucbase_servers[0]
cbs_two_url = coucbase_servers[1]
log_info("Running: 'test_server_goes_down_sanity'")
log_info("cluster_config: {}".format(cluster_config))
log_info("admin_sg: {}".format(admin_sg))
log_info("sg_url: {}".format(sg_url))
log_info("cbs_one_url: {}".format(cbs_one_url))
log_info("cbs_two_url: {}".format(cbs_two_url))
sg_db = "db"
num_docs = 100
admin_user_info = userinfo.UserInfo(
name="admin",
password="******",
channels=["ABC"],
roles=[]
)
seth_user_info = userinfo.UserInfo(
name="seth",
password="******",
channels=["ABC"],
roles=[]
)
client = MobileRestClient()
main_server = CouchbaseServer(cbs_one_url)
flakey_server = CouchbaseServer(cbs_two_url)
admin_auth = client.create_user(
admin_sg,
sg_db,
admin_user_info.name,
admin_user_info.password,
channels=admin_user_info.channels
)
client.create_user(
admin_sg,
sg_db,
seth_user_info.name,
seth_user_info.password,
channels=seth_user_info.channels
)
seth_session = client.create_session(admin_sg, sg_db, seth_user_info.name)
# allow any user docs to make it to changes
initial_changes = client.get_changes(url=sg_url, db=sg_db, since=0, auth=seth_session)
# push docs from admin
docs = client.add_docs(
url=sg_url,
db=sg_db,
number=num_docs,
id_prefix=None,
channels=admin_user_info.channels,
auth=admin_auth
)
assert len(docs) == num_docs
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=docs, auth=seth_session)
changes_before_failover = client.get_changes(url=sg_url, db=sg_db, since=initial_changes["last_seq"], auth=seth_session)
assert len(changes_before_failover["results"]) == num_docs
# Stop server via 'service stop'
flakey_server.stop()
start = time.time()
while True:
# Fail tests if all docs do not succeed before timeout
if (time.time() - start) > 60:
# Bring server back up before failing the test
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
raise keywords.exceptions.TimeoutError("Failed to rebuild changes")
try:
# Poll until failover happens (~30 second)
client.verify_docs_in_changes(url=sg_url, db=sg_db, expected_docs=docs, auth=seth_session)
# changes requests succeeded, exit loop
break
except requests.exceptions.HTTPError:
# Changes will fail until failover of the down server happens. Wait and try again.
log_info("/db/_changes failed due to server down. Retrying ...")
time.sleep(1)
# Verify no new changes
changes = client.get_changes(
url=sg_url,
db=sg_db,
since=changes_before_failover["last_seq"],
auth=seth_session,
feed="normal"
)
assert len(changes["results"]) == 0
# Check that all changes are intact from initial changes request
changes = client.get_changes(url=sg_url, db=sg_db, since=initial_changes["last_seq"], auth=seth_session)
assert len(changes["results"]) == num_docs
coucbase_servers = topology["couchbase_servers"]
# Test succeeded without timeout, bring server back into topology
flakey_server.start()
main_server.recover(flakey_server)
main_server.rebalance_in(coucbase_servers, flakey_server)
def params_from_base_suite_setup(request):
log_info("Setting up 'params_from_base_suite_setup' ...")
# pytest command line parameters
server_version = request.config.getoption("--server-version")
sync_gateway_version = request.config.getoption("--sync-gateway-version")
mode = request.config.getoption("--mode")
skip_provisioning = request.config.getoption("--skip-provisioning")
race_enabled = request.config.getoption("--race")
cbs_ssl = request.config.getoption("--server-ssl")
xattrs_enabled = request.config.getoption("--xattrs")
sg_lb = request.config.getoption("--sg-lb")
sg_ce = request.config.getoption("--sg-ce")
use_sequoia = request.config.getoption("--sequoia")
no_conflicts_enabled = request.config.getoption("--no-conflicts")
if xattrs_enabled and version_is_binary(sync_gateway_version):
check_xattr_support(server_version, sync_gateway_version)
log_info("server_version: {}".format(server_version))
log_info("sync_gateway_version: {}".format(sync_gateway_version))
log_info("mode: {}".format(mode))
log_info("skip_provisioning: {}".format(skip_provisioning))
log_info("race_enabled: {}".format(race_enabled))
log_info("cbs_ssl: {}".format(cbs_ssl))
log_info("xattrs_enabled: {}".format(xattrs_enabled))
log_info("sg_lb: {}".format(sg_lb))
log_info("sg_ce: {}".format(sg_ce))
log_info("no conflicts enabled {}".format(no_conflicts_enabled))
# sg-ce is invalid for di mode
if mode == "di" and sg_ce:
raise FeatureSupportedError(
"SGAccel is only available as an enterprise edition")
if no_conflicts_enabled and sync_gateway_version < "2.0":
raise FeatureSupportedError(
'No conflicts feature not available for sync-gateway version below 2.0, so skipping the test'
)
# Make sure mode for sync_gateway is supported ('cc' or 'di')
validate_sync_gateway_mode(mode)
# use load_balancer_cc cluster config if mode is "cc" or load_balancer_di cluster config if mode is "di"
cluster_config = "{}/load_balancer_{}".format(
keywords.constants.CLUSTER_CONFIGS_DIR, mode)
sg_config = sync_gateway_config_path_for_mode(
"sync_gateway_default_functional_tests", mode)
# Add load balancer prop and check if load balancer IP is available
if sg_lb:
persist_cluster_config_environment_prop(cluster_config,
'sg_lb_enabled', True)
log_info("Running tests with load balancer enabled: {}".format(
get_load_balancer_ip(cluster_config)))
else:
log_info("Running tests with load balancer disabled")
persist_cluster_config_environment_prop(cluster_config,
'sg_lb_enabled', False)
if cbs_ssl:
log_info("Running tests with cbs <-> sg ssl enabled")
# Enable ssl in cluster configs
persist_cluster_config_environment_prop(cluster_config,
'cbs_ssl_enabled', True)
else:
log_info("Running tests with cbs <-> sg ssl disabled")
# Disable ssl in cluster configs
persist_cluster_config_environment_prop(cluster_config,
'cbs_ssl_enabled', False)
if xattrs_enabled:
log_info("Running test with xattrs for sync meta storage")
persist_cluster_config_environment_prop(cluster_config,
'xattrs_enabled', True)
else:
log_info("Using document storage for sync meta data")
persist_cluster_config_environment_prop(cluster_config,
'xattrs_enabled', False)
try:
server_version
except NameError:
log_info("Server version is not provided")
persist_cluster_config_environment_prop(cluster_config,
'server_version', "")
else:
log_info("Running test with server version {}".format(server_version))
persist_cluster_config_environment_prop(cluster_config,
'server_version',
server_version)
try:
sync_gateway_version
except NameError:
log_info("Sync gateway version is not provided")
persist_cluster_config_environment_prop(cluster_config,
'sync_gateway_version', "")
else:
log_info("Running test with sync_gateway version {}".format(
sync_gateway_version))
persist_cluster_config_environment_prop(cluster_config,
'sync_gateway_version',
sync_gateway_version)
if no_conflicts_enabled:
log_info("Running with no conflicts")
persist_cluster_config_environment_prop(cluster_config,
'no_conflicts_enabled', True)
else:
log_info("Running with allow conflicts")
persist_cluster_config_environment_prop(cluster_config,
'no_conflicts_enabled', False)
if sync_gateway_version < "2.0.0" and no_conflicts_enabled:
pytest.skip(
"Test cannot run with no-conflicts with sg version < 2.0.0")
# Skip provisioning if user specifies '--skip-provisoning' or '--sequoia'
should_provision = True
if skip_provisioning or use_sequoia:
should_provision = False
cluster_utils = ClusterKeywords()
if should_provision:
try:
cluster_utils.provision_cluster(
cluster_config=cluster_config,
server_version=server_version,
sync_gateway_version=sync_gateway_version,
sync_gateway_config=sg_config,
race_enabled=race_enabled,
sg_ce=sg_ce)
except ProvisioningError:
logging_helper = Logging()
logging_helper.fetch_and_analyze_logs(
cluster_config=cluster_config, test_name=request.node.name)
raise
# Hit this intalled running services to verify the correct versions are installed
cluster_utils.verify_cluster_versions(
cluster_config,
expected_server_version=server_version,
expected_sync_gateway_version=sync_gateway_version)
yield {"cluster_config": cluster_config, "mode": mode}
log_info("Tearing down 'params_from_base_suite_setup' ...")
# Stop all sync_gateway and sg_accels as test finished
c = cluster.Cluster(cluster_config)
c.stop_sg_and_accel()
def test_attachment_revpos_when_ancestor_unavailable_active_revision_doesnt_share_ancestor(params_from_base_test_setup, sg_conf_name):
"""
Creates a document with an attachment, then updates that document so that
the body of the revision that originally pushed the document is no
longer available. Add a new revision that's not a child of the
active revision, and validate that it's uploaded successfully.
Example:
1. Document is created with no attachment at rev-1
2. Server adds revision with attachment at rev-2 {"hello.txt", revpos=2}
2. Document is updated multiple times on the server, goes to rev-4
3. Client attempts to add a new (conflicting) revision 3a, with ancestors rev-2a (with it's own attachment), rev-1.
4. When client attempts to push rev-3a with attachment stub {"hello.txt", revpos=2}. Should throw an error, since the revpos
of the attachment is later than the common ancestor (rev-1)
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
cluster_helper.reset_cluster(cluster_config, sg_conf)
topology = cluster_helper.get_cluster_topology(cluster_config)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
sg_db = "db"
bucket = "data-bucket"
log_info("Running 'test_attachment_revpos_when_ancestor_unavailable_active_revision_doesnt_share_ancestor'")
log_info("Using cbs_url: {}".format(cbs_url))
log_info("Using sg_url: {}".format(sg_url))
log_info("Using sg_url_admin: {}".format(sg_url_admin))
log_info("Using sg_db: {}".format(sg_db))
log_info("Using bucket: {}".format(bucket))
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
doc = document.create_doc(doc_id="doc_1", content={"sample_key": "sample_val"}, channels=sg_user_channels)
doc_gen_1 = client.add_doc(url=sg_url, db=sg_db, doc=doc, auth=sg_user_session)
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_gen_1["id"], attachment_name="sample_text.txt", auth=sg_user_session)
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_gen_1["id"], auth=sg_user_session)
client.update_doc(url=sg_url, db=sg_db, doc_id=doc_gen_1["id"], auth=sg_user_session)
parent_rev_list = ["2-foo2", doc_gen_1["rev"]]
# Sync Gateway should error since it has no references attachment in its ancestors
with pytest.raises(HTTPError) as he:
client.add_conflict(
url=sg_url,
db=sg_db,
doc_id=doc_gen_1["id"],
parent_revisions=parent_rev_list,
new_revision="3-foo3",
auth=sg_user_session
)
assert he.value[0].startswith("400 Client Error: Bad Request for url: ")
def test_listener_two_sync_gateways(setup_client_syncgateway_test):
"""
Port of https://github.com/couchbaselabs/sync-gateway-tests/blob/master/tests/cbl-replication-mismatch-2-gateways.js
Scenario:
1. Start 2 sync_gateways
2. Create sg_db_one db on sync_gateway one
3. Create sg_db_two db on sync_gateway two
4. Create ls_db_one and ls_db_two on Liteserv
5. Setup continuous push / pull replication from ls_db_one <-> sg_db_one
6. Setup continuous push / pull replication from ls_db_two <-> sg_db_two
7. Setup continuous push / pull replication from sg_db_one <-> ls_db_two
8. Setup continuous push / pull replication from sg_db_two <-> ls_db_one
9. Add num_docs / 2 to each liteserv database
10. Verify each database has num_docs docs
11. Verify all_docs in all dbs
12. Verify changes feed for sg_db_one and sg_db_two
13. Verify chnages feed for ls_db_one and ls_db_two
"""
num_docs = 500
ls_url = setup_client_syncgateway_test["ls_url"]
cluster_config = setup_client_syncgateway_test["cluster_config"]
sg_mode = setup_client_syncgateway_test["sg_mode"]
cluster_util = ClusterKeywords()
topology = cluster_util.get_cluster_topology(cluster_config)
sg_one_admin_url = topology["sync_gateways"][0]["admin"]
sg_two_admin_url = topology["sync_gateways"][1]["admin"]
cb_server_url = topology["couchbase_servers"][0]
log_info("Sync Gateway 1 admin url: {}".format(sg_one_admin_url))
log_info("Sync Gateway 2 admin url: {}".format(sg_two_admin_url))
log_info("Couchbase Server url: {}".format(cb_server_url))
c = cluster.Cluster(cluster_config)
sg_config_path = sync_gateway_config_path_for_mode(
"listener_tests/multiple_sync_gateways", sg_mode)
c.reset(sg_config_path=sg_config_path)
ls_db_one = "ls_db1"
ls_db_two = "ls_db2"
sg_db_one = "sg_db1"
sg_db_two = "sg_db2"
log_info("ls_url: {}".format(ls_url))
log_info("sg_one_admin_url: {}".format(sg_one_admin_url))
log_info("sg_two_admin_url: {}".format(sg_two_admin_url))
log_info("num_docs: {}".format(num_docs))
log_info("Running 'test_listener_two_sync_gateways' ...")
client = MobileRestClient()
# Delete sg_db2 on sync_gateway 1
client.delete_database(url=sg_one_admin_url, name=sg_db_two)
# Delete sg_db1 on sync_gateway 2
client.delete_database(url=sg_two_admin_url, name=sg_db_one)
# Create dbs on LiteServ
client.create_database(ls_url, ls_db_one)
client.create_database(ls_url, ls_db_two)
# Start continuous push pull replication ls_db_one <-> sg_db_one
client.start_replication(url=ls_url,
continuous=True,
from_db=ls_db_one,
to_url=sg_one_admin_url,
to_db=sg_db_one)
client.start_replication(url=ls_url,
continuous=True,
from_url=sg_one_admin_url,
from_db=sg_db_one,
to_db=ls_db_one)
# Start continuous push pull replication ls_db_two <-> sg_db_two
client.start_replication(url=ls_url,
continuous=True,
from_db=ls_db_two,
to_url=sg_two_admin_url,
to_db=sg_db_two)
client.start_replication(url=ls_url,
continuous=True,
from_url=sg_two_admin_url,
from_db=sg_db_two,
to_db=ls_db_two)
# Start continuous push pull replication sg_db_one <-> ls_db_two
client.start_replication(url=ls_url,
continuous=True,
from_url=sg_one_admin_url,
from_db=sg_db_one,
to_db=ls_db_two)
client.start_replication(url=ls_url,
continuous=True,
from_db=ls_db_two,
to_url=sg_one_admin_url,
to_db=sg_db_one)
# Start continuous push pull replication sg_db_two <-> ls_db_one
client.start_replication(url=ls_url,
continuous=True,
from_url=sg_two_admin_url,
from_db=sg_db_two,
to_db=ls_db_one)
client.start_replication(url=ls_url,
continuous=True,
from_db=ls_db_one,
to_url=sg_two_admin_url,
to_db=sg_db_two)
ls_db_one_docs = client.add_docs(url=ls_url,
db=ls_db_one,
number=num_docs / 2,
id_prefix="ls_db_one_doc")
assert len(ls_db_one_docs) == num_docs / 2
ls_db_two_docs = client.add_docs(url=ls_url,
db=ls_db_two,
number=num_docs / 2,
id_prefix="ls_db_two_doc")
assert len(ls_db_two_docs) == num_docs / 2
all_docs = client.merge(ls_db_one_docs, ls_db_two_docs)
assert len(all_docs) == 500
# Verify docs replicate to each db
client.verify_docs_present(url=ls_url,
db=ls_db_one,
expected_docs=all_docs)
client.verify_docs_present(url=ls_url,
db=ls_db_two,
expected_docs=all_docs)
client.verify_docs_present(url=sg_one_admin_url,
db=sg_db_one,
expected_docs=all_docs)
client.verify_docs_present(url=sg_two_admin_url,
db=sg_db_two,
expected_docs=all_docs)
# Verify changes feeds for each db
client.verify_docs_in_changes(url=ls_url,
db=ls_db_one,
expected_docs=all_docs)
client.verify_docs_in_changes(url=ls_url,
db=ls_db_two,
expected_docs=all_docs)
client.verify_docs_in_changes(url=sg_one_admin_url,
db=sg_db_one,
expected_docs=all_docs)
client.verify_docs_in_changes(url=sg_two_admin_url,
db=sg_db_two,
expected_docs=all_docs)
def provision_cluster(cluster_config, couchbase_server_config, sync_gateway_config, sg_ce=False, cbs_platform="centos7", sg_platform="centos", sa_platform="centos"):
log_info("\n>>> Cluster info:\n")
server_version = "{}-{}".format(couchbase_server_config.version, couchbase_server_config.build)
sg_version = "{}-{}".format(sync_gateway_config._version_number, sync_gateway_config._build_number)
try:
server_version
except NameError:
log_info("Server version is not provided")
persist_cluster_config_environment_prop(cluster_config, 'server_version', "")
else:
log_info("Running test with server version {}".format(server_version))
persist_cluster_config_environment_prop(cluster_config, 'server_version', server_version)
try:
sg_version
except NameError:
log_info("Sync gateway version is not provided")
persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_version', "")
else:
log_info("Running test with sync_gateway version {}".format(sg_version))
persist_cluster_config_environment_prop(cluster_config, 'sync_gateway_version', sg_version)
with open(cluster_config, "r") as ansible_hosts:
log_info(ansible_hosts.read())
log_info(couchbase_server_config)
log_info(sync_gateway_config)
if not sync_gateway_config.is_valid():
log_info("Invalid sync_gateway provisioning configuration. Exiting ...")
sys.exit(1)
cluster = Cluster(config=cluster_config)
config_path_full = os.path.abspath(sync_gateway_config.config_path)
config = Config(config_path_full)
is_valid, reason = validate_cluster(
cluster.sync_gateways,
cluster.sg_accels,
config,
)
if not is_valid:
raise ProvisioningError(reason)
log_info(">>> Provisioning cluster...")
# Get server base url and package name
cluster_keywords = ClusterKeywords()
cluster_topology = cluster_keywords.get_cluster_topology(cluster_config)
server_url = cluster_topology["couchbase_servers"][0]
cb_server = CouchbaseServer(server_url)
server_baseurl, server_package_name = couchbase_server_config.get_baseurl_package(cb_server, cbs_platform)
log_info(">>> Server package: {0}/{1}".format(server_baseurl, server_package_name))
log_info(">>> Using sync_gateway config: {}".format(sync_gateway_config.config_path))
# Reset previous installs
clean_cluster(cluster_config)
# Install server package
log_info("Installing Couchbase Server")
install_couchbase_server.install_couchbase_server(
cluster_config=cluster_config,
couchbase_server_config=couchbase_server_config,
cbs_platform=cbs_platform
)
# Install sync_gateway
log_info("Installing Sync Gateway")
install_sync_gateway.install_sync_gateway(
cluster_config=cluster_config,
sync_gateway_config=sync_gateway_config,
sg_platform=sg_platform,
sa_platform=sa_platform,
sg_ce=sg_ce
)
# Install nginx
install_nginx(cluster_config)
log_info(">>> Done provisioning cluster...")
