def params_from_base_test_setup(request, params_from_base_suite_setup):
# Code before the yeild will execute before each test starts
cluster_config = params_from_base_suite_setup["cluster_config"]
mode = params_from_base_suite_setup["mode"]
test_name = request.node.name
log_info("Setting up test '{}'".format(test_name))
# This dictionary is passed to each test
yield {"cluster_config": cluster_config, "mode": mode}
# Code after the yeild will execute when each test finishes
log_info("Tearing down test '{}'".format(test_name))
# Capture testkit socket usage
network_utils = NetworkUtils()
network_utils.list_connections()
# Verify all sync_gateways and sg_accels are reachable
c = Cluster(cluster_config)
errors = c.verify_alive(mode)
assert len(errors) == 0
# if the test failed pull logs
if request.node.rep_call.failed:
logging_helper = Logging()
logging_helper.fetch_and_analyze_logs(cluster_config=cluster_config, test_name=test_name)
def params_from_base_test_setup(request, params_from_base_suite_setup):
# Code before the yeild will execute before each test starts
# pytest command line parameters
collect_logs = request.config.getoption("--collect-logs")
cluster_config = params_from_base_suite_setup["cluster_config"]
mode = params_from_base_suite_setup["mode"]
test_name = request.node.name
log_info("Setting up test '{}'".format(test_name))
# This dictionary is passed to each test
yield {"cluster_config": cluster_config, "mode": mode}
# Code after the yeild will execute when each test finishes
log_info("Tearing down test '{}'".format(test_name))
# Capture testkit socket usage
network_utils = NetworkUtils()
network_utils.list_connections()
# Verify all sync_gateways and sg_accels are reachable
c = Cluster(cluster_config)
errors = c.verify_alive(mode)
# if the test failed pull logs
if collect_logs or request.node.rep_call.failed or len(errors) != 0:
logging_helper = Logging()
logging_helper.fetch_and_analyze_logs(cluster_config=cluster_config,
test_name=test_name)
assert len(errors) == 0
def test_bucket_shadow_low_revs_limit_repeated_deletes(params_from_base_test_setup):
"""
Validate that Sync Gateway doesn't panic (and instead creates a conflict branch
and prints a warning) after doing the following steps:
- Set revs_limit to 5
- Create a doc via SG
- Issue a delete operation for that doc via SG
- Repeat step 3 5x. (each additional delete will create a new revision in SG, but the delete on the source bucket will fail with the 'not found' error, which also means that upstream_rev won't get incremented
- Recreate the doc in the source bucket
See https://github.com/couchbaselabs/sync-gateway-testcluster/issues/291#issuecomment-191521993
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
default_config_path_shadower_low_revs = sync_gateway_config_path_for_mode("sync_gateway_bucketshadow_low_revs", mode)
default_config_path_non_shadower_low_revs = sync_gateway_config_path_for_mode("sync_gateway_default_low_revs", mode)
log_info("Running 'test_bucket_shadow_low_revs_limit_repeated_deletes'")
log_info("Using cluster_config: {}".format(cluster_config))
cluster = Cluster(config=cluster_config)
sc = init_shadow_cluster(cluster,
default_config_path_shadower_low_revs,
default_config_path_non_shadower_low_revs)
# Write doc into shadower SG
doc_id = sc.alice_shadower.add_doc()
# Wait until it gets to source bucket
get_doc_from_source_bucket_retry(doc_id, sc.source_bucket)
# Wait until upstream-rev in _sync metadata is non empty
# Otherwise, this will not reproduce a panic
while True:
doc = sc.data_bucket.get(doc_id)
if doc.success:
if "upstream_rev" in doc.value["_sync"]:
break
time.sleep(1)
# Repeatedly issue a delete operation for that doc via SG
# Keep adding tombstone revs to the one and only branch
rev_id_to_delete = None
for i in xrange(100):
resp = sc.alice_shadower.delete_doc(doc_id, rev_id_to_delete)
rev_id_to_delete = resp["rev"]
# Recreate doc with that ID in the source bucket
sc.source_bucket.upsert(doc_id, json.loads('{"foo":"bar"}'))
# Check if SG's are up
errors = cluster.verify_alive(sc.mode)
assert len(errors) == 0
# Restart Shadow SG
sc.shadower_sg.stop()
sc.shadower_sg.start(default_config_path_shadower_low_revs)
def test_dcp_reshard_sync_gateway_goes_down(params_from_base_test_setup, sg_conf):
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_dcp_reshard_sync_gateway_goes_down'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
mode = cluster.reset(sg_config_path=sg_conf)
admin = Admin(cluster.sync_gateways[0])
traun = admin.register_user(target=cluster.sync_gateways[0], db="db", name="traun", password="******", channels=["ABC", "NBC", "CBS"])
seth = admin.register_user(target=cluster.sync_gateways[0], db="db", name="seth", password="******", channels=["FOX"])
log_info(">> Users added")
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
futures = dict()
log_info(">>> Adding Seth docs") # FOX
futures[executor.submit(seth.add_docs, 8000)] = "seth"
log_info(">>> Adding Traun docs") # ABC, NBC, CBS
futures[executor.submit(traun.add_docs, 2000, bulk=True)] = "traun"
# stop sg_accel
shutdown_status = cluster.sg_accels[0].stop()
assert shutdown_status == 0
for future in concurrent.futures.as_completed(futures):
tag = futures[future]
log_info("{} Completed:".format(tag))
# TODO better way to do this
time.sleep(120)
verify_changes(traun, expected_num_docs=2000, expected_num_revisions=0, expected_docs=traun.cache)
verify_changes(seth, expected_num_docs=8000, expected_num_revisions=0, expected_docs=seth.cache)
# Verify that the sg1 is down but the other sync_gateways are running
errors = cluster.verify_alive(mode)
assert len(errors) == 1 and errors[0][0].hostname == "ac1"
# Restart the failing node so that cluster verification does not blow up in test teardown
start_status = cluster.sg_accels[0].start(sg_conf)
assert start_status == 0
def test_dcp_reshard_sync_gateway_goes_down(params_from_base_test_setup,
sg_conf):
cluster_conf = params_from_base_test_setup["cluster_config"]
log_info("Running 'test_dcp_reshard_sync_gateway_goes_down'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
mode = cluster.reset(sg_config_path=sg_conf)
admin = Admin(cluster.sync_gateways[0])
traun = admin.register_user(target=cluster.sync_gateways[0],
db="db",
name="traun",
password="******",
channels=["ABC", "NBC", "CBS"])
seth = admin.register_user(target=cluster.sync_gateways[0],
db="db",
name="seth",
password="******",
channels=["FOX"])
log_info(">> Users added")
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
futures = dict()
log_info(">>> Adding Seth docs") # FOX
futures[executor.submit(seth.add_docs, 8000)] = "seth"
log_info(">>> Adding Traun docs") # ABC, NBC, CBS
futures[executor.submit(traun.add_docs, 2000, bulk=True)] = "traun"
# stop sg_accel
shutdown_status = cluster.sg_accels[0].stop()
assert shutdown_status == 0
for future in concurrent.futures.as_completed(futures):
tag = futures[future]
log_info("{} Completed:".format(tag))
# TODO better way to do this
time.sleep(120)
verify_changes(traun,
expected_num_docs=2000,
expected_num_revisions=0,
expected_docs=traun.cache)
verify_changes(seth,
expected_num_docs=8000,
expected_num_revisions=0,
expected_docs=seth.cache)
# Verify that the sg1 is down but the other sync_gateways are running
errors = cluster.verify_alive(mode)
assert len(errors) == 1 and errors[0][0].hostname == "ac1"
# Restart the failing node so that cluster verification does not blow up in test teardown
start_status = cluster.sg_accels[0].start(sg_conf)
assert start_status == 0
def test_bucket_shadow_multiple_sync_gateways(params_from_base_test_setup):
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
log_info("Running 'test_bucket_shadow_multiple_sync_gateways'")
log_info("Using cluster_config: {}".format(cluster_config))
default_config_path_shadower = sync_gateway_config_path_for_mode("sync_gateway_bucketshadow", mode)
default_config_path_non_shadower = sync_gateway_config_path_for_mode("sync_gateway_default", mode)
cluster = Cluster(config=cluster_config)
sc = init_shadow_cluster(
cluster,
default_config_path_shadower,
default_config_path_non_shadower,
)
# Write several docs into shadower SG
doc_id_alice = sc.alice_shadower.add_doc()
doc_id_will_delete = sc.alice_shadower.add_doc()
# Write several docs into non-shadower SG
doc_id_bob = sc.bob_non_shadower.add_doc()
# Ditto as above, but bump revs rather than writing brand new docs
sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10)
sc.alice_shadower.update_doc(doc_id_alice, num_revision=10)
# Get the doc from the source bucket, possibly retrying if needed
# Otherwise an exception will be thrown and the test will fail
get_doc_from_source_bucket_retry(doc_id_bob, sc.source_bucket)
get_doc_from_source_bucket_retry(doc_id_alice, sc.source_bucket)
get_doc_from_source_bucket_retry(doc_id_will_delete, sc.source_bucket)
# Stop the SG shadower
sc.shadower_sg.stop()
# Write several docs + bump revs into non-shadower SG
sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10)
sc.bob_non_shadower.add_doc()
# Delete one of the docs
sc.bob_non_shadower.delete_doc(doc_id_will_delete)
# Bring SG shadower back up
sc.shadower_sg.start(default_config_path_shadower)
time.sleep(5) # Give tap feed a chance to initialize
# Verify SG shadower comes up without panicking, given writes from non-shadower during downtime.
errors = cluster.verify_alive(sc.mode)
assert len(errors) == 0
# If SG shadower does come up without panicking, bump a rev on a document that was modified during the downtime of the SG shadower
sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10)
# Make sure the doc that was added while shadower was down makes it to source bucket
# FAILING: Currently known to be failing, so temporarily disabled
# get_doc_from_source_bucket_retry(doc_id_shadower_down, sc.source_bucket)
# Manual check
# Grep logs for:
# WARNING: Error pushing rev of "f6eb40bf-d02a-4b4d-a3ab-779cce2fe9d5" to external bucket: MCResponse status=KEY_ENOENT, opcode=DELETE, opaque=0, msg: Not found -- db.(*Shadower).PushRevision() at shadower.go:164
# Verify SG shadower comes up without panicking, given writes from non-shadower during downtime.
time.sleep(5) # Give tap feed a chance to initialize
def test_bucket_shadow_low_revs_limit(params_from_base_test_setup):
"""
Set revs limit to 40
Add doc and makes sure it syncs to source bucket
Take shadower offline
Update one doc more than 50 times
Bring shadower online
Look for panics
Add more revisions to SG -- expected issue
Look for panics
(TODO: Update doc in shadow bucket and look for panics?)
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
log_info("Running 'test_bucket_shadow_low_revs_limit'")
log_info("Using cluster_config: {}".format(cluster_config))
default_config_path_shadower_low_revs = sync_gateway_config_path_for_mode("sync_gateway_bucketshadow_low_revs", mode)
default_config_path_non_shadower_low_revs = sync_gateway_config_path_for_mode("sync_gateway_default_low_revs", mode)
cluster = Cluster(config=cluster_config)
sc = init_shadow_cluster(cluster, default_config_path_shadower_low_revs, default_config_path_non_shadower_low_revs)
# Write doc into shadower SG
doc_id = sc.alice_shadower.add_doc()
# Update the doc just so we have a rev_id
sc.alice_shadower.update_doc(doc_id, content=fake_doc_content, num_revision=1)
# Make sure it makes it to source bucket
get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content, sc.source_bucket)
# Stop the SG shadower
sc.shadower_sg.stop()
# Update doc more than 50 times in non-shadower SG (since shadower is down)
sc.bob_non_shadower.update_doc(doc_id, num_revision=100)
sc.bob_non_shadower.update_doc(doc_id, content=fake_doc_content, num_revision=1)
# Bring SG shadower back up
sc.shadower_sg.start(default_config_path_shadower_low_revs)
# Look for panics
time.sleep(5) # Give tap feed a chance to initialize
errors = cluster.verify_alive(sc.mode)
assert len(errors) == 0
# Verify that the latest revision sync'd to source bucket
get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content, sc.source_bucket)
# Add more revisions
sc.bob_non_shadower.update_doc(doc_id, num_revision=50)
sc.bob_non_shadower.update_doc(doc_id, content=fake_doc_content, num_revision=1)
# Verify that the latest revision sync'd to source bucket
get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content, sc.source_bucket)
# Look for panics
time.sleep(5) # Wait until the shadower can process
def test_bucket_shadow_low_revs_limit_repeated_deletes(
params_from_base_test_setup):
"""
Validate that Sync Gateway doesn't panic (and instead creates a conflict branch
and prints a warning) after doing the following steps:
- Set revs_limit to 5
- Create a doc via SG
- Issue a delete operation for that doc via SG
- Repeat step 3 5x. (each additional delete will create a new revision in SG, but the delete on the source bucket will fail with the 'not found' error, which also means that upstream_rev won't get incremented
- Recreate the doc in the source bucket
See https://github.com/couchbaselabs/sync-gateway-testcluster/issues/291#issuecomment-191521993
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
if mode == "di" or xattrs_enabled:
pytest.skip("https://github.com/couchbase/sync_gateway/issues/2193")
default_config_path_shadower_low_revs = sync_gateway_config_path_for_mode(
"sync_gateway_bucketshadow_low_revs", mode)
default_config_path_non_shadower_low_revs = sync_gateway_config_path_for_mode(
"sync_gateway_default_low_revs", mode)
log_info("Running 'test_bucket_shadow_low_revs_limit_repeated_deletes'")
log_info("Using cluster_config: {}".format(cluster_config))
cluster = Cluster(config=cluster_config)
sc = init_shadow_cluster(cluster, default_config_path_shadower_low_revs,
default_config_path_non_shadower_low_revs)
# Write doc into shadower SG
doc_id = sc.alice_shadower.add_doc()
# Wait until it gets to source bucket
get_doc_from_source_bucket_retry(doc_id, sc.source_bucket)
# Wait until upstream-rev in _sync metadata is non empty
# Otherwise, this will not reproduce a panic
while True:
doc = sc.data_bucket.get(doc_id)
if doc.success:
if "upstream_rev" in doc.value["_sync"]:
break
time.sleep(1)
# Repeatedly issue a delete operation for that doc via SG
# Keep adding tombstone revs to the one and only branch
rev_id_to_delete = None
for i in xrange(100):
resp = sc.alice_shadower.delete_doc(doc_id, rev_id_to_delete)
rev_id_to_delete = resp["rev"]
# Recreate doc with that ID in the source bucket
sc.source_bucket.upsert(doc_id, json.loads('{"foo":"bar"}'))
# Check if SG's are up
errors = cluster.verify_alive(sc.mode)
assert len(errors) == 0
# Restart Shadow SG
sc.shadower_sg.stop()
sc.shadower_sg.start(default_config_path_shadower_low_revs)
def test_bucket_shadow_multiple_sync_gateways(params_from_base_test_setup):
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
if mode == "di" or xattrs_enabled:
pytest.skip("https://github.com/couchbase/sync_gateway/issues/2193")
log_info("Running 'test_bucket_shadow_multiple_sync_gateways'")
log_info("Using cluster_config: {}".format(cluster_config))
default_config_path_shadower = sync_gateway_config_path_for_mode(
"sync_gateway_bucketshadow", mode)
default_config_path_non_shadower = sync_gateway_config_path_for_mode(
"sync_gateway_default", mode)
cluster = Cluster(config=cluster_config)
sc = init_shadow_cluster(
cluster,
default_config_path_shadower,
default_config_path_non_shadower,
)
# Write several docs into shadower SG
doc_id_alice = sc.alice_shadower.add_doc()
doc_id_will_delete = sc.alice_shadower.add_doc()
# Write several docs into non-shadower SG
doc_id_bob = sc.bob_non_shadower.add_doc()
# Ditto as above, but bump revs rather than writing brand new docs
sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10)
sc.alice_shadower.update_doc(doc_id_alice, num_revision=10)
# Get the doc from the source bucket, possibly retrying if needed
# Otherwise an exception will be thrown and the test will fail
get_doc_from_source_bucket_retry(doc_id_bob, sc.source_bucket)
get_doc_from_source_bucket_retry(doc_id_alice, sc.source_bucket)
get_doc_from_source_bucket_retry(doc_id_will_delete, sc.source_bucket)
# Stop the SG shadower
sc.shadower_sg.stop()
# Write several docs + bump revs into non-shadower SG
sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10)
sc.bob_non_shadower.add_doc()
# Delete one of the docs
sc.bob_non_shadower.delete_doc(doc_id_will_delete)
# Bring SG shadower back up
sc.shadower_sg.start(default_config_path_shadower)
time.sleep(5) # Give tap feed a chance to initialize
# Verify SG shadower comes up without panicking, given writes from non-shadower during downtime.
errors = cluster.verify_alive(sc.mode)
assert len(errors) == 0
# If SG shadower does come up without panicking, bump a rev on a document that was modified during the downtime of the SG shadower
sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10)
# Make sure the doc that was added while shadower was down makes it to source bucket
# FAILING: Currently known to be failing, so temporarily disabled
# get_doc_from_source_bucket_retry(doc_id_shadower_down, sc.source_bucket)
# Manual check
# Grep logs for:
# WARNING: Error pushing rev of "f6eb40bf-d02a-4b4d-a3ab-779cce2fe9d5" to external bucket: MCResponse status=KEY_ENOENT, opcode=DELETE, opaque=0, msg: Not found -- db.(*Shadower).PushRevision() at shadower.go:164
# Verify SG shadower comes up without panicking, given writes from non-shadower during downtime.
time.sleep(5) # Give tap feed a chance to initialize
def test_bucket_shadow_low_revs_limit(params_from_base_test_setup):
"""
Set revs limit to 40
Add doc and makes sure it syncs to source bucket
Take shadower offline
Update one doc more than 50 times
Bring shadower online
Look for panics
Add more revisions to SG -- expected issue
Look for panics
(TODO: Update doc in shadow bucket and look for panics?)
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup["xattrs_enabled"]
if mode == "di" or xattrs_enabled:
pytest.skip("https://github.com/couchbase/sync_gateway/issues/2193")
log_info("Running 'test_bucket_shadow_low_revs_limit'")
log_info("Using cluster_config: {}".format(cluster_config))
default_config_path_shadower_low_revs = sync_gateway_config_path_for_mode(
"sync_gateway_bucketshadow_low_revs", mode)
default_config_path_non_shadower_low_revs = sync_gateway_config_path_for_mode(
"sync_gateway_default_low_revs", mode)
cluster = Cluster(config=cluster_config)
sc = init_shadow_cluster(cluster, default_config_path_shadower_low_revs,
default_config_path_non_shadower_low_revs)
# Write doc into shadower SG
doc_id = sc.alice_shadower.add_doc()
# Update the doc just so we have a rev_id
sc.alice_shadower.update_doc(doc_id,
content=fake_doc_content,
num_revision=1)
# Make sure it makes it to source bucket
get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content,
sc.source_bucket)
# Stop the SG shadower
sc.shadower_sg.stop()
# Update doc more than 50 times in non-shadower SG (since shadower is down)
sc.bob_non_shadower.update_doc(doc_id, num_revision=100)
sc.bob_non_shadower.update_doc(doc_id,
content=fake_doc_content,
num_revision=1)
# Bring SG shadower back up
sc.shadower_sg.start(default_config_path_shadower_low_revs)
# Look for panics
time.sleep(5) # Give tap feed a chance to initialize
errors = cluster.verify_alive(sc.mode)
assert len(errors) == 0
# Verify that the latest revision sync'd to source bucket
get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content,
sc.source_bucket)
# Add more revisions
sc.bob_non_shadower.update_doc(doc_id, num_revision=50)
sc.bob_non_shadower.update_doc(doc_id,
content=fake_doc_content,
num_revision=1)
# Verify that the latest revision sync'd to source bucket
get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content,
sc.source_bucket)
# Look for panics
time.sleep(5) # Wait until the shadower can process
