def params_from_base_test_setup(request, params_from_base_suite_setup): # Code before the yeild will execute before each test starts cluster_config = params_from_base_suite_setup["cluster_config"] mode = params_from_base_suite_setup["mode"] test_name = request.node.name log_info("Setting up test '{}'".format(test_name)) # This dictionary is passed to each test yield {"cluster_config": cluster_config, "mode": mode} # Code after the yeild will execute when each test finishes log_info("Tearing down test '{}'".format(test_name)) # Capture testkit socket usage network_utils = NetworkUtils() network_utils.list_connections() # Verify all sync_gateways and sg_accels are reachable c = Cluster(cluster_config) errors = c.verify_alive(mode) assert len(errors) == 0 # if the test failed pull logs if request.node.rep_call.failed: logging_helper = Logging() logging_helper.fetch_and_analyze_logs(cluster_config=cluster_config, test_name=test_name)
def params_from_base_test_setup(request, params_from_base_suite_setup): # Code before the yeild will execute before each test starts # pytest command line parameters collect_logs = request.config.getoption("--collect-logs") cluster_config = params_from_base_suite_setup["cluster_config"] mode = params_from_base_suite_setup["mode"] test_name = request.node.name log_info("Setting up test '{}'".format(test_name)) # This dictionary is passed to each test yield {"cluster_config": cluster_config, "mode": mode} # Code after the yeild will execute when each test finishes log_info("Tearing down test '{}'".format(test_name)) # Capture testkit socket usage network_utils = NetworkUtils() network_utils.list_connections() # Verify all sync_gateways and sg_accels are reachable c = Cluster(cluster_config) errors = c.verify_alive(mode) # if the test failed pull logs if collect_logs or request.node.rep_call.failed or len(errors) != 0: logging_helper = Logging() logging_helper.fetch_and_analyze_logs(cluster_config=cluster_config, test_name=test_name) assert len(errors) == 0
def test_bucket_shadow_low_revs_limit_repeated_deletes(params_from_base_test_setup): """ Validate that Sync Gateway doesn't panic (and instead creates a conflict branch and prints a warning) after doing the following steps: - Set revs_limit to 5 - Create a doc via SG - Issue a delete operation for that doc via SG - Repeat step 3 5x. (each additional delete will create a new revision in SG, but the delete on the source bucket will fail with the 'not found' error, which also means that upstream_rev won't get incremented - Recreate the doc in the source bucket See https://github.com/couchbaselabs/sync-gateway-testcluster/issues/291#issuecomment-191521993 """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] default_config_path_shadower_low_revs = sync_gateway_config_path_for_mode("sync_gateway_bucketshadow_low_revs", mode) default_config_path_non_shadower_low_revs = sync_gateway_config_path_for_mode("sync_gateway_default_low_revs", mode) log_info("Running 'test_bucket_shadow_low_revs_limit_repeated_deletes'") log_info("Using cluster_config: {}".format(cluster_config)) cluster = Cluster(config=cluster_config) sc = init_shadow_cluster(cluster, default_config_path_shadower_low_revs, default_config_path_non_shadower_low_revs) # Write doc into shadower SG doc_id = sc.alice_shadower.add_doc() # Wait until it gets to source bucket get_doc_from_source_bucket_retry(doc_id, sc.source_bucket) # Wait until upstream-rev in _sync metadata is non empty # Otherwise, this will not reproduce a panic while True: doc = sc.data_bucket.get(doc_id) if doc.success: if "upstream_rev" in doc.value["_sync"]: break time.sleep(1) # Repeatedly issue a delete operation for that doc via SG # Keep adding tombstone revs to the one and only branch rev_id_to_delete = None for i in xrange(100): resp = sc.alice_shadower.delete_doc(doc_id, rev_id_to_delete) rev_id_to_delete = resp["rev"] # Recreate doc with that ID in the source bucket sc.source_bucket.upsert(doc_id, json.loads('{"foo":"bar"}')) # Check if SG's are up errors = cluster.verify_alive(sc.mode) assert len(errors) == 0 # Restart Shadow SG sc.shadower_sg.stop() sc.shadower_sg.start(default_config_path_shadower_low_revs)
def test_dcp_reshard_sync_gateway_goes_down(params_from_base_test_setup, sg_conf): cluster_conf = params_from_base_test_setup["cluster_config"] log_info("Running 'test_dcp_reshard_sync_gateway_goes_down'") log_info("cluster_conf: {}".format(cluster_conf)) log_info("sg_conf: {}".format(sg_conf)) cluster = Cluster(config=cluster_conf) mode = cluster.reset(sg_config_path=sg_conf) admin = Admin(cluster.sync_gateways[0]) traun = admin.register_user(target=cluster.sync_gateways[0], db="db", name="traun", password="******", channels=["ABC", "NBC", "CBS"]) seth = admin.register_user(target=cluster.sync_gateways[0], db="db", name="seth", password="******", channels=["FOX"]) log_info(">> Users added") with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: futures = dict() log_info(">>> Adding Seth docs") # FOX futures[executor.submit(seth.add_docs, 8000)] = "seth" log_info(">>> Adding Traun docs") # ABC, NBC, CBS futures[executor.submit(traun.add_docs, 2000, bulk=True)] = "traun" # stop sg_accel shutdown_status = cluster.sg_accels[0].stop() assert shutdown_status == 0 for future in concurrent.futures.as_completed(futures): tag = futures[future] log_info("{} Completed:".format(tag)) # TODO better way to do this time.sleep(120) verify_changes(traun, expected_num_docs=2000, expected_num_revisions=0, expected_docs=traun.cache) verify_changes(seth, expected_num_docs=8000, expected_num_revisions=0, expected_docs=seth.cache) # Verify that the sg1 is down but the other sync_gateways are running errors = cluster.verify_alive(mode) assert len(errors) == 1 and errors[0][0].hostname == "ac1" # Restart the failing node so that cluster verification does not blow up in test teardown start_status = cluster.sg_accels[0].start(sg_conf) assert start_status == 0
def test_bucket_shadow_multiple_sync_gateways(params_from_base_test_setup): cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] log_info("Running 'test_bucket_shadow_multiple_sync_gateways'") log_info("Using cluster_config: {}".format(cluster_config)) default_config_path_shadower = sync_gateway_config_path_for_mode("sync_gateway_bucketshadow", mode) default_config_path_non_shadower = sync_gateway_config_path_for_mode("sync_gateway_default", mode) cluster = Cluster(config=cluster_config) sc = init_shadow_cluster( cluster, default_config_path_shadower, default_config_path_non_shadower, ) # Write several docs into shadower SG doc_id_alice = sc.alice_shadower.add_doc() doc_id_will_delete = sc.alice_shadower.add_doc() # Write several docs into non-shadower SG doc_id_bob = sc.bob_non_shadower.add_doc() # Ditto as above, but bump revs rather than writing brand new docs sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10) sc.alice_shadower.update_doc(doc_id_alice, num_revision=10) # Get the doc from the source bucket, possibly retrying if needed # Otherwise an exception will be thrown and the test will fail get_doc_from_source_bucket_retry(doc_id_bob, sc.source_bucket) get_doc_from_source_bucket_retry(doc_id_alice, sc.source_bucket) get_doc_from_source_bucket_retry(doc_id_will_delete, sc.source_bucket) # Stop the SG shadower sc.shadower_sg.stop() # Write several docs + bump revs into non-shadower SG sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10) sc.bob_non_shadower.add_doc() # Delete one of the docs sc.bob_non_shadower.delete_doc(doc_id_will_delete) # Bring SG shadower back up sc.shadower_sg.start(default_config_path_shadower) time.sleep(5) # Give tap feed a chance to initialize # Verify SG shadower comes up without panicking, given writes from non-shadower during downtime. errors = cluster.verify_alive(sc.mode) assert len(errors) == 0 # If SG shadower does come up without panicking, bump a rev on a document that was modified during the downtime of the SG shadower sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10) # Make sure the doc that was added while shadower was down makes it to source bucket # FAILING: Currently known to be failing, so temporarily disabled # get_doc_from_source_bucket_retry(doc_id_shadower_down, sc.source_bucket) # Manual check # Grep logs for: # WARNING: Error pushing rev of "f6eb40bf-d02a-4b4d-a3ab-779cce2fe9d5" to external bucket: MCResponse status=KEY_ENOENT, opcode=DELETE, opaque=0, msg: Not found -- db.(*Shadower).PushRevision() at shadower.go:164 # Verify SG shadower comes up without panicking, given writes from non-shadower during downtime. time.sleep(5) # Give tap feed a chance to initialize
def test_bucket_shadow_low_revs_limit(params_from_base_test_setup): """ Set revs limit to 40 Add doc and makes sure it syncs to source bucket Take shadower offline Update one doc more than 50 times Bring shadower online Look for panics Add more revisions to SG -- expected issue Look for panics (TODO: Update doc in shadow bucket and look for panics?) """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] log_info("Running 'test_bucket_shadow_low_revs_limit'") log_info("Using cluster_config: {}".format(cluster_config)) default_config_path_shadower_low_revs = sync_gateway_config_path_for_mode("sync_gateway_bucketshadow_low_revs", mode) default_config_path_non_shadower_low_revs = sync_gateway_config_path_for_mode("sync_gateway_default_low_revs", mode) cluster = Cluster(config=cluster_config) sc = init_shadow_cluster(cluster, default_config_path_shadower_low_revs, default_config_path_non_shadower_low_revs) # Write doc into shadower SG doc_id = sc.alice_shadower.add_doc() # Update the doc just so we have a rev_id sc.alice_shadower.update_doc(doc_id, content=fake_doc_content, num_revision=1) # Make sure it makes it to source bucket get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content, sc.source_bucket) # Stop the SG shadower sc.shadower_sg.stop() # Update doc more than 50 times in non-shadower SG (since shadower is down) sc.bob_non_shadower.update_doc(doc_id, num_revision=100) sc.bob_non_shadower.update_doc(doc_id, content=fake_doc_content, num_revision=1) # Bring SG shadower back up sc.shadower_sg.start(default_config_path_shadower_low_revs) # Look for panics time.sleep(5) # Give tap feed a chance to initialize errors = cluster.verify_alive(sc.mode) assert len(errors) == 0 # Verify that the latest revision sync'd to source bucket get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content, sc.source_bucket) # Add more revisions sc.bob_non_shadower.update_doc(doc_id, num_revision=50) sc.bob_non_shadower.update_doc(doc_id, content=fake_doc_content, num_revision=1) # Verify that the latest revision sync'd to source bucket get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content, sc.source_bucket) # Look for panics time.sleep(5) # Wait until the shadower can process
def test_bucket_shadow_low_revs_limit_repeated_deletes( params_from_base_test_setup): """ Validate that Sync Gateway doesn't panic (and instead creates a conflict branch and prints a warning) after doing the following steps: - Set revs_limit to 5 - Create a doc via SG - Issue a delete operation for that doc via SG - Repeat step 3 5x. (each additional delete will create a new revision in SG, but the delete on the source bucket will fail with the 'not found' error, which also means that upstream_rev won't get incremented - Recreate the doc in the source bucket See https://github.com/couchbaselabs/sync-gateway-testcluster/issues/291#issuecomment-191521993 """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] xattrs_enabled = params_from_base_test_setup["xattrs_enabled"] if mode == "di" or xattrs_enabled: pytest.skip("https://github.com/couchbase/sync_gateway/issues/2193") default_config_path_shadower_low_revs = sync_gateway_config_path_for_mode( "sync_gateway_bucketshadow_low_revs", mode) default_config_path_non_shadower_low_revs = sync_gateway_config_path_for_mode( "sync_gateway_default_low_revs", mode) log_info("Running 'test_bucket_shadow_low_revs_limit_repeated_deletes'") log_info("Using cluster_config: {}".format(cluster_config)) cluster = Cluster(config=cluster_config) sc = init_shadow_cluster(cluster, default_config_path_shadower_low_revs, default_config_path_non_shadower_low_revs) # Write doc into shadower SG doc_id = sc.alice_shadower.add_doc() # Wait until it gets to source bucket get_doc_from_source_bucket_retry(doc_id, sc.source_bucket) # Wait until upstream-rev in _sync metadata is non empty # Otherwise, this will not reproduce a panic while True: doc = sc.data_bucket.get(doc_id) if doc.success: if "upstream_rev" in doc.value["_sync"]: break time.sleep(1) # Repeatedly issue a delete operation for that doc via SG # Keep adding tombstone revs to the one and only branch rev_id_to_delete = None for i in xrange(100): resp = sc.alice_shadower.delete_doc(doc_id, rev_id_to_delete) rev_id_to_delete = resp["rev"] # Recreate doc with that ID in the source bucket sc.source_bucket.upsert(doc_id, json.loads('{"foo":"bar"}')) # Check if SG's are up errors = cluster.verify_alive(sc.mode) assert len(errors) == 0 # Restart Shadow SG sc.shadower_sg.stop() sc.shadower_sg.start(default_config_path_shadower_low_revs)
def test_bucket_shadow_multiple_sync_gateways(params_from_base_test_setup): cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] xattrs_enabled = params_from_base_test_setup["xattrs_enabled"] if mode == "di" or xattrs_enabled: pytest.skip("https://github.com/couchbase/sync_gateway/issues/2193") log_info("Running 'test_bucket_shadow_multiple_sync_gateways'") log_info("Using cluster_config: {}".format(cluster_config)) default_config_path_shadower = sync_gateway_config_path_for_mode( "sync_gateway_bucketshadow", mode) default_config_path_non_shadower = sync_gateway_config_path_for_mode( "sync_gateway_default", mode) cluster = Cluster(config=cluster_config) sc = init_shadow_cluster( cluster, default_config_path_shadower, default_config_path_non_shadower, ) # Write several docs into shadower SG doc_id_alice = sc.alice_shadower.add_doc() doc_id_will_delete = sc.alice_shadower.add_doc() # Write several docs into non-shadower SG doc_id_bob = sc.bob_non_shadower.add_doc() # Ditto as above, but bump revs rather than writing brand new docs sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10) sc.alice_shadower.update_doc(doc_id_alice, num_revision=10) # Get the doc from the source bucket, possibly retrying if needed # Otherwise an exception will be thrown and the test will fail get_doc_from_source_bucket_retry(doc_id_bob, sc.source_bucket) get_doc_from_source_bucket_retry(doc_id_alice, sc.source_bucket) get_doc_from_source_bucket_retry(doc_id_will_delete, sc.source_bucket) # Stop the SG shadower sc.shadower_sg.stop() # Write several docs + bump revs into non-shadower SG sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10) sc.bob_non_shadower.add_doc() # Delete one of the docs sc.bob_non_shadower.delete_doc(doc_id_will_delete) # Bring SG shadower back up sc.shadower_sg.start(default_config_path_shadower) time.sleep(5) # Give tap feed a chance to initialize # Verify SG shadower comes up without panicking, given writes from non-shadower during downtime. errors = cluster.verify_alive(sc.mode) assert len(errors) == 0 # If SG shadower does come up without panicking, bump a rev on a document that was modified during the downtime of the SG shadower sc.bob_non_shadower.update_doc(doc_id_bob, num_revision=10) # Make sure the doc that was added while shadower was down makes it to source bucket # FAILING: Currently known to be failing, so temporarily disabled # get_doc_from_source_bucket_retry(doc_id_shadower_down, sc.source_bucket) # Manual check # Grep logs for: # WARNING: Error pushing rev of "f6eb40bf-d02a-4b4d-a3ab-779cce2fe9d5" to external bucket: MCResponse status=KEY_ENOENT, opcode=DELETE, opaque=0, msg: Not found -- db.(*Shadower).PushRevision() at shadower.go:164 # Verify SG shadower comes up without panicking, given writes from non-shadower during downtime. time.sleep(5) # Give tap feed a chance to initialize
def test_bucket_shadow_low_revs_limit(params_from_base_test_setup): """ Set revs limit to 40 Add doc and makes sure it syncs to source bucket Take shadower offline Update one doc more than 50 times Bring shadower online Look for panics Add more revisions to SG -- expected issue Look for panics (TODO: Update doc in shadow bucket and look for panics?) """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] xattrs_enabled = params_from_base_test_setup["xattrs_enabled"] if mode == "di" or xattrs_enabled: pytest.skip("https://github.com/couchbase/sync_gateway/issues/2193") log_info("Running 'test_bucket_shadow_low_revs_limit'") log_info("Using cluster_config: {}".format(cluster_config)) default_config_path_shadower_low_revs = sync_gateway_config_path_for_mode( "sync_gateway_bucketshadow_low_revs", mode) default_config_path_non_shadower_low_revs = sync_gateway_config_path_for_mode( "sync_gateway_default_low_revs", mode) cluster = Cluster(config=cluster_config) sc = init_shadow_cluster(cluster, default_config_path_shadower_low_revs, default_config_path_non_shadower_low_revs) # Write doc into shadower SG doc_id = sc.alice_shadower.add_doc() # Update the doc just so we have a rev_id sc.alice_shadower.update_doc(doc_id, content=fake_doc_content, num_revision=1) # Make sure it makes it to source bucket get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content, sc.source_bucket) # Stop the SG shadower sc.shadower_sg.stop() # Update doc more than 50 times in non-shadower SG (since shadower is down) sc.bob_non_shadower.update_doc(doc_id, num_revision=100) sc.bob_non_shadower.update_doc(doc_id, content=fake_doc_content, num_revision=1) # Bring SG shadower back up sc.shadower_sg.start(default_config_path_shadower_low_revs) # Look for panics time.sleep(5) # Give tap feed a chance to initialize errors = cluster.verify_alive(sc.mode) assert len(errors) == 0 # Verify that the latest revision sync'd to source bucket get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content, sc.source_bucket) # Add more revisions sc.bob_non_shadower.update_doc(doc_id, num_revision=50) sc.bob_non_shadower.update_doc(doc_id, content=fake_doc_content, num_revision=1) # Verify that the latest revision sync'd to source bucket get_doc_with_content_from_source_bucket_retry(doc_id, fake_doc_content, sc.source_bucket) # Look for panics time.sleep(5) # Wait until the shadower can process