def test_seq(params_from_base_test_setup, sg_conf_name, num_users, num_docs, num_revisions):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running seq")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
log_info("num_users: {}".format(num_users))
log_info("num_docs: {}".format(num_docs))
log_info("num_revisions: {}".format(num_revisions))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
admin = Admin(cluster.sync_gateways[0])
# all users will share docs due to having the same channel
users = admin.register_bulk_users(target=cluster.sync_gateways[0], db="db", name_prefix="user", number=num_users, password="******", channels=["ABC"])
for user in users:
user.add_docs(num_docs, bulk=True)
for user in users:
user.update_docs(num_revisions)
time.sleep(5)
user_0_changes = users[0].get_changes(since=0)
doc_seq = user_0_changes["results"][num_docs / 2]["seq"]
# https://github.com/couchbase/sync_gateway/issues/1475#issuecomment-172426052
# verify you can issue _changes with since=12313-0::1023.15
for user in users:
changes = user.get_changes(since=doc_seq)
log_info("Trying changes with since={}".format(doc_seq))
assert len(changes["results"]) > 0
second_to_last_doc_entry_seq = changes["results"][-2]["seq"]
last_doc_entry_seq = changes["results"][-1]["seq"]
log_info('Second to last doc "seq": {}'.format(second_to_last_doc_entry_seq))
log_info('Last doc "seq": {}'.format(last_doc_entry_seq))
if mode == "di":
# Verify last "seq" follows the formate 12313-0, not 12313-0::1023.15
log_info('Verify that the last "seq" is a plain hashed value')
assert len(second_to_last_doc_entry_seq.split("::")) == 2
assert len(last_doc_entry_seq.split("::")) == 1
elif mode == "cc":
assert second_to_last_doc_entry_seq > 0
assert last_doc_entry_seq > 0
else:
raise ValueError("Unsupported 'mode' !!")
all_doc_caches = [user.cache for user in users]
all_docs = {k: v for cache in all_doc_caches for k, v in cache.items()}
verify_changes(users, expected_num_docs=num_users * num_docs, expected_num_revisions=num_revisions, expected_docs=all_docs)
def test_multiple_db_single_data_bucket_single_index_bucket(params_from_base_test_setup, sg_conf_name, num_users, num_docs_per_user):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'multiple_db_unique_data_bucket_unique_index_bucket'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
log_info("Using num_users: {}".format(num_users))
log_info("Using num_docs_per_user: {}".format(num_docs_per_user))
# 2 dbs share the same data and index bucket
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
num_db_users = num_users
num_db2_users = num_users
num_docs_per_user = num_docs_per_user
admin = Admin(cluster.sync_gateways[0])
db_one_users = admin.register_bulk_users(target=cluster.sync_gateways[0], db="db", name_prefix="bulk_db_user", number=num_db_users, password="******", channels=["ABC"])
db_two_users = admin.register_bulk_users(target=cluster.sync_gateways[0], db="db2", name_prefix="bulk_db2_user", number=num_db2_users, password="******", channels=["ABC"])
all_users = list(db_one_users)
all_users.extend(db_two_users)
assert len(all_users) == num_db_users + num_db2_users
# Round robin
num_sgs = len(cluster.sync_gateways)
count = 1
for user in all_users:
user.add_docs(num_docs_per_user, bulk=True)
user.target = cluster.sync_gateways[(count + 1) % num_sgs]
count += 1
time.sleep(10)
# Get list of all docs from users caches
cached_docs_from_all_users = {k: v for user in all_users for k, v in user.cache.items()}
# Verify each user has all of the docs
verify_changes(all_users, expected_num_docs=(num_users * 2) * num_docs_per_user, expected_num_revisions=0, expected_docs=cached_docs_from_all_users)
def test_multiple_db_single_data_bucket_single_index_bucket(params_from_base_test_setup, sg_conf_name, num_users, num_docs_per_user):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'multiple_db_unique_data_bucket_unique_index_bucket'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
log_info("Using num_users: {}".format(num_users))
log_info("Using num_docs_per_user: {}".format(num_docs_per_user))
# 2 dbs share the same data and index bucket
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
num_db_users = num_users
num_db2_users = num_users
num_docs_per_user = num_docs_per_user
admin = Admin(cluster.sync_gateways[0])
db_one_users = admin.register_bulk_users(target=cluster.sync_gateways[0], db="db", name_prefix="bulk_db_user", number=num_db_users, password="******", channels=["ABC"])
db_two_users = admin.register_bulk_users(target=cluster.sync_gateways[0], db="db2", name_prefix="bulk_db2_user", number=num_db2_users, password="******", channels=["ABC"])
all_users = list(db_one_users)
all_users.extend(db_two_users)
assert len(all_users) == num_db_users + num_db2_users
# Round robin
num_sgs = len(cluster.sync_gateways)
count = 1
for user in all_users:
user.add_docs(num_docs_per_user, bulk=True)
user.target = cluster.sync_gateways[(count + 1) % num_sgs]
count += 1
time.sleep(10)
# Get list of all docs from users caches
cached_docs_from_all_users = {k: v for user in all_users for k, v in user.cache.items()}
# Verify each user has all of the docs
verify_changes(all_users, expected_num_docs=(num_users * 2) * num_docs_per_user, expected_num_revisions=0, expected_docs=cached_docs_from_all_users)
def test_db_online_offline_webhooks_offline_two(params_from_base_test_setup, sg_conf_name, num_users, num_channels, num_docs, num_revisions):
start = time.time()
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'test_db_online_offline_webhooks_offline_two'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using num_users: {}".format(num_users))
log_info("Using num_channels: {}".format(num_channels))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using num_revisions: {}".format(num_revisions))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_conf)
init_completed = time.time()
log_info("Initialization completed. Time taken:{}s".format(init_completed - start))
channels = ["channel-" + str(i) for i in range(num_channels)]
password = "******"
ws = WebServer()
ws.start()
sgs = cluster.sync_gateways
admin = Admin(sgs[0])
# Register User
log_info("Register User")
user_objects = admin.register_bulk_users(target=sgs[0], db="db", name_prefix="User",
number=num_users, password=password, channels=channels)
# Add User
log_info("Add docs")
in_parallel(user_objects, 'add_docs', num_docs)
# Update docs
log_info("Update docs")
in_parallel(user_objects, 'update_docs', num_revisions)
time.sleep(10)
status = cluster.servers[0].delete_bucket("data-bucket")
assert status == 0
log_info("Sleeping for 120 seconds...")
time.sleep(120)
webhook_events = ws.get_data()
time.sleep(5)
log_info("webhook event {}".format(webhook_events))
last_event = webhook_events[-1]
assert last_event['state'] == 'offline'
ws.stop()
def test_online_to_offline_longpoll_changes_feed_controlled_close_sanity_mulitple_users(params_from_base_test_setup, sg_conf_name, num_docs, num_users):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using num_users: {}".format(num_users))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
admin = Admin(cluster.sync_gateways[0])
users = admin.register_bulk_users(target=cluster.sync_gateways[0], db="db", name_prefix="user", password="******", number=num_users, channels=["ABC"])
feed_close_results = list()
with concurrent.futures.ThreadPoolExecutor(max_workers=libraries.testkit.settings.MAX_REQUEST_WORKERS) as executor:
# start longpoll tracking with no timeout, will block until longpoll is closed by db going offline
futures = {executor.submit(user.start_longpoll_changes_tracking, termination_doc_id=None, timeout=0, loop=False): user.name for user in users}
time.sleep(5)
futures[executor.submit(admin.take_db_offline, "db")] = "db_offline_task"
for future in concurrent.futures.as_completed(futures):
task_name = futures[future]
if task_name == "db_offline_task":
log_info("DB OFFLINE")
# make sure db_offline returns 200
assert future.result() == 200
if task_name.startswith("user"):
# Long poll will exit with 503, return docs in the exception
log_info("POLLING DONE")
try:
docs_in_changes, last_seq_num = future.result()
feed_close_results.append((docs_in_changes, last_seq_num))
except Exception as e:
log_info("Longpoll feed close error: {}".format(e))
# long poll should be closed so this exception should never happen
assert 0
# Assert that the feed close results length is num_users
assert len(feed_close_results) == num_users
# Account for _user doc
# last_seq may be of the form '1' for channel cache or '1-0' for distributed index
for feed_result in feed_close_results:
docs_in_changes = feed_result[0]
seq_num_component = feed_result[1].split("-")
assert len(docs_in_changes) == 0
assert int(seq_num_component[0]) > 0
def test_webhooks(params_from_base_test_setup, sg_conf_name, num_users, num_channels, num_docs, num_revisions):
start = time.time()
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'test_webhooks'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using num_users: {}".format(num_users))
log_info("Using num_channels: {}".format(num_channels))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using num_revisions: {}".format(num_revisions))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_conf)
init_completed = time.time()
log_info("Initialization completed. Time taken:{}s".format(init_completed - start))
channels = ["channel-" + str(i) for i in range(num_channels)]
password = "******"
ws = WebServer()
ws.start()
sgs = cluster.sync_gateways
admin = Admin(sgs[0])
# Register User
log_info("Register User")
user_objects = admin.register_bulk_users(target=sgs[0], db="db", name_prefix="User",
number=num_users, password=password, channels=channels)
# Add User
log_info("Add docs")
in_parallel(user_objects, 'add_docs', num_docs)
# Update docs
log_info("Update docs")
in_parallel(user_objects, 'update_docs', num_revisions)
time.sleep(30)
ws.stop()
expected_events = (num_users * num_docs * num_revisions) + (num_users * num_docs)
received_events = len(ws.get_data())
log_info("expected_events: {} received_events {}".format(expected_events, received_events))
assert expected_events == received_events
def test_continuous_changes_parametrized(params_from_base_test_setup, sg_conf_name, num_users, num_docs, num_revisions):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'continuous_changes_parametrized'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
log_info("num_users: {}".format(num_users))
log_info("num_docs: {}".format(num_docs))
log_info("num_revisions: {}".format(num_revisions))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
admin = Admin(cluster.sync_gateways[0])
users = admin.register_bulk_users(target=cluster.sync_gateways[0], db="db", name_prefix="user", number=num_users, password="******", channels=["ABC", "TERMINATE"])
abc_doc_pusher = admin.register_user(target=cluster.sync_gateways[0], db="db", name="abc_doc_pusher", password="******", channels=["ABC"])
doc_terminator = admin.register_user(target=cluster.sync_gateways[0], db="db", name="doc_terminator", password="******", channels=["TERMINATE"])
with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
futures = {executor.submit(user.start_continuous_changes_tracking, termination_doc_id="killcontinuous"): user.name for user in users}
futures[executor.submit(abc_doc_pusher.add_docs, num_docs)] = "doc_pusher"
for future in concurrent.futures.as_completed(futures):
task_name = futures[future]
# Send termination doc to seth continuous changes feed subscriber
if task_name == "doc_pusher":
errors = future.result()
assert len(errors) == 0
abc_doc_pusher.update_docs(num_revs_per_doc=num_revisions)
time.sleep(10)
doc_terminator.add_doc("killcontinuous")
elif task_name.startswith("user"):
# When the user has continuous _changes feed closed, return the docs and verify the user got all the channel docs
docs_in_changes = future.result()
# Expect number of docs + the termination doc + _user doc
verify_same_docs(expected_num_docs=num_docs, doc_dict_one=docs_in_changes, doc_dict_two=abc_doc_pusher.cache)
# Expect number of docs + the termination doc
verify_changes(abc_doc_pusher, expected_num_docs=num_docs, expected_num_revisions=num_revisions, expected_docs=abc_doc_pusher.cache)
def test_roles_sanity(params_from_base_test_setup, sg_conf_name):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'roles_sanity'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
radio_stations = ["KMOW", "HWOD", "KDWB"]
tv_stations = ["ABC", "CBS", "NBC"]
number_of_djs = 10
number_of_vjs = 10
number_of_docs_per_pusher = 500
admin = Admin(cluster.sync_gateways[0])
admin.create_role("db", name="radio_stations", channels=radio_stations)
admin.create_role("db", name="tv_stations", channels=tv_stations)
djs = admin.register_bulk_users(target=cluster.sync_gateways[0], db="db", name_prefix="dj", number=number_of_djs, password="******", roles=["radio_stations"])
vjs = admin.register_bulk_users(target=cluster.sync_gateways[0], db="db", name_prefix="vj", number=number_of_vjs, password="******", roles=["tv_stations"])
mogul = admin.register_user(target=cluster.sync_gateways[0], db="db", name="mogul", password="******", roles=["tv_stations", "radio_stations"])
radio_doc_caches = []
for radio_station in radio_stations:
doc_pusher = admin.register_user(target=cluster.sync_gateways[0], db="db", name="{}_doc_pusher".format(radio_station), password="******", channels=[radio_station])
doc_pusher.add_docs(number_of_docs_per_pusher, bulk=True)
radio_doc_caches.append(doc_pusher.cache)
radio_docs = {k: v for cache in radio_doc_caches for k, v in cache.items()}
tv_doc_caches = []
for tv_station in tv_stations:
doc_pusher = admin.register_user(target=cluster.sync_gateways[0], db="db", name="{}_doc_pusher".format(tv_station), password="******", channels=[tv_station])
doc_pusher.add_docs(number_of_docs_per_pusher, bulk=True)
tv_doc_caches.append(doc_pusher.cache)
tv_docs = {k: v for cache in tv_doc_caches for k, v in cache.items()}
# Verify djs get docs for all the channels associated with the radio_stations role
expected_num_radio_docs = len(radio_stations) * number_of_docs_per_pusher
verify_changes(djs, expected_num_docs=expected_num_radio_docs, expected_num_revisions=0, expected_docs=radio_docs)
# Verify vjs get docs for all the channels associated with the tv_stations role
expected_num_tv_docs = len(tv_stations) * number_of_docs_per_pusher
verify_changes(vjs, expected_num_docs=expected_num_tv_docs, expected_num_revisions=0, expected_docs=tv_docs)
# Verify mogul gets docs for all the channels associated with the radio_stations + tv_stations roles
all_docs_caches = list(radio_doc_caches)
all_docs_caches.extend(tv_doc_caches)
all_docs = {k: v for cache in all_docs_caches for k, v in cache.items()}
verify_changes(mogul, expected_num_docs=expected_num_radio_docs + expected_num_tv_docs, expected_num_revisions=0, expected_docs=all_docs)
def test_bucket_online_offline_resync_sanity(params_from_base_test_setup, sg_conf_name, num_users, num_docs, num_revisions):
cluster_conf = params_from_base_test_setup["cluster_config"]
test_mode = params_from_base_test_setup["mode"]
if test_mode == "di":
pytest.skip("Unsupported feature in distributed index")
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, test_mode)
log_info("Running 'test_bucket_online_offline_resync_sanity'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using num_users: {}".format(num_users))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using num_revisions: {}".format(num_revisions))
start = time.time()
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_conf)
init_completed = time.time()
log_info("Initialization completed. Time taken:{}s".format(init_completed - start))
num_channels = 1
channels = ["channel-" + str(i) for i in range(num_channels)]
password = "******"
sgs = cluster.sync_gateways
admin = Admin(sgs[0])
# Register User
log_info("Register User")
user_objects = admin.register_bulk_users(target=sgs[0], db="db", name_prefix="User",
number=num_users, password=password, channels=channels)
user_x = admin.register_user(target=sgs[0], db="db", name="User-X", password="******", channels=["channel_x"])
# Add User
log_info("Add docs")
in_parallel(user_objects, 'add_docs', num_docs)
# Update docs
log_info("Update docs")
in_parallel(user_objects, 'update_docs', num_revisions)
time.sleep(10)
# Get changes for all users
in_parallel(user_objects, 'get_changes')
# every user should have same number of docs
# total/expected docs = num_users * num_docs
recieved_docs = in_parallel(user_objects, 'get_num_docs')
expected_docs = num_users * num_docs
for user_obj, docs in recieved_docs.items():
log_info('User {} got {} docs, expected docs: {}'.format(user_obj.name, docs, expected_docs))
assert docs == expected_docs
# Verify that
# user created doc-ids exist in docs received in changes feed
# expected revision is equal to received revision
expected_revision = str(num_revisions + 1)
docs_rev_dict = in_parallel(user_objects, 'get_num_revisions')
rev_errors = []
for user_obj, docs_revision_dict in docs_rev_dict.items():
for doc_id in docs_revision_dict.keys():
rev = docs_revision_dict[doc_id]
log_info('User {} doc_id {} has {} revisions, expected revision: {}'.format(user_obj.name,
doc_id, rev, expected_revision))
if rev != expected_revision:
rev_errors.append(doc_id)
log_error('User {} doc_id {} got revision {}, expected revision {}'.format(
user_obj.name,
doc_id,
rev,
expected_revision)
)
assert len(rev_errors) == 0
# Verify each User created docs are part of changes feed
output = in_parallel(user_objects, 'check_doc_ids_in_changes_feed')
assert True in output.values()
# Take "db" offline
status = admin.take_db_offline(db="db")
assert status == 200
sg_restart_config = sync_gateway_config_path_for_mode("bucket_online_offline/db_online_offline_access_restricted", test_mode)
restart_status = cluster.sync_gateways[0].restart(sg_restart_config)
assert restart_status == 0
time.sleep(10)
num_changes = admin.db_resync(db="db")
log_info("expecting num_changes {} == num_docs {} * num_users {}".format(num_changes, num_docs, num_users))
assert num_changes['payload']['changes'] == num_docs * num_users
status = admin.bring_db_online(db="db")
assert status == 200
time.sleep(5)
global_cache = list()
for user in user_objects:
global_cache.append(user.cache)
all_docs = {k: v for user_cache in global_cache for k, v in user_cache.items()}
verify_changes(user_x, expected_num_docs=expected_docs, expected_num_revisions=num_revisions, expected_docs=all_docs)
end = time.time()
log_info("Test ended.")
log_info("Main test duration: {}".format(end - init_completed))
log_info("Test setup time: {}".format(init_completed - start))
log_info("Total Time taken: {}s".format(end - start))
def test_bucket_online_offline_resync_with_offline(params_from_base_test_setup, sg_conf_name, num_users, num_docs, num_revisions):
start = time.time()
cluster_conf = params_from_base_test_setup["cluster_config"]
test_mode = params_from_base_test_setup["mode"]
if test_mode == "di":
pytest.skip("Unsupported feature in distributed index")
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, test_mode)
log_info("Running 'test_bucket_online_offline_resync_with_online'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using num_users: {}".format(num_users))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using num_revisions: {}".format(num_revisions))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_conf)
init_completed = time.time()
log_info("Initialization completed. Time taken:{}s".format(init_completed - start))
num_channels = 1
channels = ["channel-" + str(i) for i in range(num_channels)]
password = "******"
sgs = cluster.sync_gateways
admin = Admin(sgs[0])
# Register User
log_info("Register User")
user_objects = admin.register_bulk_users(target=sgs[0], db="db", name_prefix="User",
number=num_users, password=password, channels=channels)
user_x = admin.register_user(target=sgs[0], db="db", name="User-X", password="******", channels=["channel_x"])
# Add User
log_info("Add docs")
in_parallel(user_objects, 'add_docs', num_docs)
# Update docs
log_info("Update docs")
in_parallel(user_objects, 'update_docs', num_revisions)
time.sleep(10)
# Get changes for all users
in_parallel(user_objects, 'get_changes')
# every user should have same number of docs
# total/expected docs = num_users * num_docs
recieved_docs = in_parallel(user_objects, 'get_num_docs')
expected_docs = num_users * num_docs
for user_obj, docs in recieved_docs.items():
log_info('User {} got {} docs, expected docs: {}'.format(user_obj.name, docs, expected_docs))
assert docs == expected_docs
# Verify that
# user created doc-ids exist in docs received in changes feed
# expected revision is equal to received revision
expected_revision = str(num_revisions + 1)
docs_rev_dict = in_parallel(user_objects, 'get_num_revisions')
rev_errors = []
for user_obj, docs_revision_dict in docs_rev_dict.items():
for doc_id in docs_revision_dict.keys():
rev = docs_revision_dict[doc_id]
log_info('User {} doc_id {} has {} revisions, expected revision: {}'.format(
user_obj.name,
doc_id,
rev,
expected_revision
))
if rev != expected_revision:
rev_errors.append(doc_id)
log_error('User {} doc_id {} got revision {}, expected revision {}'.format(
user_obj.name,
doc_id,
rev,
expected_revision
))
assert len(rev_errors) == 0
# Verify each User created docs are part of changes feed
output = in_parallel(user_objects, 'check_doc_ids_in_changes_feed')
assert True in output.values()
# Take "db" offline
status = admin.take_db_offline(db="db")
assert status == 200
sg_restart_config = sync_gateway_config_path_for_mode("bucket_online_offline/db_online_offline_access_restricted", test_mode)
restart_status = cluster.sync_gateways[0].restart(sg_restart_config)
assert restart_status == 0
log_info("Sleeping....")
time.sleep(10)
pool = ThreadPool(processes=1)
log_info("Restarted SG....")
time.sleep(5)
db_info = admin.get_db_info("db")
log_info("Status of db = {}".format(db_info["state"]))
assert db_info["state"] == "Offline"
try:
async_resync_result = pool.apply_async(admin.db_resync, ("db",))
log_info("resync issued !!!!!!")
except Exception as e:
log_info("Catch resync exception: {}".format(e))
time.sleep(1)
resync_occured = False
for i in range(20):
db_info = admin.get_db_info("db")
log_info("Status of db = {}".format(db_info["state"]))
if db_info["state"] == "Resyncing":
resync_occured = True
log_info("Resync occured")
try:
status = admin.get_db_info(db="db")
log_info("Got db_info request status: {}".format(status))
except HTTPError as e:
log_info("status = {} exception = {}".format(status, e.response.status_code))
assert False
else:
log_info("Got 200 ok for supported operation")
time.sleep(1)
if resync_occured:
break
time.sleep(10)
status = admin.bring_db_online(db="db")
log_info("online request issued !!!!! response status: {}".format(status))
time.sleep(5)
db_info = admin.get_db_info("db")
log_info("Status of db = {}".format(db_info["state"]))
assert db_info["state"] == "Online"
resync_result = async_resync_result.get()
log_info("resync_changes {}".format(resync_result))
log_info("expecting num_changes == num_docs {} * num_users {}".format(num_docs, num_users))
assert resync_result['payload']['changes'] == num_docs * num_users
assert resync_result['status_code'] == 200
time.sleep(5)
global_cache = list()
for user in user_objects:
global_cache.append(user.cache)
all_docs = {k: v for user_cache in global_cache for k, v in user_cache.items()}
verify_changes(user_x, expected_num_docs=expected_docs, expected_num_revisions=num_revisions, expected_docs=all_docs)
end = time.time()
log_info("Test ended.")
log_info("Main test duration: {}".format(end - init_completed))
log_info("Test setup time: {}".format(init_completed - start))
log_info("Total Time taken: {}s".format(end - start))
def test_roles_sanity(params_from_base_test_setup, sg_conf_name):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'roles_sanity'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
radio_stations = ["KMOW", "HWOD", "KDWB"]
tv_stations = ["ABC", "CBS", "NBC"]
number_of_djs = 10
number_of_vjs = 10
number_of_docs_per_pusher = 500
admin = Admin(cluster.sync_gateways[0])
admin.create_role("db", name="radio_stations", channels=radio_stations)
admin.create_role("db", name="tv_stations", channels=tv_stations)
djs = admin.register_bulk_users(target=cluster.sync_gateways[0],
db="db",
name_prefix="dj",
number=number_of_djs,
password="******",
roles=["radio_stations"])
vjs = admin.register_bulk_users(target=cluster.sync_gateways[0],
db="db",
name_prefix="vj",
number=number_of_vjs,
password="******",
roles=["tv_stations"])
mogul = admin.register_user(target=cluster.sync_gateways[0],
db="db",
name="mogul",
password="******",
roles=["tv_stations", "radio_stations"])
radio_doc_caches = []
for radio_station in radio_stations:
doc_pusher = admin.register_user(
target=cluster.sync_gateways[0],
db="db",
name="{}_doc_pusher".format(radio_station),
password="******",
channels=[radio_station])
doc_pusher.add_docs(number_of_docs_per_pusher, bulk=True)
radio_doc_caches.append(doc_pusher.cache)
radio_docs = {k: v for cache in radio_doc_caches for k, v in cache.items()}
tv_doc_caches = []
for tv_station in tv_stations:
doc_pusher = admin.register_user(
target=cluster.sync_gateways[0],
db="db",
name="{}_doc_pusher".format(tv_station),
password="******",
channels=[tv_station])
doc_pusher.add_docs(number_of_docs_per_pusher, bulk=True)
tv_doc_caches.append(doc_pusher.cache)
tv_docs = {k: v for cache in tv_doc_caches for k, v in cache.items()}
# Verify djs get docs for all the channels associated with the radio_stations role
expected_num_radio_docs = len(radio_stations) * number_of_docs_per_pusher
verify_changes(djs,
expected_num_docs=expected_num_radio_docs,
expected_num_revisions=0,
expected_docs=radio_docs)
# Verify vjs get docs for all the channels associated with the tv_stations role
expected_num_tv_docs = len(tv_stations) * number_of_docs_per_pusher
verify_changes(vjs,
expected_num_docs=expected_num_tv_docs,
expected_num_revisions=0,
expected_docs=tv_docs)
# Verify mogul gets docs for all the channels associated with the radio_stations + tv_stations roles
all_docs_caches = list(radio_doc_caches)
all_docs_caches.extend(tv_doc_caches)
all_docs = {k: v for cache in all_docs_caches for k, v in cache.items()}
verify_changes(mogul,
expected_num_docs=expected_num_radio_docs +
expected_num_tv_docs,
expected_num_revisions=0,
expected_docs=all_docs)
def test_continuous_changes_parametrized(params_from_base_test_setup,
sg_conf_name, num_users, num_docs,
num_revisions):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'continuous_changes_parametrized'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
log_info("num_users: {}".format(num_users))
log_info("num_docs: {}".format(num_docs))
log_info("num_revisions: {}".format(num_revisions))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
admin = Admin(cluster.sync_gateways[0])
users = admin.register_bulk_users(target=cluster.sync_gateways[0],
db="db",
name_prefix="user",
number=num_users,
password="******",
channels=["ABC", "TERMINATE"])
abc_doc_pusher = admin.register_user(target=cluster.sync_gateways[0],
db="db",
name="abc_doc_pusher",
password="******",
channels=["ABC"])
doc_terminator = admin.register_user(target=cluster.sync_gateways[0],
db="db",
name="doc_terminator",
password="******",
channels=["TERMINATE"])
with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
futures = {
executor.submit(user.start_continuous_changes_tracking,
termination_doc_id="killcontinuous"): user.name
for user in users
}
futures[executor.submit(abc_doc_pusher.add_docs,
num_docs)] = "doc_pusher"
for future in concurrent.futures.as_completed(futures):
task_name = futures[future]
# Send termination doc to seth continuous changes feed subscriber
if task_name == "doc_pusher":
errors = future.result()
assert len(errors) == 0
abc_doc_pusher.update_docs(num_revs_per_doc=num_revisions)
time.sleep(10)
doc_terminator.add_doc("killcontinuous")
elif task_name.startswith("user"):
# When the user has continuous _changes feed closed, return the docs and verify the user got all the channel docs
docs_in_changes = future.result()
# Expect number of docs + the termination doc + _user doc
verify_same_docs(expected_num_docs=num_docs,
doc_dict_one=docs_in_changes,
doc_dict_two=abc_doc_pusher.cache)
# Expect number of docs + the termination doc
verify_changes(abc_doc_pusher,
expected_num_docs=num_docs,
expected_num_revisions=num_revisions,
expected_docs=abc_doc_pusher.cache)
def test_bucket_online_offline_resync_with_offline(params_from_base_test_setup,
sg_conf_name, num_users,
num_docs, num_revisions):
start = time.time()
cluster_conf = params_from_base_test_setup["cluster_config"]
test_mode = params_from_base_test_setup["mode"]
if test_mode == "di":
pytest.skip("Unsupported feature in distributed index")
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, test_mode)
log_info("Running 'test_bucket_online_offline_resync_with_online'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using num_users: {}".format(num_users))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using num_revisions: {}".format(num_revisions))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_conf)
init_completed = time.time()
log_info("Initialization completed. Time taken:{}s".format(init_completed -
start))
num_channels = 1
channels = ["channel-" + str(i) for i in range(num_channels)]
password = "******"
sgs = cluster.sync_gateways
admin = Admin(sgs[0])
# Register User
log_info("Register User")
user_objects = admin.register_bulk_users(target=sgs[0],
db="db",
name_prefix="User",
number=num_users,
password=password,
channels=channels)
user_x = admin.register_user(target=sgs[0],
db="db",
name="User-X",
password="******",
channels=["channel_x"])
# Add User
log_info("Add docs")
in_parallel(user_objects, 'add_docs', num_docs)
# Update docs
log_info("Update docs")
in_parallel(user_objects, 'update_docs', num_revisions)
time.sleep(10)
# Get changes for all users
in_parallel(user_objects, 'get_changes')
# every user should have same number of docs
# total/expected docs = num_users * num_docs
recieved_docs = in_parallel(user_objects, 'get_num_docs')
expected_docs = num_users * num_docs
for user_obj, docs in recieved_docs.items():
log_info('User {} got {} docs, expected docs: {}'.format(
user_obj.name, docs, expected_docs))
assert docs == expected_docs
# Verify that
# user created doc-ids exist in docs received in changes feed
# expected revision is equal to received revision
expected_revision = str(num_revisions + 1)
docs_rev_dict = in_parallel(user_objects, 'get_num_revisions')
rev_errors = []
for user_obj, docs_revision_dict in docs_rev_dict.items():
for doc_id in docs_revision_dict.keys():
rev = docs_revision_dict[doc_id]
log_info(
'User {} doc_id {} has {} revisions, expected revision: {}'.
format(user_obj.name, doc_id, rev, expected_revision))
if rev != expected_revision:
rev_errors.append(doc_id)
log_error(
'User {} doc_id {} got revision {}, expected revision {}'.
format(user_obj.name, doc_id, rev, expected_revision))
assert len(rev_errors) == 0
# Verify each User created docs are part of changes feed
output = in_parallel(user_objects, 'check_doc_ids_in_changes_feed')
assert True in output.values()
# Take "db" offline
sg_client = MobileRestClient()
status = sg_client.take_db_offline(cluster_conf=cluster_conf, db="db")
assert status == 0
sg_restart_config = sync_gateway_config_path_for_mode(
"bucket_online_offline/db_online_offline_access_restricted", test_mode)
restart_status = cluster.sync_gateways[0].restart(sg_restart_config)
assert restart_status == 0
log_info("Sleeping....")
time.sleep(10)
pool = ThreadPool(processes=1)
log_info("Restarted SG....")
time.sleep(5)
db_info = admin.get_db_info("db")
log_info("Status of db = {}".format(db_info["state"]))
assert db_info["state"] == "Offline"
try:
async_resync_result = pool.apply_async(admin.db_resync, ("db", ))
log_info("resync issued !!!!!!")
except Exception as e:
log_info("Catch resync exception: {}".format(e))
time.sleep(1)
resync_occured = False
for i in range(20):
db_info = admin.get_db_info("db")
log_info("Status of db = {}".format(db_info["state"]))
if db_info["state"] == "Resyncing":
resync_occured = True
log_info("Resync occured")
try:
status = admin.get_db_info(db="db")
log_info("Got db_info request status: {}".format(status))
except HTTPError as e:
log_info("status = {} exception = {}".format(
status, e.response.status_code))
assert False
else:
log_info("Got 200 ok for supported operation")
time.sleep(1)
if resync_occured:
break
time.sleep(10)
status = sg_client.bring_db_online(cluster_conf=cluster_conf, db="db")
log_info("online request issued !!!!! response status: {}".format(status))
time.sleep(5)
db_info = admin.get_db_info("db")
log_info("Status of db = {}".format(db_info["state"]))
assert db_info["state"] == "Online"
resync_result = async_resync_result.get()
log_info("resync_changes {}".format(resync_result))
log_info("expecting num_changes == num_docs {} * num_users {}".format(
num_docs, num_users))
assert resync_result['payload']['changes'] == num_docs * num_users
assert resync_result['status_code'] == 200
time.sleep(5)
global_cache = list()
for user in user_objects:
global_cache.append(user.cache)
all_docs = {
k: v
for user_cache in global_cache for k, v in user_cache.items()
}
verify_changes(user_x,
expected_num_docs=expected_docs,
expected_num_revisions=num_revisions,
expected_docs=all_docs)
end = time.time()
log_info("Test ended.")
log_info("Main test duration: {}".format(end - init_completed))
log_info("Test setup time: {}".format(init_completed - start))
log_info("Total Time taken: {}s".format(end - start))
def test_seq(params_from_base_test_setup, sg_conf_name, num_users, num_docs,
num_revisions):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running seq")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
log_info("num_users: {}".format(num_users))
log_info("num_docs: {}".format(num_docs))
log_info("num_revisions: {}".format(num_revisions))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
admin = Admin(cluster.sync_gateways[0])
# all users will share docs due to having the same channel
users = admin.register_bulk_users(target=cluster.sync_gateways[0],
db="db",
name_prefix="user",
number=num_users,
password="******",
channels=["ABC"])
for user in users:
user.add_docs(num_docs, bulk=True)
for user in users:
user.update_docs(num_revisions)
time.sleep(5)
user_0_changes = users[0].get_changes(since=0)
doc_seq = user_0_changes["results"][num_docs / 2]["seq"]
# https://github.com/couchbase/sync_gateway/issues/1475#issuecomment-172426052
# verify you can issue _changes with since=12313-0::1023.15
for user in users:
changes = user.get_changes(since=doc_seq)
log_info("Trying changes with since={}".format(doc_seq))
assert len(changes["results"]) > 0
second_to_last_doc_entry_seq = changes["results"][-2]["seq"]
last_doc_entry_seq = changes["results"][-1]["seq"]
log_info('Second to last doc "seq": {}'.format(
second_to_last_doc_entry_seq))
log_info('Last doc "seq": {}'.format(last_doc_entry_seq))
if mode == "di":
# Verify last "seq" follows the formate 12313-0, not 12313-0::1023.15
log_info('Verify that the last "seq" is a plain hashed value')
assert len(second_to_last_doc_entry_seq.split("::")) == 2
assert len(last_doc_entry_seq.split("::")) == 1
elif mode == "cc":
assert second_to_last_doc_entry_seq > 0
assert last_doc_entry_seq > 0
else:
raise ValueError("Unsupported 'mode' !!")
all_doc_caches = [user.cache for user in users]
all_docs = {k: v for cache in all_doc_caches for k, v in cache.items()}
verify_changes(users,
expected_num_docs=num_users * num_docs,
expected_num_revisions=num_revisions,
expected_docs=all_docs)
def test_db_online_offline_webhooks_offline(params_from_base_test_setup, sg_conf_name, num_users, num_channels, num_docs, num_revisions):
start = time.time()
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
if mode == "di":
pytest.skip("Offline tests not supported in Di mode -- see https://github.com/couchbase/sync_gateway/issues/2423#issuecomment-300841425")
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'test_db_online_offline_webhooks_offline'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using num_users: {}".format(num_users))
log_info("Using num_channels: {}".format(num_channels))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using num_revisions: {}".format(num_revisions))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_conf)
init_completed = time.time()
log_info("Initialization completed. Time taken:{}s".format(init_completed - start))
channels = ["channel-" + str(i) for i in range(num_channels)]
password = "******"
ws = WebServer()
ws.start()
sgs = cluster.sync_gateways
admin = Admin(sgs[0])
# Register User
log_info("Register User")
user_objects = admin.register_bulk_users(target=sgs[0], db="db", name_prefix="User",
number=num_users, password=password, channels=channels)
# Add User
log_info("Add docs")
in_parallel(user_objects, 'add_docs', num_docs)
# Update docs
log_info("Update docs")
in_parallel(user_objects, 'update_docs', num_revisions)
time.sleep(10)
# Take db offline
sg_client = MobileRestClient()
status = sg_client.take_db_offline(cluster_conf=cluster_conf, db="db")
assert status == 0
time.sleep(5)
db_info = admin.get_db_info("db")
log_info("Expecting db state {} found db state {}".format("Offline", db_info['state']))
assert db_info["state"] == "Offline"
webhook_events = ws.get_data()
time.sleep(5)
log_info("webhook event {}".format(webhook_events))
try:
last_event = webhook_events[-1]
assert last_event['state'] == 'offline'
# Bring db online
status = sg_client.bring_db_online(cluster_conf=cluster_conf, db="db")
assert status == 0
time.sleep(5)
db_info = admin.get_db_info("db")
log_info("Expecting db state {} found db state {}".format("Online", db_info['state']))
assert db_info["state"] == "Online"
time.sleep(5)
webhook_events = ws.get_data()
last_event = webhook_events[-1]
assert last_event['state'] == 'online'
time.sleep(10)
log_info("webhook event {}".format(webhook_events))
except IndexError:
log_info("Received index error")
raise
finally:
ws.stop()
def test_mulitple_users_mulitiple_channels_mulitple_revisions(
params_from_base_test_setup, sg_conf_name, num_users, num_channels,
num_docs, num_revisions):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'mulitple_users_mulitiple_channels_mulitple_revisions'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
log_info("num_users: {}".format(num_users))
log_info("num_channels: {}".format(num_channels))
log_info("num_docs: {}".format(num_docs))
log_info("num_revisions: {}".format(num_revisions))
start = time.time()
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
init_completed = time.time()
log_info("Initialization completed. Time taken:{}s".format(init_completed -
start))
channels = ["channel-" + str(i) for i in range(num_channels)]
password = "******"
sgs = cluster.sync_gateways
admin = Admin(sgs[0])
# Register User
log_info("Register User")
user_objects = admin.register_bulk_users(target=sgs[0],
db="db",
name_prefix="User",
number=num_users,
password=password,
channels=channels)
# Add User
log_info("Add docs")
in_parallel(user_objects, 'add_docs', num_docs)
# Update docs
log_info("Update docs")
in_parallel(user_objects, 'update_docs', num_revisions)
# Adding sleep to let sg to catch-up...
# Without sleep this test fails in Channel-Cache mode and changes feed doesn't return the expected
# num_revisions in docs.
# The test passes in Distributed-Index mode.
time.sleep(10)
# Get changes for all users
in_parallel(user_objects, 'get_changes')
# every user should have same number of docs
# total/expected docs = num_users * num_docs
recieved_docs = in_parallel(user_objects, 'get_num_docs')
expected_docs = num_users * num_docs
for user_obj, docs in recieved_docs.items():
log_info('User {} got {} docs, expected docs: {}'.format(
user_obj.name, docs, expected_docs))
assert docs == expected_docs
# Verify that
# user created doc-ids exist in docs received in changes feed
# expected revision is equal to received revision
expected_revision = str(num_revisions + 1)
docs_rev_dict = in_parallel(user_objects, 'get_num_revisions')
rev_errors = []
for user_obj, docs_revision_dict in docs_rev_dict.items():
for doc_id in docs_revision_dict.keys():
rev = docs_revision_dict[doc_id]
log_info(
'User {} doc_id {} has {} revisions, expected revision: {}'.
format(user_obj.name, doc_id, rev, expected_revision))
if rev != expected_revision:
rev_errors.append(doc_id)
log_error(
'User {} doc_id {} got revision {}, expected revision {}'.
format(user_obj.name, doc_id, rev, expected_revision))
assert len(rev_errors) == 0
# Verify each User created docs are part of changes feed
output = in_parallel(user_objects, 'check_doc_ids_in_changes_feed')
assert True in output.values()
end = time.time()
log_info("Test ended.")
log_info("Main test duration: {}".format(end - init_completed))
log_info("Test setup time: {}".format(init_completed - start))
log_info("Total Time taken: {}s".format(end - start))
def test_bucket_online_offline_resync_sanity(params_from_base_test_setup,
sg_conf_name, num_users, num_docs,
num_revisions):
cluster_conf = params_from_base_test_setup["cluster_config"]
test_mode = params_from_base_test_setup["mode"]
if test_mode == "di":
pytest.skip("Unsupported feature in distributed index")
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, test_mode)
log_info("Running 'test_bucket_online_offline_resync_sanity'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using num_users: {}".format(num_users))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using num_revisions: {}".format(num_revisions))
start = time.time()
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_conf)
init_completed = time.time()
log_info("Initialization completed. Time taken:{}s".format(init_completed -
start))
num_channels = 1
channels = ["channel-" + str(i) for i in range(num_channels)]
password = "******"
sgs = cluster.sync_gateways
admin = Admin(sgs[0])
# Register User
log_info("Register User")
user_objects = admin.register_bulk_users(target=sgs[0],
db="db",
name_prefix="User",
number=num_users,
password=password,
channels=channels)
user_x = admin.register_user(target=sgs[0],
db="db",
name="User-X",
password="******",
channels=["channel_x"])
# Add User
log_info("Add docs")
in_parallel(user_objects, 'add_docs', num_docs)
# Update docs
log_info("Update docs")
in_parallel(user_objects, 'update_docs', num_revisions)
time.sleep(10)
# Get changes for all users
in_parallel(user_objects, 'get_changes')
# every user should have same number of docs
# total/expected docs = num_users * num_docs
recieved_docs = in_parallel(user_objects, 'get_num_docs')
expected_docs = num_users * num_docs
for user_obj, docs in recieved_docs.items():
log_info('User {} got {} docs, expected docs: {}'.format(
user_obj.name, docs, expected_docs))
assert docs == expected_docs
# Verify that
# user created doc-ids exist in docs received in changes feed
# expected revision is equal to received revision
expected_revision = str(num_revisions + 1)
docs_rev_dict = in_parallel(user_objects, 'get_num_revisions')
rev_errors = []
for user_obj, docs_revision_dict in docs_rev_dict.items():
for doc_id in docs_revision_dict.keys():
rev = docs_revision_dict[doc_id]
log_info(
'User {} doc_id {} has {} revisions, expected revision: {}'.
format(user_obj.name, doc_id, rev, expected_revision))
if rev != expected_revision:
rev_errors.append(doc_id)
log_error(
'User {} doc_id {} got revision {}, expected revision {}'.
format(user_obj.name, doc_id, rev, expected_revision))
assert len(rev_errors) == 0
# Verify each User created docs are part of changes feed
output = in_parallel(user_objects, 'check_doc_ids_in_changes_feed')
assert True in output.values()
# Take "db" offline
sg_client = MobileRestClient()
status = sg_client.take_db_offline(cluster_conf=cluster_conf, db="db")
assert status == 0
sg_restart_config = sync_gateway_config_path_for_mode(
"bucket_online_offline/db_online_offline_access_restricted", test_mode)
restart_status = cluster.sync_gateways[0].restart(sg_restart_config)
assert restart_status == 0
time.sleep(10)
num_changes = admin.db_resync(db="db")
log_info("expecting num_changes {} == num_docs {} * num_users {}".format(
num_changes, num_docs, num_users))
assert num_changes['payload']['changes'] == num_docs * num_users
# Take "db" online
status = sg_client.bring_db_online(cluster_conf=cluster_conf, db="db")
assert status == 0
time.sleep(5)
global_cache = list()
for user in user_objects:
global_cache.append(user.cache)
all_docs = {
k: v
for user_cache in global_cache for k, v in user_cache.items()
}
verify_changes(user_x,
expected_num_docs=expected_docs,
expected_num_revisions=num_revisions,
expected_docs=all_docs)
end = time.time()
log_info("Test ended.")
log_info("Main test duration: {}".format(end - init_completed))
log_info("Test setup time: {}".format(init_completed - start))
log_info("Total Time taken: {}s".format(end - start))
def test_mulitple_users_mulitiple_channels_mulitple_revisions(
params_from_base_test_setup, sg_conf_name, num_users, num_channels, num_docs, num_revisions
):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'mulitple_users_mulitiple_channels_mulitple_revisions'")
log_info("cluster_conf: {}".format(cluster_conf))
log_info("sg_conf: {}".format(sg_conf))
log_info("num_users: {}".format(num_users))
log_info("num_channels: {}".format(num_channels))
log_info("num_docs: {}".format(num_docs))
log_info("num_revisions: {}".format(num_revisions))
start = time.time()
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
init_completed = time.time()
log_info("Initialization completed. Time taken:{}s".format(init_completed - start))
channels = ["channel-" + str(i) for i in range(num_channels)]
password = "******"
sgs = cluster.sync_gateways
admin = Admin(sgs[0])
# Register User
log_info("Register User")
user_objects = admin.register_bulk_users(
target=sgs[0], db="db", name_prefix="User", number=num_users, password=password, channels=channels
)
# Add User
log_info("Add docs")
in_parallel(user_objects, "add_docs", num_docs)
# Update docs
log_info("Update docs")
in_parallel(user_objects, "update_docs", num_revisions)
# Adding sleep to let sg to catch-up...
# Without sleep this test fails in Channel-Cache mode and changes feed doesn't return the expected
# num_revisions in docs.
# The test passes in Distributed-Index mode.
time.sleep(10)
# Get changes for all users
in_parallel(user_objects, "get_changes")
# every user should have same number of docs
# total/expected docs = num_users * num_docs
recieved_docs = in_parallel(user_objects, "get_num_docs")
expected_docs = num_users * num_docs
for user_obj, docs in recieved_docs.items():
log_info("User {} got {} docs, expected docs: {}".format(user_obj.name, docs, expected_docs))
assert docs == expected_docs
# Verify that
# user created doc-ids exist in docs received in changes feed
# expected revision is equal to received revision
expected_revision = str(num_revisions + 1)
docs_rev_dict = in_parallel(user_objects, "get_num_revisions")
rev_errors = []
for user_obj, docs_revision_dict in docs_rev_dict.items():
for doc_id in docs_revision_dict.keys():
rev = docs_revision_dict[doc_id]
log_info(
"User {} doc_id {} has {} revisions, expected revision: {}".format(
user_obj.name, doc_id, rev, expected_revision
)
)
if rev != expected_revision:
rev_errors.append(doc_id)
log_error(
"User {} doc_id {} got revision {}, expected revision {}".format(
user_obj.name, doc_id, rev, expected_revision
)
)
assert len(rev_errors) == 0
# Verify each User created docs are part of changes feed
output = in_parallel(user_objects, "check_doc_ids_in_changes_feed")
assert True in output.values()
end = time.time()
log_info("Test ended.")
log_info("Main test duration: {}".format(end - init_completed))
log_info("Test setup time: {}".format(init_completed - start))
log_info("Total Time taken: {}s".format(end - start))
def test_webhooks(params_from_base_test_setup, sg_conf_name, num_users,
num_channels, num_docs, num_revisions):
"""
Scenario:
- Start a webserver on machine running the test to recieved webhook events
- Create users
- Add docs to Sync Gateway
- Update docs on Sync Gateway
- Verify the webserver recieved all expected webhook events
"""
start = time.time()
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'test_webhooks'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using num_users: {}".format(num_users))
log_info("Using num_channels: {}".format(num_channels))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using num_revisions: {}".format(num_revisions))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_conf)
init_completed = time.time()
log_info("Initialization completed. Time taken:{}s".format(init_completed -
start))
channels = ["channel-" + str(i) for i in range(num_channels)]
password = "******"
ws = WebServer()
ws.start()
sgs = cluster.sync_gateways
admin = Admin(sgs[0])
# Register User
log_info("Register User")
user_objects = admin.register_bulk_users(target=sgs[0],
db="db",
name_prefix="User",
number=num_users,
password=password,
channels=channels)
# Add User
log_info("Add docs")
in_parallel(user_objects, 'add_docs', num_docs)
# Update docs
log_info("Update docs")
in_parallel(user_objects, 'update_docs', num_revisions)
time.sleep(30)
expected_events = (num_users * num_docs * num_revisions) + (num_users *
num_docs)
received_events = ws.get_data()
received_doc_events = []
for ev in received_events:
if "_id" in ev:
received_doc_events.append(ev)
log_info("expected_events: {} received_events {}".format(
expected_events, received_events))
# Stop ws before asserting
# Else successive tests will fail to start ws
ws.stop()
assert expected_events == len(received_doc_events)
def test_overloaded_channel_cache(params_from_base_test_setup, sg_conf_name, num_docs, user_channels, filter, limit):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
if mode == "di":
pytest.skip("Unsupported feature in distributed index")
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'test_overloaded_channel_cache'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using user_channels: {}".format(user_channels))
log_info("Using filter: {}".format(filter))
log_info("Using limit: {}".format(limit))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
target_sg = cluster.sync_gateways[0]
admin = Admin(target_sg)
users = admin.register_bulk_users(target_sg, "db", "user", 1000, "password", [user_channels])
assert len(users) == 1000
doc_pusher = admin.register_user(target_sg, "db", "abc_doc_pusher", "password", ["ABC"])
doc_pusher.add_docs(num_docs, bulk=True)
# Give a few seconds to let changes register
time.sleep(2)
start = time.time()
with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
changes_requests = []
errors = []
for user in users:
if filter and limit is not None:
changes_requests.append(executor.submit(user.get_changes, since=0, limit=limit, filter="sync_gateway/bychannel", channels=["ABC"]))
elif filter and limit is None:
changes_requests.append(executor.submit(user.get_changes, filter="sync_gateway/bychannel", channels=["ABC"]))
elif not filter and limit is not None:
changes_requests.append(executor.submit(user.get_changes, limit=limit))
elif not filter and limit is None:
changes_requests.append(executor.submit(user.get_changes))
for future in concurrent.futures.as_completed(changes_requests):
changes = future.result()
if limit is not None:
assert len(changes["results"]) == 50
else:
assert len(changes["results"]) == 5001
# changes feed should all be successful
log_info(len(errors))
assert len(errors) == 0
if limit is not None:
# HACK: Should be less than a minute unless blocking on view calls
end = time.time()
time_for_users_to_get_all_changes = end - start
log_info("Time for users to get all changes: {}".format(time_for_users_to_get_all_changes))
assert time_for_users_to_get_all_changes < 120, "Time to get all changes was greater than a minute: {}s".format(
time_for_users_to_get_all_changes
)
# Sanity check that a subset of users have _changes feed intact
for i in range(10):
verify_changes(users[i], expected_num_docs=num_docs, expected_num_revisions=0, expected_docs=doc_pusher.cache)
# Get sync_gateway expvars
resp = requests.get(url="http://{}:4985/_expvar".format(target_sg.ip))
resp.raise_for_status()
resp_obj = resp.json()
if user_channels == "*" and num_docs == 5000:
# "*" channel includes _user docs so the verify_changes will result in 10 view queries
assert resp_obj["syncGateway_changeCache"]["view_queries"] == 10
else:
# If number of view queries == 0 the key will not exist in the expvars
assert "view_queries" not in resp_obj["syncGateway_changeCache"]
def test_sync_require_roles(params_from_base_test_setup, sg_conf_name):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'sync_require_roles'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
radio_stations = ["KMOW", "HWOD", "KDWB"]
tv_stations = ["ABC", "CBS", "NBC"]
number_of_djs = 10
number_of_vjs = 10
number_of_docs_per_pusher = 100
admin = Admin(cluster.sync_gateways[0])
admin.create_role("db", name="radio_stations", channels=radio_stations)
admin.create_role("db", name="tv_stations", channels=tv_stations)
djs = admin.register_bulk_users(target=cluster.sync_gateways[0],
db="db",
name_prefix="dj",
number=number_of_djs,
password="******",
roles=["radio_stations"])
vjs = admin.register_bulk_users(target=cluster.sync_gateways[0],
db="db",
name_prefix="vj",
number=number_of_vjs,
password="******",
roles=["tv_stations"])
mogul = admin.register_user(target=cluster.sync_gateways[0],
db="db",
name="mogul",
password="******",
roles=["tv_stations", "radio_stations"])
radio_doc_caches = []
for radio_station in radio_stations:
doc_pusher = admin.register_user(
target=cluster.sync_gateways[0],
db="db",
name="{}_doc_pusher".format(radio_station),
password="******",
channels=[radio_station],
roles=["radio_stations"])
doc_pusher.add_docs(number_of_docs_per_pusher, bulk=True)
radio_doc_caches.append(doc_pusher.cache)
expected_num_radio_docs = len(radio_stations) * number_of_docs_per_pusher
# All docs that have been pushed with the "radio_stations" role
all_radio_docs = {
k: v
for cache in radio_doc_caches for k, v in cache.items()
}
tv_doc_caches = []
for tv_station in tv_stations:
doc_pusher = admin.register_user(
target=cluster.sync_gateways[0],
db="db",
name="{}_doc_pusher".format(tv_station),
password="******",
channels=[tv_station],
roles=["tv_stations"])
doc_pusher.add_docs(number_of_docs_per_pusher, bulk=True)
tv_doc_caches.append(doc_pusher.cache)
expected_num_tv_docs = len(tv_stations) * number_of_docs_per_pusher
# All docs that have been pushed with the "tv_stations" role
all_tv_docs = {k: v for cache in tv_doc_caches for k, v in cache.items()}
# Read only users
radio_channels_no_roles_user = admin.register_user(
target=cluster.sync_gateways[0],
db="db",
name="bad_radio_user",
password="******",
channels=radio_stations)
tv_channel_no_roles_user = admin.register_user(
target=cluster.sync_gateways[0],
db="db",
name="bad_tv_user",
password="******",
channels=tv_stations)
# Should not be allowed
radio_channels_no_roles_user.add_docs(13, name_prefix="bad_doc")
tv_channel_no_roles_user.add_docs(26, name_prefix="bad_doc")
read_only_user_caches = [
radio_channels_no_roles_user.cache, tv_channel_no_roles_user.cache
]
read_only_user_docs = {
k: v
for cache in read_only_user_caches for k, v in cache.items()
}
# Dictionary should be empty if they were blocked from pushing docs
assert len(read_only_user_docs.items()) == 0
# It seems be non deterministic but sometimes when issuing the changes call return, some of the documents are returned but not all.
# There is currently no retry loop in verify_changes and I'm guessing that the bulk_docs requests are still processing.
time.sleep(5)
# Should recieve docs from radio_channels
verify_changes(radio_channels_no_roles_user,
expected_num_docs=expected_num_radio_docs,
expected_num_revisions=0,
expected_docs=all_radio_docs)
# Should recieve docs from tv_channels
verify_changes(tv_channel_no_roles_user,
expected_num_docs=expected_num_tv_docs,
expected_num_revisions=0,
expected_docs=all_tv_docs)
# verify all djs with the 'radio_stations' role get the docs with radio station channels
verify_changes(djs,
expected_num_docs=expected_num_radio_docs,
expected_num_revisions=0,
expected_docs=all_radio_docs)
# verify all djs with the 'radio_stations' role get the docs with radio station channels
verify_changes(vjs,
expected_num_docs=expected_num_tv_docs,
expected_num_revisions=0,
expected_docs=all_tv_docs)
# Verify mogul gets docs for all the channels associated with the radio_stations + tv_stations roles
all_doc_caches = list(radio_doc_caches)
all_doc_caches.extend(tv_doc_caches)
all_docs = {k: v for cache in all_doc_caches for k, v in cache.items()}
for k, v in all_docs.items():
assert not k.startswith("bad_doc")
verify_changes(mogul,
expected_num_docs=expected_num_radio_docs +
expected_num_tv_docs,
expected_num_revisions=0,
expected_docs=all_docs)
def test_overloaded_channel_cache(params_from_base_test_setup, sg_conf_name, num_docs, user_channels, filter, limit):
"""
The purpose of this test is to verify that channel cache backfill via view queries is working properly.
It works by doing the following:
- Set channel cache size in Sync Gateway config to a small number, eg, 750. This means that only 750 docs fit in the channel cache
- Add a large number of docs, eg, 1000.
- Issue a _changes request that will return all 1000 docs
Expected behavior / Verification:
- Since 1000 docs requested from changes feed, but only 750 docs fit in channel cache, then it will need to do a view query
to get the remaining 250 changes
- Verify that the changes feed returns all 1000 expected docs
- Check the expvar statistics to verify that view queries were made
"""
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
if mode == "di":
pytest.skip("Unsupported feature in distributed index")
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'test_overloaded_channel_cache'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using user_channels: {}".format(user_channels))
log_info("Using filter: {}".format(filter))
log_info("Using limit: {}".format(limit))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
target_sg = cluster.sync_gateways[0]
admin = Admin(target_sg)
users = admin.register_bulk_users(target_sg, "db", "user", 1000, "password", [user_channels])
assert len(users) == 1000
doc_pusher = admin.register_user(target_sg, "db", "abc_doc_pusher", "password", ["ABC"])
doc_pusher.add_docs(num_docs, bulk=True)
# Give a few seconds to let changes register
time.sleep(2)
start = time.time()
# This uses a ProcessPoolExecutor due to https://github.com/couchbaselabs/mobile-testkit/issues/1142
with concurrent.futures.ProcessPoolExecutor(max_workers=100) as executor:
changes_requests = []
errors = []
for user in users:
if filter and limit is not None:
changes_requests.append(executor.submit(user.get_changes, since=0, limit=limit, filter="sync_gateway/bychannel", channels=["ABC"]))
elif filter and limit is None:
changes_requests.append(executor.submit(user.get_changes, filter="sync_gateway/bychannel", channels=["ABC"]))
elif not filter and limit is not None:
changes_requests.append(executor.submit(user.get_changes, limit=limit))
elif not filter and limit is None:
changes_requests.append(executor.submit(user.get_changes))
for future in concurrent.futures.as_completed(changes_requests):
changes = future.result()
if limit is not None:
assert len(changes["results"]) == 50
else:
assert len(changes["results"]) == 5001
# changes feed should all be successful
log_info(len(errors))
assert len(errors) == 0
if limit is not None:
# HACK: Should be less than a minute unless blocking on view calls
end = time.time()
time_for_users_to_get_all_changes = end - start
log_info("Time for users to get all changes: {}".format(time_for_users_to_get_all_changes))
assert time_for_users_to_get_all_changes < 240, "Time to get all changes was greater than 2 minutes: {}s".format(
time_for_users_to_get_all_changes
)
# Sanity check that a subset of users have _changes feed intact
for i in range(10):
verify_changes(users[i], expected_num_docs=num_docs, expected_num_revisions=0, expected_docs=doc_pusher.cache)
# Get sync_gateway expvars
resp = requests.get(url="http://{}:4985/_expvar".format(target_sg.ip))
resp.raise_for_status()
resp_obj = resp.json()
# Since Sync Gateway will need to issue view queries to handle _changes requests that don't
# fit in the channel cache, we expect there to be several view queries
assert resp_obj["syncGateway_changeCache"]["view_queries"] > 0
def test_sync_require_roles(params_from_base_test_setup, sg_conf_name):
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'sync_require_roles'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using sg_conf: {}".format(sg_conf))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
radio_stations = ["KMOW", "HWOD", "KDWB"]
tv_stations = ["ABC", "CBS", "NBC"]
number_of_djs = 10
number_of_vjs = 10
number_of_docs_per_pusher = 100
admin = Admin(cluster.sync_gateways[0])
admin.create_role("db", name="radio_stations", channels=radio_stations)
admin.create_role("db", name="tv_stations", channels=tv_stations)
djs = admin.register_bulk_users(target=cluster.sync_gateways[0], db="db", name_prefix="dj", number=number_of_djs, password="******", roles=["radio_stations"])
vjs = admin.register_bulk_users(target=cluster.sync_gateways[0], db="db", name_prefix="vj", number=number_of_vjs, password="******", roles=["tv_stations"])
mogul = admin.register_user(target=cluster.sync_gateways[0], db="db", name="mogul", password="******", roles=["tv_stations", "radio_stations"])
radio_doc_caches = []
for radio_station in radio_stations:
doc_pusher = admin.register_user(target=cluster.sync_gateways[0], db="db", name="{}_doc_pusher".format(radio_station), password="******", channels=[radio_station], roles=["radio_stations"])
doc_pusher.add_docs(number_of_docs_per_pusher, bulk=True)
radio_doc_caches.append(doc_pusher.cache)
expected_num_radio_docs = len(radio_stations) * number_of_docs_per_pusher
# All docs that have been pushed with the "radio_stations" role
all_radio_docs = {k: v for cache in radio_doc_caches for k, v in cache.items()}
tv_doc_caches = []
for tv_station in tv_stations:
doc_pusher = admin.register_user(target=cluster.sync_gateways[0], db="db", name="{}_doc_pusher".format(tv_station), password="******", channels=[tv_station], roles=["tv_stations"])
doc_pusher.add_docs(number_of_docs_per_pusher, bulk=True)
tv_doc_caches.append(doc_pusher.cache)
expected_num_tv_docs = len(tv_stations) * number_of_docs_per_pusher
# All docs that have been pushed with the "tv_stations" role
all_tv_docs = {k: v for cache in tv_doc_caches for k, v in cache.items()}
# Read only users
radio_channels_no_roles_user = admin.register_user(target=cluster.sync_gateways[0], db="db", name="bad_radio_user", password="******", channels=radio_stations)
tv_channel_no_roles_user = admin.register_user(target=cluster.sync_gateways[0], db="db", name="bad_tv_user", password="******", channels=tv_stations)
# Should not be allowed
radio_channels_no_roles_user.add_docs(13, name_prefix="bad_doc")
tv_channel_no_roles_user.add_docs(26, name_prefix="bad_doc")
read_only_user_caches = [radio_channels_no_roles_user.cache, tv_channel_no_roles_user.cache]
read_only_user_docs = {k: v for cache in read_only_user_caches for k, v in cache.items()}
# Dictionary should be empty if they were blocked from pushing docs
assert len(read_only_user_docs.items()) == 0
# It seems be non deterministic but sometimes when issuing the changes call return, some of the documents are returned but not all.
# There is currently no retry loop in verify_changes and I'm guessing that the bulk_docs requests are still processing.
time.sleep(5)
# Should recieve docs from radio_channels
verify_changes(radio_channels_no_roles_user, expected_num_docs=expected_num_radio_docs, expected_num_revisions=0, expected_docs=all_radio_docs)
# Should recieve docs from tv_channels
verify_changes(tv_channel_no_roles_user, expected_num_docs=expected_num_tv_docs, expected_num_revisions=0, expected_docs=all_tv_docs)
# verify all djs with the 'radio_stations' role get the docs with radio station channels
verify_changes(djs, expected_num_docs=expected_num_radio_docs, expected_num_revisions=0, expected_docs=all_radio_docs)
# verify all djs with the 'radio_stations' role get the docs with radio station channels
verify_changes(vjs, expected_num_docs=expected_num_tv_docs, expected_num_revisions=0, expected_docs=all_tv_docs)
# Verify mogul gets docs for all the channels associated with the radio_stations + tv_stations roles
all_doc_caches = list(radio_doc_caches)
all_doc_caches.extend(tv_doc_caches)
all_docs = {k: v for cache in all_doc_caches for k, v in cache.items()}
for k, v in all_docs.items():
assert not k.startswith("bad_doc")
verify_changes(mogul, expected_num_docs=expected_num_radio_docs + expected_num_tv_docs, expected_num_revisions=0, expected_docs=all_docs)
