def test_db_online_offline_webhooks_offline_two(params_from_base_test_setup, sg_conf_name, num_users, num_channels, num_docs, num_revisions):
start = time.time()
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'test_db_online_offline_webhooks_offline_two'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using num_users: {}".format(num_users))
log_info("Using num_channels: {}".format(num_channels))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using num_revisions: {}".format(num_revisions))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_conf)
init_completed = time.time()
log_info("Initialization completed. Time taken:{}s".format(init_completed - start))
channels = ["channel-" + str(i) for i in range(num_channels)]
password = "******"
ws = WebServer()
ws.start()
sgs = cluster.sync_gateways
admin = Admin(sgs[0])
# Register User
log_info("Register User")
user_objects = admin.register_bulk_users(target=sgs[0], db="db", name_prefix="User",
number=num_users, password=password, channels=channels)
# Add User
log_info("Add docs")
in_parallel(user_objects, 'add_docs', num_docs)
# Update docs
log_info("Update docs")
in_parallel(user_objects, 'update_docs', num_revisions)
time.sleep(10)
status = cluster.servers[0].delete_bucket("data-bucket")
assert status == 0
log_info("Sleeping for 120 seconds...")
time.sleep(120)
webhook_events = ws.get_data()
time.sleep(5)
log_info("webhook event {}".format(webhook_events))
last_event = webhook_events[-1]
assert last_event['state'] == 'offline'
ws.stop()
def test_webhooks(params_from_base_test_setup, sg_conf_name, num_users, num_channels, num_docs, num_revisions):
start = time.time()
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'test_webhooks'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using num_users: {}".format(num_users))
log_info("Using num_channels: {}".format(num_channels))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using num_revisions: {}".format(num_revisions))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_conf)
init_completed = time.time()
log_info("Initialization completed. Time taken:{}s".format(init_completed - start))
channels = ["channel-" + str(i) for i in range(num_channels)]
password = "******"
ws = WebServer()
ws.start()
sgs = cluster.sync_gateways
admin = Admin(sgs[0])
# Register User
log_info("Register User")
user_objects = admin.register_bulk_users(target=sgs[0], db="db", name_prefix="User",
number=num_users, password=password, channels=channels)
# Add User
log_info("Add docs")
in_parallel(user_objects, 'add_docs', num_docs)
# Update docs
log_info("Update docs")
in_parallel(user_objects, 'update_docs', num_revisions)
time.sleep(30)
ws.stop()
expected_events = (num_users * num_docs * num_revisions) + (num_users * num_docs)
received_events = len(ws.get_data())
log_info("expected_events: {} received_events {}".format(expected_events, received_events))
assert expected_events == received_events
def test_db_online_offline_webhooks_offline(params_from_base_test_setup, sg_conf_name, num_users, num_channels, num_docs, num_revisions):
start = time.time()
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
if mode == "di":
pytest.skip("Offline tests not supported in Di mode -- see https://github.com/couchbase/sync_gateway/issues/2423#issuecomment-300841425")
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'test_db_online_offline_webhooks_offline'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using num_users: {}".format(num_users))
log_info("Using num_channels: {}".format(num_channels))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using num_revisions: {}".format(num_revisions))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_conf)
init_completed = time.time()
log_info("Initialization completed. Time taken:{}s".format(init_completed - start))
channels = ["channel-" + str(i) for i in range(num_channels)]
password = "******"
ws = WebServer()
ws.start()
sgs = cluster.sync_gateways
admin = Admin(sgs[0])
# Register User
log_info("Register User")
user_objects = admin.register_bulk_users(target=sgs[0], db="db", name_prefix="User",
number=num_users, password=password, channels=channels)
# Add User
log_info("Add docs")
in_parallel(user_objects, 'add_docs', num_docs)
# Update docs
log_info("Update docs")
in_parallel(user_objects, 'update_docs', num_revisions)
time.sleep(10)
# Take db offline
sg_client = MobileRestClient()
status = sg_client.take_db_offline(cluster_conf=cluster_conf, db="db")
assert status == 0
time.sleep(5)
db_info = admin.get_db_info("db")
log_info("Expecting db state {} found db state {}".format("Offline", db_info['state']))
assert db_info["state"] == "Offline"
webhook_events = ws.get_data()
time.sleep(5)
log_info("webhook event {}".format(webhook_events))
try:
last_event = webhook_events[-1]
assert last_event['state'] == 'offline'
# Bring db online
status = sg_client.bring_db_online(cluster_conf=cluster_conf, db="db")
assert status == 0
time.sleep(5)
db_info = admin.get_db_info("db")
log_info("Expecting db state {} found db state {}".format("Online", db_info['state']))
assert db_info["state"] == "Online"
time.sleep(5)
webhook_events = ws.get_data()
last_event = webhook_events[-1]
assert last_event['state'] == 'online'
time.sleep(10)
log_info("webhook event {}".format(webhook_events))
except IndexError:
log_info("Received index error")
raise
finally:
ws.stop()
def test_webhooks(params_from_base_test_setup, sg_conf_name, num_users,
num_channels, num_docs, num_revisions):
"""
Scenario:
- Start a webserver on machine running the test to recieved webhook events
- Create users
- Add docs to Sync Gateway
- Update docs on Sync Gateway
- Verify the webserver recieved all expected webhook events
"""
start = time.time()
cluster_conf = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
log_info("Running 'test_webhooks'")
log_info("Using cluster_conf: {}".format(cluster_conf))
log_info("Using num_users: {}".format(num_users))
log_info("Using num_channels: {}".format(num_channels))
log_info("Using num_docs: {}".format(num_docs))
log_info("Using num_revisions: {}".format(num_revisions))
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_conf)
init_completed = time.time()
log_info("Initialization completed. Time taken:{}s".format(init_completed -
start))
channels = ["channel-" + str(i) for i in range(num_channels)]
password = "******"
ws = WebServer()
ws.start()
sgs = cluster.sync_gateways
admin = Admin(sgs[0])
# Register User
log_info("Register User")
user_objects = admin.register_bulk_users(target=sgs[0],
db="db",
name_prefix="User",
number=num_users,
password=password,
channels=channels)
# Add User
log_info("Add docs")
in_parallel(user_objects, 'add_docs', num_docs)
# Update docs
log_info("Update docs")
in_parallel(user_objects, 'update_docs', num_revisions)
time.sleep(30)
expected_events = (num_users * num_docs * num_revisions) + (num_users *
num_docs)
received_events = ws.get_data()
received_doc_events = []
for ev in received_events:
if "_id" in ev:
received_doc_events.append(ev)
log_info("expected_events: {} received_events {}".format(
expected_events, received_events))
# Stop ws before asserting
# Else successive tests will fail to start ws
ws.stop()
assert expected_events == len(received_doc_events)
def test_webhooks_crud(params_from_base_test_setup, sg_conf_name, filtered):
""" Tests for webhook notification on import
xattr mode
1. Start sync gateway with autoimport
1. Write 'num_docs_per_client' docs via SDK
1. Write 'num_docs_per_client' docs via SG
1. Verify 'num_docs_per_client' * 2 webhook events (id, rev, body)
1. Update SG docs once each via SDK
1. Update SDK docs once each via SG
1. Verify 'num_docs_per_client' * 2 webhook events (id, rev, body)
1. Delete SG docs via SDK
1. Delete SDK docs via SG
1. Verify 'num_docs_per_client' * 2 webhook events (id, rev, body)
to verify no dups, wait 10s after recieveing expected webhooks
docmeta mode
1. Write 'num_docs_per_client' docs via SG
1. Verify 'num_docs_per_client' webhook events (id, rev, body)
1. Update SG docs once each via SG
1. Verify 'num_docs_per_client' webhook events (id, rev, body)
1. Delete SG docs via SG
1. Verify 'num_docs_per_client' webhook events (id, rev, body)
if filtered, the scenario will add a filtered propery to every other doc.
The webhook validation will only look for the filtered docs
"""
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
log_info('Webhooks filtered?: {}'.format(filtered))
cluster_conf = params_from_base_test_setup['cluster_config']
cluster_topology = params_from_base_test_setup['cluster_topology']
mode = params_from_base_test_setup['mode']
sg_admin_url = cluster_topology['sync_gateways'][0]['admin']
sg_url = cluster_topology['sync_gateways'][0]['public']
cbs_url = cluster_topology['couchbase_servers'][0]
sg_db = 'db'
bucket_name = 'data-bucket'
num_docs_per_client = 100
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_conf)
# Start webhook server on test runner
webhook_server = WebServer()
webhook_server.start()
sg_client = MobileRestClient()
cbs_ip = host_for_url(cbs_url)
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name),
password='******')
sg_info = UserInfo('sg_user', 'pass', channels=['shared'], roles=[])
sdk_info = UserInfo('sdk_user', 'pass', channels=['shared'], roles=[])
sg_client.create_user(url=sg_admin_url,
db=sg_db,
name=sg_info.name,
password=sg_info.password,
channels=sg_info.channels)
sg_auth = sg_client.create_session(url=sg_admin_url,
db=sg_db,
name=sg_info.name,
password=sg_info.password)
# Create sg docs
doc_content = {'aphex': 'twin'}
sg_docs = document.create_docs(doc_id_prefix='sg_user_doc',
number=num_docs_per_client,
content=doc_content,
channels=sg_info.channels)
# Add filtered property to every other doc
count = 0
for sg_doc in sg_docs:
if count % 2 == 0:
sg_doc['filtered'] = True
count += 1
sg_doc_ids = [doc['_id'] for doc in sg_docs]
sg_filtered_doc_ids = [doc['_id'] for doc in sg_docs if 'filtered' in doc]
assert len(sg_doc_ids) == num_docs_per_client
assert len(sg_filtered_doc_ids) == num_docs_per_client / 2
# Create sdk docs
sdk_docs = {
'sdk_user_doc_{}'.format(i): {
'channels': sdk_info.channels,
'content': doc_content
}
for i in range(num_docs_per_client)
}
# Add filtered property to every other doc
count = 0
for _, doc_val in sdk_docs.items():
if count % 2 == 0:
doc_val['filtered'] = True
count += 1
sdk_doc_ids = [doc for doc in sdk_docs]
sdk_filtered_doc_ids = [k for k, v in sdk_docs.items() if 'filtered' in v]
assert len(sdk_doc_ids) == num_docs_per_client
assert len(sdk_filtered_doc_ids) == num_docs_per_client / 2
all_docs = sg_doc_ids + sdk_doc_ids
all_filtered_docs = sg_filtered_doc_ids + sdk_filtered_doc_ids
assert len(all_docs) == num_docs_per_client * 2
# If xattr mode, add sg + sdk docs
# If non xattr mode, add sg docs
add_docs(sg_client=sg_client,
sg_url=sg_url,
sg_db=sg_db,
sg_docs=sg_docs,
sg_auth=sg_auth,
sdk_client=sdk_client,
sdk_docs=sdk_docs,
num_docs_per_client=num_docs_per_client,
xattrs=xattrs_enabled)
# Wait for added docs to trigger webhooks
if xattrs_enabled and filtered:
poll_for_webhook_data(webhook_server, all_filtered_docs, 1,
doc_content)
elif xattrs_enabled and not filtered:
poll_for_webhook_data(webhook_server, all_docs, 1, doc_content)
elif not xattrs_enabled and filtered:
poll_for_webhook_data(webhook_server, sg_filtered_doc_ids, 1,
doc_content)
else:
poll_for_webhook_data(webhook_server, sg_doc_ids, 1, doc_content)
webhook_server.clear_data()
# Update sdk docs from sg
# If xattr mode, update sdk docs from sg, update sg docs from SDK
# If non xattr mode, update sg docs from sg
updated_doc_content = {'brian': 'eno'}
update_docs(sg_client=sg_client,
sg_url=sg_url,
sg_db=sg_db,
sg_doc_ids=sg_doc_ids,
sg_auth=sg_auth,
sdk_client=sdk_client,
sdk_doc_ids=sdk_doc_ids,
updated_doc_content=updated_doc_content,
xattrs=xattrs_enabled)
# Wait for updates to trigger webhooks
if xattrs_enabled and filtered:
poll_for_webhook_data(webhook_server, all_filtered_docs, 2,
updated_doc_content)
elif xattrs_enabled and not filtered:
poll_for_webhook_data(webhook_server, all_docs, 2, updated_doc_content)
elif not xattrs_enabled and filtered:
poll_for_webhook_data(webhook_server, sg_filtered_doc_ids, 2,
updated_doc_content)
else:
poll_for_webhook_data(webhook_server, sg_doc_ids, 2,
updated_doc_content)
webhook_server.clear_data()
delete_docs(sg_client=sg_client,
sg_url=sg_url,
sg_db=sg_db,
sg_doc_ids=sg_doc_ids,
sg_auth=sg_auth,
sdk_client=sdk_client,
sdk_doc_ids=sdk_doc_ids,
xattrs=xattrs_enabled)
# Wait for deletes to trigger webhook events, filter includes all deleted docs
if xattrs_enabled:
poll_for_webhook_data(webhook_server,
all_docs,
3,
updated_doc_content,
deleted=True)
else:
poll_for_webhook_data(webhook_server,
sg_doc_ids,
3,
updated_doc_content,
deleted=True)
webhook_server.clear_data()
# Stop webhook server
webhook_server.stop()