def test_setting_expiry_in_bulk_docs(params_from_base_test_setup, sg_conf_name):
"""
1. PUT /db/_bulk_docs with 10 documents. Set the "_exp":3 on 5 of these documents
2. Wait five seconds
3. POST /db/_bulk_get for the 10 documents. Validate that only the 5 non-expiring documents are returned
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(
cluster_config=cluster_config,
sync_gateway_config=sg_conf
)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_setting_expiry_in_bulk_docs'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
client = MobileRestClient()
client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name)
doc_exp_3_bodies = document.create_docs(doc_id_prefix="exp_3", number=5, expiry=3, channels=sg_user_channels)
doc_exp_10_bodies = document.create_docs(doc_id_prefix="exp_10", number=5, expiry=10, channels=sg_user_channels)
bulk_bodies = doc_exp_3_bodies + doc_exp_10_bodies
bulk_docs = client.add_bulk_docs(url=sg_url, db=sg_db, docs=bulk_bodies, auth=sg_user_session)
# Allow exp_3 docs to expire
time.sleep(5)
bulk_get_docs = client.get_bulk_docs(url=sg_url, db=sg_db, docs=bulk_docs, auth=sg_user_session)
expected_ids = ["exp_10_0", "exp_10_1", "exp_10_2", "exp_10_3", "exp_10_4"]
expected_missing_ids = ["exp_3_0", "exp_3_1", "exp_3_2", "exp_3_3", "exp_3_4"]
client.verify_doc_ids_found_in_response(response=bulk_get_docs, expected_doc_ids=expected_ids)
client.verify_doc_ids_not_found_in_response(response=bulk_get_docs, expected_missing_doc_ids=expected_missing_ids)
def verify_sg_docs_revision_history(url, db, added_docs):
sg_client = MobileRestClient()
expected_doc_map = {
added_doc["id"]: added_doc["rev"]
for added_doc in added_docs
}
doc_ids = expected_doc_map.keys()
log_info("Bulk getting docs from sync gateway")
docs = sg_client.get_bulk_docs(url, db, doc_ids, rev_history="true")
assert len(docs[0]) == len(doc_ids)
for doc in docs:
for doc_dict in doc:
rev = doc_dict["_rev"]
rev_gen = int(rev.split("-")[0])
doc_id = doc_dict["_id"]
# Verify meta data
log_info("Verifying that doc {} has rev {}".format(
doc_id, expected_doc_map[doc_id]))
assert rev == expected_doc_map[doc_id]
log_info(
"Doc {}: Expected number of revs: {}, Actual revs: {}".format(
doc_id, rev_gen, len(doc_dict["_revisions"]["ids"])))
assert len(doc_dict["_revisions"]["ids"]) == rev_gen
log_info(
"Verifying that doc {} is associated with sg_user_channel channel"
.format(doc_id))
assert doc_dict["channels"][0] == "sg_user_channel"
# Verify doc body
log_info("Verifying doc body for {}".format(doc_id))
assert "guid" in doc_dict
assert "index" in doc_dict
assert "latitude" in doc_dict
assert "email" in doc_dict
assert "picture" in doc_dict
assert len(doc_dict["tags"]) == 3
assert "date_time_added" in doc_dict
assert "company" in doc_dict
assert "eyeColor" in doc_dict
assert "phone" in doc_dict
assert "updates" in doc_dict
assert "address" in doc_dict
assert len(doc_dict["friends"]) == 2
assert "isActive" in doc_dict
assert "about" in doc_dict
assert "name" in doc_dict
assert "age" in doc_dict
assert "registered" in doc_dict
assert "longitude" in doc_dict
assert "_attachments" in doc_dict
assert "range" in doc_dict
assert "balance" in doc_dict
log_info("Verified doc body for {}".format(doc_id))
def test_setting_expiry_in_bulk_docs(params_from_base_test_setup,
sg_conf_name):
"""
1. PUT /db/_bulk_docs with 10 documents. Set the "_exp":3 on 5 of these documents
2. Wait five seconds
3. POST /db/_bulk_get for the 10 documents. Validate that only the 5 non-expiring documents are returned
"""
cluster_config = params_from_base_test_setup["cluster_config"]
mode = params_from_base_test_setup["mode"]
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster_helper = ClusterKeywords()
topology = cluster_helper.get_cluster_topology(cluster_config)
cluster_helper.reset_cluster(cluster_config=cluster_config,
sync_gateway_config=sg_conf)
cbs_url = topology["couchbase_servers"][0]
sg_url = topology["sync_gateways"][0]["public"]
sg_url_admin = topology["sync_gateways"][0]["admin"]
log_info("Running 'test_setting_expiry_in_bulk_docs'")
log_info("cbs_url: {}".format(cbs_url))
log_info("sg_url: {}".format(sg_url))
log_info("sg_url_admin: {}".format(sg_url_admin))
sg_db = "db"
sg_user_name = "sg_user"
sg_user_password = "******"
sg_user_channels = ["NBC", "ABC"]
bucket_name = "data-bucket"
cbs_ip = host_for_url(cbs_url)
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name),
password='******')
client = MobileRestClient()
client.create_user(url=sg_url_admin,
db=sg_db,
name=sg_user_name,
password=sg_user_password,
channels=sg_user_channels)
sg_user_session = client.create_session(url=sg_url_admin,
db=sg_db,
name=sg_user_name)
doc_exp_3_bodies = document.create_docs(doc_id_prefix="exp_3",
number=5,
expiry=3,
channels=sg_user_channels)
doc_exp_10_bodies = document.create_docs(doc_id_prefix="exp_10",
number=5,
expiry=10,
channels=sg_user_channels)
bulk_bodies = doc_exp_3_bodies + doc_exp_10_bodies
bulk_docs = client.add_bulk_docs(url=sg_url,
db=sg_db,
docs=bulk_bodies,
auth=sg_user_session)
# Allow exp_3 docs to expire
time.sleep(5)
bulk_docs_ids = [doc["id"] for doc in bulk_docs]
expected_ids = ["exp_10_0", "exp_10_1", "exp_10_2", "exp_10_3", "exp_10_4"]
expected_missing_ids = [
"exp_3_0", "exp_3_1", "exp_3_2", "exp_3_3", "exp_3_4"
]
bulk_get_docs, errors = client.get_bulk_docs(url=sg_url,
db=sg_db,
doc_ids=bulk_docs_ids,
auth=sg_user_session,
validate=False)
assert len(bulk_get_docs) == len(expected_ids)
assert len(errors) == len(expected_missing_ids)
bulk_get_doc_ids = [doc["_id"] for doc in bulk_get_docs]
error_ids = [doc["id"] for doc in errors]
assert bulk_get_doc_ids == expected_ids
assert error_ids == expected_missing_ids
client.verify_doc_ids_found_in_response(response=bulk_get_docs,
expected_doc_ids=expected_ids)
client.verify_doc_ids_not_found_in_response(
response=errors, expected_missing_doc_ids=expected_missing_ids)
for expired_doc in error_ids:
verify_doc_deletion_on_server(doc_id=expired_doc,
sdk_client=sdk_client,
sg_client=client,
sg_admin_url=sg_url_admin,
sg_db=sg_db,
xattrs_enabled=xattrs_enabled)
def update_docs_task(users, user_type, user_index, sg_url, sg_db,
docs_per_user_per_update):
user_name = '{}_{}'.format(user_type, user_index)
# Get a random value to determin the update method
# ~ 90% ops bulk, 10% ops single
rand = random.random()
if rand <= 0.90:
update_method = 'bulk_docs'
else:
update_method = 'put'
sg_client = MobileRestClient()
# Get a random user
current_user_auth = users[user_name]['auth']
current_user_doc_ids = list(users[user_name]['doc_ids'])
# Get a random subset of docs to update
user_docs_subset_to_update = []
for _ in range(docs_per_user_per_update):
random_doc_id = random.choice(current_user_doc_ids)
user_docs_subset_to_update.append(random_doc_id)
current_user_doc_ids.remove(random_doc_id)
log_info('Updating {} docs, method ({}) number of updates: {} ({})'.format(
len(user_docs_subset_to_update), update_method,
users[user_name]['updates'], user_name))
# Update the user's docs
if update_method == 'bulk_docs':
# Get docs for that user
user_docs, errors = sg_client.get_bulk_docs(
url=sg_url,
db=sg_db,
doc_ids=user_docs_subset_to_update,
auth=current_user_auth)
assert len(errors) == 0
# Update the 'updates' property
for doc in user_docs:
doc['updates'] += 1
# Add the docs via build_docs
sg_client.add_bulk_docs(url=sg_url,
db=sg_db,
docs=user_docs,
auth=current_user_auth)
else:
# Do a single GET / PUT for each of the user docs
for doc_id in user_docs_subset_to_update:
doc = sg_client.get_doc(url=sg_url,
db=sg_db,
doc_id=doc_id,
auth=current_user_auth)
doc['updates'] += 1
sg_client.put_doc(url=sg_url,
db=sg_db,
doc_id=doc_id,
doc_body=doc,
rev=doc['_rev'],
auth=current_user_auth)
return user_name
def test_document_resurrection(params_from_base_test_setup, sg_conf_name, deletion_type):
"""
Scenarios:
Doc meta mode / tombstone
- Create docs (set A) via Sync Gateway
- Delete docs (set A) via Sync Gateway
- Verify docs (set A) are deleted via Sync Gateway
- Create docs (set A) via Sync Gateway
- Verify revs (set A) are generation 3 via Sync Gateway
Doc meta mode / purge
- Create docs (set A) via Sync Gateway
- Purge docs (set A) via Sync Gateway
- Verify docs (set A) are deleted via Sync Gateway
- Create docs (set A) via Sync Gateway
- Verify revs (set A) are generation 1 via Sync Gateway
XATTRs / tombstone
- Create docs (set A) via Sync Gateway
- Create docs (set B) via SDK
- Delete SDK docs (set B) via Sync Gateway
- Delete SG docs (set A) via SDK
- Verify docs (set B) are deleted via Sync Gateway
- Verify docs (set B) are deleted via SDK
- Verify docs (set A) are deleted via Sync Gateway
- Verify docs (set A) are deleted via SDK
- Create docs (set A) via Sync Gateway
- Create docs (set B) via SDK
- Verify revs (set A, B) are generation 3 via Sync Gateway
XATTRs / purge
- Create docs (set A) via Sync Gateway
- Create docs (set B) via SDK
- Purge SDK docs (set B) via Sync Gateway
- Delete SG docs (set A) via SDK
- Verify docs (set B) are deleted via Sync Gateway
- Verify docs (set B) are deleted via SDK
- Verify docs (set A) are deleted via Sync Gateway
- Verify docs (set A) are deleted via SDK
- Create docs (set A) via Sync Gateway
- Create docs (set B) via SDK
- Verify revs (set A, B) are generation 1 via Sync Gateway
"""
cluster_conf = params_from_base_test_setup['cluster_config']
cluster_topology = params_from_base_test_setup['cluster_topology']
mode = params_from_base_test_setup['mode']
xattrs_enabled = params_from_base_test_setup['xattrs_enabled']
cbs_url = cluster_topology['couchbase_servers'][0]
sg_admin_url = cluster_topology['sync_gateways'][0]['admin']
sg_url = cluster_topology['sync_gateways'][0]['public']
bucket_name = 'data-bucket'
sg_db = 'db'
cbs_host = host_for_url(cbs_url)
num_docs_per_client = 10
# Reset cluster
sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
cluster = Cluster(config=cluster_conf)
cluster.reset(sg_config_path=sg_conf)
# Initialize clients
sg_client = MobileRestClient()
sdk_client = Bucket('couchbase://{}/{}'.format(cbs_host, bucket_name), password='******')
# Create Sync Gateway user
sg_user_channels = ['NASA', 'NATGEO']
sg_client.create_user(url=sg_admin_url, db=sg_db, name='seth', password='******', channels=sg_user_channels)
sg_user_auth = sg_client.create_session(url=sg_admin_url, db=sg_db, name='seth', password='******')
# Create / Add docs from SG
sg_doc_bodies = document.create_docs(
doc_id_prefix='sg_doc',
number=num_docs_per_client,
content={'foo': 'bar'},
channels=sg_user_channels,
attachments_generator=attachment.generate_2_png_10_10
)
sg_doc_ids = [doc['_id'] for doc in sg_doc_bodies]
sg_bulk_docs_resp = sg_client.add_bulk_docs(url=sg_url, db=sg_db, docs=sg_doc_bodies, auth=sg_user_auth)
assert len(sg_bulk_docs_resp) == num_docs_per_client
all_doc_ids = sg_doc_ids
assert len(all_doc_ids) == num_docs_per_client
if xattrs_enabled:
# Create / Add docs from sdk
log_info('Adding docs via SDK')
sdk_doc_bodies = document.create_docs(
doc_id_prefix='sdk_doc',
number=num_docs_per_client,
content={'foo': 'bar'},
channels=sg_user_channels,
)
sdk_docs = {doc['_id']: doc for doc in sdk_doc_bodies}
sdk_doc_ids = [doc['_id'] for doc in sdk_doc_bodies]
log_info('Creating SDK docs')
sdk_client.upsert_multi(sdk_docs)
all_doc_ids = sg_doc_ids + sdk_doc_ids
assert len(all_doc_ids) == num_docs_per_client * 2
if deletion_type == 'tombstone':
# Set the target docs.
# Doc meta mode: Delete Sync Gateway docs via Sync Gateway
# XATTR mode: Delete SDK docs via Sync Gateway
sg_doc_ids_to_delete = sg_doc_ids
if xattrs_enabled:
sg_doc_ids_to_delete = sdk_doc_ids
# SG delete target docs
for doc_id in sg_doc_ids_to_delete:
doc = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id, auth=sg_user_auth)
deleted = sg_client.delete_doc(url=sg_url, db=sg_db, doc_id=doc_id, rev=doc['_rev'], auth=sg_user_auth)
log_info(deleted)
if xattrs_enabled:
log_info('Deleting SG docs via SDK')
sdk_client.remove_multi(sg_doc_ids)
elif deletion_type == 'purge':
# SG Purge all docs
all_docs, errors = sg_client.get_bulk_docs(url=sg_url, db=sg_db, doc_ids=all_doc_ids, auth=sg_user_auth)
if xattrs_enabled:
assert len(all_docs) == num_docs_per_client * 2
assert len(errors) == 0
else:
assert len(all_docs) == num_docs_per_client
assert len(errors) == 0
log_info('Purging docs via Sync Gateway')
sg_client.purge_docs(url=sg_admin_url, db=sg_db, docs=all_docs)
else:
raise ValueError('Invalid test parameters')
# Verify deletes via Sync Gateway
deleted_docs_to_verify = sg_doc_ids
assert len(deleted_docs_to_verify) == num_docs_per_client
# If running is xattr mode, make sure to verify SG + SDK docs
if xattrs_enabled:
deleted_docs_to_verify = sg_doc_ids + sdk_doc_ids
assert len(deleted_docs_to_verify) == num_docs_per_client * 2
if xattrs_enabled and deletion_type == 'tombstone':
# Verify SDK + SG docs are deleted from Sync Gateway
verify_sg_deletes(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth)
# Verify SDK + SG docs are deleted from SDK
verify_sdk_deletes(sdk_client, deleted_docs_to_verify)
elif xattrs_enabled and deletion_type == 'purge':
# Verify SDK + SG docs are purged from Sync Gateway
verify_sg_purges(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth)
# Verify SDK + SG docs are deleted from SDK
verify_sdk_deletes(sdk_client, deleted_docs_to_verify)
elif not xattrs_enabled and deletion_type == 'tombstone':
# Doc meta: Verify SG docs are all deleted via SG
verify_sg_deletes(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth)
elif not xattrs_enabled and deletion_type == 'purge':
# Doc meta: Verify SG docs are all deleted via SG
verify_sg_purges(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth)
else:
raise ValueError('Invalid test parameters')
# Recreate deleted docs from Sync Gateway
sg_bulk_docs_resp = sg_client.add_bulk_docs(url=sg_url, db=sg_db, docs=sg_doc_bodies, auth=sg_user_auth)
assert len(sg_bulk_docs_resp) == num_docs_per_client
if xattrs_enabled:
log_info('Recreating SDK docs')
# Recreate deleted docs from SDK
sdk_client.upsert_multi(sdk_docs)
# Get docs via Sync Gateway
doc_ids_to_get = sg_doc_ids
if xattrs_enabled:
doc_ids_to_get = sg_doc_ids + sdk_doc_ids
docs, errors = sg_client.get_bulk_docs(
url=sg_url,
db=sg_db,
doc_ids=doc_ids_to_get,
auth=sg_user_auth,
validate=False
)
if xattrs_enabled:
assert len(docs) == num_docs_per_client * 2
assert len(errors) == 0
else:
assert len(docs) == num_docs_per_client
assert len(errors) == 0
if xattrs_enabled:
# Get SDK docs and makes sure all docs were recreated
all_docs_from_sdk = sdk_client.get_multi(doc_ids_to_get)
assert len(all_docs_from_sdk) == num_docs_per_client * 2
log_info('Found: {} recreated docs via SDK'.format(len(all_docs_from_sdk)))
# Make sure we are able to get recreated docs via SDK
doc_ids_to_get_scratch = list(doc_ids_to_get)
assert len(doc_ids_to_get_scratch) == num_docs_per_client * 2
for doc_id in all_docs_from_sdk:
doc_ids_to_get_scratch.remove(doc_id)
assert len(doc_ids_to_get_scratch) == 0
# Make sure we are able to get recreated docs via SDK
doc_ids_to_get_scratch = list(doc_ids_to_get)
if xattrs_enabled:
# SG + SDK docs
assert len(doc_ids_to_get_scratch) == num_docs_per_client * 2
else:
# SG docs
assert len(doc_ids_to_get_scratch) == num_docs_per_client
for doc in docs:
# Verify expected document revisions
if xattrs_enabled:
if deletion_type == 'purge':
# SG purges SG docs and recreates them, expecting 1- rev
# SDK removes SDK docs and recreates them, expecting 1- rev
assert doc['_rev'].startswith('1-')
else:
# SG tombstones SG docs and recreates them, expecting 3- rev
# SDK removes SDK docs and recreates them, expecting 1- rev
if doc['_id'].startswith('sg_'):
assert doc['_rev'].startswith('3-')
else:
assert doc['_rev'].startswith('1-')
else:
if deletion_type == 'purge':
# SG purges SG docs and recreates them, expecting 1- rev
assert doc['_rev'].startswith('1-')
else:
# SG tombstones SG docs and recreates them, expecting 3- rev
assert doc['_rev'].startswith('3-')
doc_ids_to_get_scratch.remove(doc['_id'])
# Make sure all docs were found
assert len(doc_ids_to_get_scratch) == 0
