def test_setting_expiry_in_bulk_docs(params_from_base_test_setup, sg_conf_name): """ 1. PUT /db/_bulk_docs with 10 documents. Set the "_exp":3 on 5 of these documents 2. Wait five seconds 3. POST /db/_bulk_get for the 10 documents. Validate that only the 5 non-expiring documents are returned """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) cluster_helper = ClusterKeywords() topology = cluster_helper.get_cluster_topology(cluster_config) cluster_helper.reset_cluster( cluster_config=cluster_config, sync_gateway_config=sg_conf ) cbs_url = topology["couchbase_servers"][0] sg_url = topology["sync_gateways"][0]["public"] sg_url_admin = topology["sync_gateways"][0]["admin"] log_info("Running 'test_setting_expiry_in_bulk_docs'") log_info("cbs_url: {}".format(cbs_url)) log_info("sg_url: {}".format(sg_url)) log_info("sg_url_admin: {}".format(sg_url_admin)) sg_db = "db" sg_user_name = "sg_user" sg_user_password = "******" sg_user_channels = ["NBC", "ABC"] client = MobileRestClient() client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels) sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name) doc_exp_3_bodies = document.create_docs(doc_id_prefix="exp_3", number=5, expiry=3, channels=sg_user_channels) doc_exp_10_bodies = document.create_docs(doc_id_prefix="exp_10", number=5, expiry=10, channels=sg_user_channels) bulk_bodies = doc_exp_3_bodies + doc_exp_10_bodies bulk_docs = client.add_bulk_docs(url=sg_url, db=sg_db, docs=bulk_bodies, auth=sg_user_session) # Allow exp_3 docs to expire time.sleep(5) bulk_get_docs = client.get_bulk_docs(url=sg_url, db=sg_db, docs=bulk_docs, auth=sg_user_session) expected_ids = ["exp_10_0", "exp_10_1", "exp_10_2", "exp_10_3", "exp_10_4"] expected_missing_ids = ["exp_3_0", "exp_3_1", "exp_3_2", "exp_3_3", "exp_3_4"] client.verify_doc_ids_found_in_response(response=bulk_get_docs, expected_doc_ids=expected_ids) client.verify_doc_ids_not_found_in_response(response=bulk_get_docs, expected_missing_doc_ids=expected_missing_ids)
def verify_sg_docs_revision_history(url, db, added_docs): sg_client = MobileRestClient() expected_doc_map = { added_doc["id"]: added_doc["rev"] for added_doc in added_docs } doc_ids = expected_doc_map.keys() log_info("Bulk getting docs from sync gateway") docs = sg_client.get_bulk_docs(url, db, doc_ids, rev_history="true") assert len(docs[0]) == len(doc_ids) for doc in docs: for doc_dict in doc: rev = doc_dict["_rev"] rev_gen = int(rev.split("-")[0]) doc_id = doc_dict["_id"] # Verify meta data log_info("Verifying that doc {} has rev {}".format( doc_id, expected_doc_map[doc_id])) assert rev == expected_doc_map[doc_id] log_info( "Doc {}: Expected number of revs: {}, Actual revs: {}".format( doc_id, rev_gen, len(doc_dict["_revisions"]["ids"]))) assert len(doc_dict["_revisions"]["ids"]) == rev_gen log_info( "Verifying that doc {} is associated with sg_user_channel channel" .format(doc_id)) assert doc_dict["channels"][0] == "sg_user_channel" # Verify doc body log_info("Verifying doc body for {}".format(doc_id)) assert "guid" in doc_dict assert "index" in doc_dict assert "latitude" in doc_dict assert "email" in doc_dict assert "picture" in doc_dict assert len(doc_dict["tags"]) == 3 assert "date_time_added" in doc_dict assert "company" in doc_dict assert "eyeColor" in doc_dict assert "phone" in doc_dict assert "updates" in doc_dict assert "address" in doc_dict assert len(doc_dict["friends"]) == 2 assert "isActive" in doc_dict assert "about" in doc_dict assert "name" in doc_dict assert "age" in doc_dict assert "registered" in doc_dict assert "longitude" in doc_dict assert "_attachments" in doc_dict assert "range" in doc_dict assert "balance" in doc_dict log_info("Verified doc body for {}".format(doc_id))
def test_setting_expiry_in_bulk_docs(params_from_base_test_setup, sg_conf_name): """ 1. PUT /db/_bulk_docs with 10 documents. Set the "_exp":3 on 5 of these documents 2. Wait five seconds 3. POST /db/_bulk_get for the 10 documents. Validate that only the 5 non-expiring documents are returned """ cluster_config = params_from_base_test_setup["cluster_config"] mode = params_from_base_test_setup["mode"] xattrs_enabled = params_from_base_test_setup['xattrs_enabled'] sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) cluster_helper = ClusterKeywords() topology = cluster_helper.get_cluster_topology(cluster_config) cluster_helper.reset_cluster(cluster_config=cluster_config, sync_gateway_config=sg_conf) cbs_url = topology["couchbase_servers"][0] sg_url = topology["sync_gateways"][0]["public"] sg_url_admin = topology["sync_gateways"][0]["admin"] log_info("Running 'test_setting_expiry_in_bulk_docs'") log_info("cbs_url: {}".format(cbs_url)) log_info("sg_url: {}".format(sg_url)) log_info("sg_url_admin: {}".format(sg_url_admin)) sg_db = "db" sg_user_name = "sg_user" sg_user_password = "******" sg_user_channels = ["NBC", "ABC"] bucket_name = "data-bucket" cbs_ip = host_for_url(cbs_url) sdk_client = Bucket('couchbase://{}/{}'.format(cbs_ip, bucket_name), password='******') client = MobileRestClient() client.create_user(url=sg_url_admin, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels) sg_user_session = client.create_session(url=sg_url_admin, db=sg_db, name=sg_user_name) doc_exp_3_bodies = document.create_docs(doc_id_prefix="exp_3", number=5, expiry=3, channels=sg_user_channels) doc_exp_10_bodies = document.create_docs(doc_id_prefix="exp_10", number=5, expiry=10, channels=sg_user_channels) bulk_bodies = doc_exp_3_bodies + doc_exp_10_bodies bulk_docs = client.add_bulk_docs(url=sg_url, db=sg_db, docs=bulk_bodies, auth=sg_user_session) # Allow exp_3 docs to expire time.sleep(5) bulk_docs_ids = [doc["id"] for doc in bulk_docs] expected_ids = ["exp_10_0", "exp_10_1", "exp_10_2", "exp_10_3", "exp_10_4"] expected_missing_ids = [ "exp_3_0", "exp_3_1", "exp_3_2", "exp_3_3", "exp_3_4" ] bulk_get_docs, errors = client.get_bulk_docs(url=sg_url, db=sg_db, doc_ids=bulk_docs_ids, auth=sg_user_session, validate=False) assert len(bulk_get_docs) == len(expected_ids) assert len(errors) == len(expected_missing_ids) bulk_get_doc_ids = [doc["_id"] for doc in bulk_get_docs] error_ids = [doc["id"] for doc in errors] assert bulk_get_doc_ids == expected_ids assert error_ids == expected_missing_ids client.verify_doc_ids_found_in_response(response=bulk_get_docs, expected_doc_ids=expected_ids) client.verify_doc_ids_not_found_in_response( response=errors, expected_missing_doc_ids=expected_missing_ids) for expired_doc in error_ids: verify_doc_deletion_on_server(doc_id=expired_doc, sdk_client=sdk_client, sg_client=client, sg_admin_url=sg_url_admin, sg_db=sg_db, xattrs_enabled=xattrs_enabled)
def update_docs_task(users, user_type, user_index, sg_url, sg_db, docs_per_user_per_update): user_name = '{}_{}'.format(user_type, user_index) # Get a random value to determin the update method # ~ 90% ops bulk, 10% ops single rand = random.random() if rand <= 0.90: update_method = 'bulk_docs' else: update_method = 'put' sg_client = MobileRestClient() # Get a random user current_user_auth = users[user_name]['auth'] current_user_doc_ids = list(users[user_name]['doc_ids']) # Get a random subset of docs to update user_docs_subset_to_update = [] for _ in range(docs_per_user_per_update): random_doc_id = random.choice(current_user_doc_ids) user_docs_subset_to_update.append(random_doc_id) current_user_doc_ids.remove(random_doc_id) log_info('Updating {} docs, method ({}) number of updates: {} ({})'.format( len(user_docs_subset_to_update), update_method, users[user_name]['updates'], user_name)) # Update the user's docs if update_method == 'bulk_docs': # Get docs for that user user_docs, errors = sg_client.get_bulk_docs( url=sg_url, db=sg_db, doc_ids=user_docs_subset_to_update, auth=current_user_auth) assert len(errors) == 0 # Update the 'updates' property for doc in user_docs: doc['updates'] += 1 # Add the docs via build_docs sg_client.add_bulk_docs(url=sg_url, db=sg_db, docs=user_docs, auth=current_user_auth) else: # Do a single GET / PUT for each of the user docs for doc_id in user_docs_subset_to_update: doc = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id, auth=current_user_auth) doc['updates'] += 1 sg_client.put_doc(url=sg_url, db=sg_db, doc_id=doc_id, doc_body=doc, rev=doc['_rev'], auth=current_user_auth) return user_name
def test_document_resurrection(params_from_base_test_setup, sg_conf_name, deletion_type): """ Scenarios: Doc meta mode / tombstone - Create docs (set A) via Sync Gateway - Delete docs (set A) via Sync Gateway - Verify docs (set A) are deleted via Sync Gateway - Create docs (set A) via Sync Gateway - Verify revs (set A) are generation 3 via Sync Gateway Doc meta mode / purge - Create docs (set A) via Sync Gateway - Purge docs (set A) via Sync Gateway - Verify docs (set A) are deleted via Sync Gateway - Create docs (set A) via Sync Gateway - Verify revs (set A) are generation 1 via Sync Gateway XATTRs / tombstone - Create docs (set A) via Sync Gateway - Create docs (set B) via SDK - Delete SDK docs (set B) via Sync Gateway - Delete SG docs (set A) via SDK - Verify docs (set B) are deleted via Sync Gateway - Verify docs (set B) are deleted via SDK - Verify docs (set A) are deleted via Sync Gateway - Verify docs (set A) are deleted via SDK - Create docs (set A) via Sync Gateway - Create docs (set B) via SDK - Verify revs (set A, B) are generation 3 via Sync Gateway XATTRs / purge - Create docs (set A) via Sync Gateway - Create docs (set B) via SDK - Purge SDK docs (set B) via Sync Gateway - Delete SG docs (set A) via SDK - Verify docs (set B) are deleted via Sync Gateway - Verify docs (set B) are deleted via SDK - Verify docs (set A) are deleted via Sync Gateway - Verify docs (set A) are deleted via SDK - Create docs (set A) via Sync Gateway - Create docs (set B) via SDK - Verify revs (set A, B) are generation 1 via Sync Gateway """ cluster_conf = params_from_base_test_setup['cluster_config'] cluster_topology = params_from_base_test_setup['cluster_topology'] mode = params_from_base_test_setup['mode'] xattrs_enabled = params_from_base_test_setup['xattrs_enabled'] cbs_url = cluster_topology['couchbase_servers'][0] sg_admin_url = cluster_topology['sync_gateways'][0]['admin'] sg_url = cluster_topology['sync_gateways'][0]['public'] bucket_name = 'data-bucket' sg_db = 'db' cbs_host = host_for_url(cbs_url) num_docs_per_client = 10 # Reset cluster sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode) cluster = Cluster(config=cluster_conf) cluster.reset(sg_config_path=sg_conf) # Initialize clients sg_client = MobileRestClient() sdk_client = Bucket('couchbase://{}/{}'.format(cbs_host, bucket_name), password='******') # Create Sync Gateway user sg_user_channels = ['NASA', 'NATGEO'] sg_client.create_user(url=sg_admin_url, db=sg_db, name='seth', password='******', channels=sg_user_channels) sg_user_auth = sg_client.create_session(url=sg_admin_url, db=sg_db, name='seth', password='******') # Create / Add docs from SG sg_doc_bodies = document.create_docs( doc_id_prefix='sg_doc', number=num_docs_per_client, content={'foo': 'bar'}, channels=sg_user_channels, attachments_generator=attachment.generate_2_png_10_10 ) sg_doc_ids = [doc['_id'] for doc in sg_doc_bodies] sg_bulk_docs_resp = sg_client.add_bulk_docs(url=sg_url, db=sg_db, docs=sg_doc_bodies, auth=sg_user_auth) assert len(sg_bulk_docs_resp) == num_docs_per_client all_doc_ids = sg_doc_ids assert len(all_doc_ids) == num_docs_per_client if xattrs_enabled: # Create / Add docs from sdk log_info('Adding docs via SDK') sdk_doc_bodies = document.create_docs( doc_id_prefix='sdk_doc', number=num_docs_per_client, content={'foo': 'bar'}, channels=sg_user_channels, ) sdk_docs = {doc['_id']: doc for doc in sdk_doc_bodies} sdk_doc_ids = [doc['_id'] for doc in sdk_doc_bodies] log_info('Creating SDK docs') sdk_client.upsert_multi(sdk_docs) all_doc_ids = sg_doc_ids + sdk_doc_ids assert len(all_doc_ids) == num_docs_per_client * 2 if deletion_type == 'tombstone': # Set the target docs. # Doc meta mode: Delete Sync Gateway docs via Sync Gateway # XATTR mode: Delete SDK docs via Sync Gateway sg_doc_ids_to_delete = sg_doc_ids if xattrs_enabled: sg_doc_ids_to_delete = sdk_doc_ids # SG delete target docs for doc_id in sg_doc_ids_to_delete: doc = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id, auth=sg_user_auth) deleted = sg_client.delete_doc(url=sg_url, db=sg_db, doc_id=doc_id, rev=doc['_rev'], auth=sg_user_auth) log_info(deleted) if xattrs_enabled: log_info('Deleting SG docs via SDK') sdk_client.remove_multi(sg_doc_ids) elif deletion_type == 'purge': # SG Purge all docs all_docs, errors = sg_client.get_bulk_docs(url=sg_url, db=sg_db, doc_ids=all_doc_ids, auth=sg_user_auth) if xattrs_enabled: assert len(all_docs) == num_docs_per_client * 2 assert len(errors) == 0 else: assert len(all_docs) == num_docs_per_client assert len(errors) == 0 log_info('Purging docs via Sync Gateway') sg_client.purge_docs(url=sg_admin_url, db=sg_db, docs=all_docs) else: raise ValueError('Invalid test parameters') # Verify deletes via Sync Gateway deleted_docs_to_verify = sg_doc_ids assert len(deleted_docs_to_verify) == num_docs_per_client # If running is xattr mode, make sure to verify SG + SDK docs if xattrs_enabled: deleted_docs_to_verify = sg_doc_ids + sdk_doc_ids assert len(deleted_docs_to_verify) == num_docs_per_client * 2 if xattrs_enabled and deletion_type == 'tombstone': # Verify SDK + SG docs are deleted from Sync Gateway verify_sg_deletes(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth) # Verify SDK + SG docs are deleted from SDK verify_sdk_deletes(sdk_client, deleted_docs_to_verify) elif xattrs_enabled and deletion_type == 'purge': # Verify SDK + SG docs are purged from Sync Gateway verify_sg_purges(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth) # Verify SDK + SG docs are deleted from SDK verify_sdk_deletes(sdk_client, deleted_docs_to_verify) elif not xattrs_enabled and deletion_type == 'tombstone': # Doc meta: Verify SG docs are all deleted via SG verify_sg_deletes(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth) elif not xattrs_enabled and deletion_type == 'purge': # Doc meta: Verify SG docs are all deleted via SG verify_sg_purges(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth) else: raise ValueError('Invalid test parameters') # Recreate deleted docs from Sync Gateway sg_bulk_docs_resp = sg_client.add_bulk_docs(url=sg_url, db=sg_db, docs=sg_doc_bodies, auth=sg_user_auth) assert len(sg_bulk_docs_resp) == num_docs_per_client if xattrs_enabled: log_info('Recreating SDK docs') # Recreate deleted docs from SDK sdk_client.upsert_multi(sdk_docs) # Get docs via Sync Gateway doc_ids_to_get = sg_doc_ids if xattrs_enabled: doc_ids_to_get = sg_doc_ids + sdk_doc_ids docs, errors = sg_client.get_bulk_docs( url=sg_url, db=sg_db, doc_ids=doc_ids_to_get, auth=sg_user_auth, validate=False ) if xattrs_enabled: assert len(docs) == num_docs_per_client * 2 assert len(errors) == 0 else: assert len(docs) == num_docs_per_client assert len(errors) == 0 if xattrs_enabled: # Get SDK docs and makes sure all docs were recreated all_docs_from_sdk = sdk_client.get_multi(doc_ids_to_get) assert len(all_docs_from_sdk) == num_docs_per_client * 2 log_info('Found: {} recreated docs via SDK'.format(len(all_docs_from_sdk))) # Make sure we are able to get recreated docs via SDK doc_ids_to_get_scratch = list(doc_ids_to_get) assert len(doc_ids_to_get_scratch) == num_docs_per_client * 2 for doc_id in all_docs_from_sdk: doc_ids_to_get_scratch.remove(doc_id) assert len(doc_ids_to_get_scratch) == 0 # Make sure we are able to get recreated docs via SDK doc_ids_to_get_scratch = list(doc_ids_to_get) if xattrs_enabled: # SG + SDK docs assert len(doc_ids_to_get_scratch) == num_docs_per_client * 2 else: # SG docs assert len(doc_ids_to_get_scratch) == num_docs_per_client for doc in docs: # Verify expected document revisions if xattrs_enabled: if deletion_type == 'purge': # SG purges SG docs and recreates them, expecting 1- rev # SDK removes SDK docs and recreates them, expecting 1- rev assert doc['_rev'].startswith('1-') else: # SG tombstones SG docs and recreates them, expecting 3- rev # SDK removes SDK docs and recreates them, expecting 1- rev if doc['_id'].startswith('sg_'): assert doc['_rev'].startswith('3-') else: assert doc['_rev'].startswith('1-') else: if deletion_type == 'purge': # SG purges SG docs and recreates them, expecting 1- rev assert doc['_rev'].startswith('1-') else: # SG tombstones SG docs and recreates them, expecting 3- rev assert doc['_rev'].startswith('3-') doc_ids_to_get_scratch.remove(doc['_id']) # Make sure all docs were found assert len(doc_ids_to_get_scratch) == 0