def test_deleted_docs_from_changes_active_only(params_from_base_test_setup, sg_conf_name): """ https://github.com/couchbase/sync_gateway/issues/2955 1. Create a document 2. Delete the document 3. Restart Sync Gateway (to force rebuild of cache from view) 4. Issue an active_only=true changes request 5. Issue an active_only=false changes request The deleted document was not being included in the result set in step 5. """ cluster_config = params_from_base_test_setup["cluster_config"] topology = params_from_base_test_setup["cluster_topology"] sg_admin_url = topology["sync_gateways"][0]["admin"] sg_db = "db" num_docs = 10 client = MobileRestClient() # Add doc to SG added_doc = client.add_docs(url=sg_admin_url, db=sg_db, number=num_docs, id_prefix="test_changes") # Delete 1 doc doc_id = added_doc[0]["id"] log_info("Deleting {}".format(doc_id)) doc = client.get_doc(url=sg_admin_url, db=sg_db, doc_id=doc_id) doc_rev = doc['_rev'] client.delete_doc(sg_admin_url, sg_db, doc_id, doc_rev) # Restart SG sg_obj = SyncGateway() sg_obj.restart_sync_gateways(cluster_config) # Changes request with active_only=true session = Session() request_url = "{}/{}/_changes?active_only=true".format(sg_admin_url, sg_db) log_info("Issuing changes request {}".format(request_url)) resp = session.get(request_url) resp.raise_for_status() resp_obj = resp.json() log_info( "Checking that the deleted doc is not included in the active_only=true changes request" ) for d in resp_obj["results"]: assert doc_id not in d # Changes request with active_only=false request_url = "{}/{}/_changes?active_only=false".format( sg_admin_url, sg_db) log_info("Issuing changes request {}".format(request_url)) resp = session.get(request_url) resp.raise_for_status() resp_obj = resp.json() doc_found = False for d in resp_obj["results"]: if doc_id != d["id"]: continue else: assert doc_id == d["id"] assert d["deleted"] doc_found = True break log_info( "Checking that the deleted doc is included in the active_only=false changes request" ) assert doc_found
def test_upgrade(params_from_base_test_setup): """ @summary The initial versions of SG and CBS has already been provisioned at this point We have to upgrade them to the upgraded versions """ cluster_config = params_from_base_test_setup['cluster_config'] mode = params_from_base_test_setup['mode'] xattrs_enabled = params_from_base_test_setup['xattrs_enabled'] ls_url = params_from_base_test_setup["ls_url"] server_version = params_from_base_test_setup['server_version'] sync_gateway_version = params_from_base_test_setup['sync_gateway_version'] server_upgraded_version = params_from_base_test_setup[ 'server_upgraded_version'] sync_gateway_upgraded_version = params_from_base_test_setup[ 'sync_gateway_upgraded_version'] sg_url = params_from_base_test_setup['sg_url'] sg_admin_url = params_from_base_test_setup['sg_admin_url'] num_docs = int(params_from_base_test_setup['num_docs']) cbs_platform = params_from_base_test_setup['cbs_platform'] cbs_toy_build = params_from_base_test_setup['cbs_toy_build'] sg_conf = "{}/resources/sync_gateway_configs/sync_gateway_default_functional_tests_{}.json".format( os.getcwd(), mode) # Add data to liteserv client = MobileRestClient() log_info("ls_url: {}".format(ls_url)) ls_db = client.create_database(ls_url, name="ls_db") # Create user and session on SG sg_user_channels = ["sg_user_channel"] sg_db = "db" sg_user_name = "sg_user" sg_user_password = "******" client.create_user(url=sg_admin_url, db=sg_db, name=sg_user_name, password=sg_user_password, channels=sg_user_channels) sg_session = client.create_session(url=sg_admin_url, db=sg_db, name=sg_user_name, password=sg_user_password) log_info( "Starting continuous push pull replication from liteserv to sync gateway" ) repl_one = client.start_replication(url=ls_url, continuous=True, from_db=ls_db, to_url=sg_url, to_db=sg_db, to_auth=sg_session) client.wait_for_replication_status_idle(ls_url, repl_one) log_info("Starting replication from sync gateway to liteserv") client.start_replication(url=ls_url, continuous=True, from_url=sg_url, from_db=sg_db, from_auth=sg_session, to_db=ls_db) # Add docs to liteserv added_docs = add_docs_to_client_task(client=client, url=ls_url, db=ls_db, channels=sg_user_channels, num_docs=num_docs) log_info("Added {} docs".format(len(added_docs))) # start updating docs terminator_doc_id = 'terminator' with ProcessPoolExecutor() as up: # Start updates in background process updates_future = up.submit(update_docs, client, ls_url, ls_db, added_docs, sg_session, terminator_doc_id) # Supported upgrade process # 1. Upgrade SGs first docmeta -> docmeta - CBS 5.0.0 does not support TAP. # 2. Upgrade the CBS cluster. # 3. Enable import/xattrs on SGs # Upgrade SG docmeta -> docmeta cluster_util = ClusterKeywords() topology = cluster_util.get_cluster_topology(cluster_config, lb_enable=False) sync_gateways = topology["sync_gateways"] sg_accels = topology["sg_accels"] upgrade_sync_gateway(sync_gateways, sync_gateway_version, sync_gateway_upgraded_version, sg_conf, cluster_config) if mode == "di": upgrade_sg_accel(sg_accels, sync_gateway_version, sync_gateway_upgraded_version, sg_conf, cluster_config) # Upgrade CBS cluster = Cluster(config=cluster_config) if len(cluster.servers) < 2: raise Exception("Please provide at least 3 servers") server_urls = [] for server in cluster.servers: server_urls.append(server.url) primary_server = cluster.servers[0] secondary_server = cluster.servers[1] servers = cluster.servers[1:] upgrade_server_cluster(servers, primary_server, secondary_server, server_version, server_upgraded_version, server_urls, cluster_config, cbs_platform, toy_build=cbs_toy_build) # Restart SGs after the server upgrade sg_obj = SyncGateway() for sg in sync_gateways: sg_ip = host_for_url(sg["admin"]) log_info("Restarting sync gateway {}".format(sg_ip)) sg_obj.restart_sync_gateways(cluster_config=cluster_config, url=sg_ip) time.sleep(5) if mode == "di": ac_obj = SyncGateway() for ac in sg_accels: ac_ip = host_for_url(ac) log_info("Restarting sg accel {}".format(ac_ip)) ac_obj.restart_sync_gateways(cluster_config=cluster_config, url=ac_ip) time.sleep(5) if xattrs_enabled: # Enable xattrs on all SG/SGAccel nodes # cc - Start 1 SG with import enabled, all with XATTRs enabled # di - All SGs/SGAccels with xattrs enabled - this will also enable import on SGAccel # - Do not enable import in SG. if mode == "cc": enable_import = True elif mode == "di": enable_import = False if mode == "di": ac_obj = SyncGateway() for ac in sg_accels: ac_ip = host_for_url(ac) ac_obj.enable_import_xattrs(cluster_config=cluster_config, sg_conf=sg_conf, url=ac_ip, enable_import=False) sg_obj = SyncGateway() for sg in sync_gateways: sg_ip = host_for_url(sg["admin"]) sg_obj.enable_import_xattrs(cluster_config=cluster_config, sg_conf=sg_conf, url=sg_ip, enable_import=enable_import) enable_import = False # Check Import showing up on all nodes send_changes_termination_doc(auth=sg_session, terminator_doc_id=terminator_doc_id, terminator_channel=sg_user_channels, ls_url=ls_url, ls_db=ls_db) log_info("Waiting for doc updates to complete") updated_doc_revs = updates_future.result() log_info("Stopping replication from liteserv to sync gateway") # Stop repl_one client.stop_replication(url=ls_url, continuous=True, from_db=ls_db, to_url=sg_url, to_db=sg_db, to_auth=sg_session) log_info("Stopping replication from sync gateway to liteserv") # Stop repl_two client.stop_replication(url=ls_url, continuous=True, from_url=sg_url, from_db=sg_db, from_auth=sg_session, to_db=ls_db) # Gather the new revs for verification log_info("Gathering the updated revs for verification") doc_ids = [] for i in range(len(added_docs)): doc_ids.append(added_docs[i]["id"]) if added_docs[i]["id"] in updated_doc_revs: added_docs[i]["rev"] = updated_doc_revs[added_docs[i]["id"]] # Verify rev, doc bdy and revision history of all docs verify_sg_docs_revision_history(url=sg_admin_url, db=sg_db, added_docs=added_docs) if xattrs_enabled: # Verify through SDK that there is no _sync property in the doc body bucket_name = 'data-bucket' sdk_client = Bucket('couchbase://{}/{}'.format( primary_server.host, bucket_name), password='******', timeout=SDK_TIMEOUT) log_info("Fetching docs from SDK") docs_from_sdk = sdk_client.get_multi(doc_ids) log_info("Verifying that there is no _sync property in the docs") for i in docs_from_sdk: if "_sync" in docs_from_sdk[i].value: raise Exception( "_sync section found in docs after upgrade")