def test_inline_large_attachments(setup_client_syncgateway_test):
    """
    1.  Start LiteServ and Sync Gateway
    2.  Create 2 databases on LiteServ (ls_db1, ls_db2)
    3.  Start continuous push replication from ls_db1 to sg_db
    4.  Start continuous pull replication from sg_db to ls_db2
    5.  PUT 5 large inline attachments to ls_db1
    6.  DELETE the docs on ls_db1
    7.  PUT same 5 large inline attachments to ls_db1
    8.  Verify docs replicate to ls_db2
    9.  Purge ls_db1
    10. Verify docs removed
    """

    log_info("Running 'test_inline_large_attachments' ...")

    sg_url = setup_client_syncgateway_test["sg_url"]
    sg_url_admin = setup_client_syncgateway_test["sg_admin_url"]

    ls_url = setup_client_syncgateway_test["ls_url"]

    log_info("ls_url: {}".format(ls_url))
    log_info("sg_url: {}".format(sg_url))
    log_info("sg_url_admin: {}".format(sg_url_admin))

    ls_db1 = "ls_db1"
    ls_db2 = "ls_db2"
    sg_db = "db"

    client = MobileRestClient()
    client.create_database(ls_url, ls_db1)
    client.create_database(ls_url, ls_db2)

    # Start continuous push replication from ls_db1 -> sg_db
    client.start_replication(url=ls_url, continuous=True, from_db=ls_db1, to_url=sg_url, to_db=sg_db)

    # Start continuous push replication from sg_db -> ls_db2
    client.start_replication(url=ls_url, continuous=True, from_url=sg_url, from_db=sg_db, to_db=ls_db2)

    # doc with 2.36 PNG attachment
    attachment_docs = []
    for i in range(5):
        doc = document.create_doc(
            doc_id="large_attach_{}".format(i), attachment_name="golden_gate_large.jpg", channels=["ABC"]
        )
        attachment_docs.append(doc)

    # add large attachments to ls_db1
    docs = []
    for doc in attachment_docs:
        docs.append(client.add_doc(ls_url, ls_db1, doc, use_post=False))

    # Delete docs
    client.delete_docs(ls_url, ls_db1, docs)
    client.verify_docs_deleted(ls_url, ls_db1, docs)

    # Recreated docs
    recreated_docs = []
    for doc in attachment_docs:
        recreated_docs.append(client.add_doc(ls_url, ls_db1, doc, use_post=False))

    client.verify_docs_present(ls_url, ls_db1, recreated_docs)
    client.verify_docs_present(sg_url, sg_db, recreated_docs)
    client.verify_docs_present(ls_url, ls_db2, recreated_docs)

    purged_docs = client.purge_docs(ls_url, ls_db1, recreated_docs)
    log_info(purged_docs)

    # All purged docs should have replicated and should be gone now.
    # This is currently failing due to some docs not replicating to ls_db2
    client.verify_docs_deleted(ls_url, ls_db1, recreated_docs)
    client.verify_docs_deleted(sg_url, sg_db, recreated_docs)
    client.verify_docs_deleted(ls_url, ls_db2, recreated_docs)
def test_inline_large_attachments(setup_client_syncgateway_test):
    """
    1.  Start LiteServ and Sync Gateway
    2.  Create 2 databases on LiteServ (ls_db1, ls_db2)
    3.  Start continuous push replication from ls_db1 to sg_db
    4.  Start continuous pull replication from sg_db to ls_db2
    5.  PUT 5 large inline attachments to ls_db1
    6.  DELETE the docs on ls_db1
    7.  PUT same 5 large inline attachments to ls_db1
    8.  Verify docs replicate to ls_db2
    9.  Purge ls_db1
    10. Verify docs removed
    """

    log_info("Running 'test_inline_large_attachments' ...")

    cluster_config = setup_client_syncgateway_test["cluster_config"]
    sg_mode = setup_client_syncgateway_test["sg_mode"]
    sg_url = setup_client_syncgateway_test["sg_url"]
    sg_url_admin = setup_client_syncgateway_test["sg_admin_url"]

    ls_url = setup_client_syncgateway_test["ls_url"]

    log_info("ls_url: {}".format(ls_url))
    log_info("sg_url: {}".format(sg_url))
    log_info("sg_url_admin: {}".format(sg_url_admin))

    ls_db1 = "ls_db1"
    ls_db2 = "ls_db2"
    sg_db = "db"

    # Reset cluster to ensure no data in system
    sg_config = sync_gateway_config_path_for_mode(
        "listener_tests/listener_tests", sg_mode)
    c = cluster.Cluster(config=cluster_config)
    c.reset(sg_config_path=sg_config)

    client = MobileRestClient()
    client.create_database(ls_url, ls_db1)
    client.create_database(ls_url, ls_db2)

    # Start continuous push replication from ls_db1 -> sg_db
    client.start_replication(url=ls_url,
                             continuous=True,
                             from_db=ls_db1,
                             to_url=sg_url_admin,
                             to_db=sg_db)

    # Start continuous push replication from sg_db -> ls_db2
    client.start_replication(url=ls_url,
                             continuous=True,
                             from_url=sg_url_admin,
                             from_db=sg_db,
                             to_db=ls_db2)

    # doc with 2.36 PNG attachment
    attachment_docs = []
    for i in range(5):
        atts = attachment.load_from_data_dir(["golden_gate_large.jpg"])
        doc = document.create_doc(doc_id="large_attach_{}".format(i),
                                  attachments=atts,
                                  channels=["ABC"])
        attachment_docs.append(doc)

    # add large attachments to ls_db1
    docs = []
    for doc in attachment_docs:
        docs.append(client.add_doc(ls_url, ls_db1, doc, use_post=False))

    # Delete docs
    client.delete_docs(ls_url, ls_db1, docs)
    client.verify_docs_deleted(ls_url, ls_db1, docs)

    # Recreated docs
    recreated_docs = []
    for doc in attachment_docs:
        recreated_docs.append(
            client.add_doc(ls_url, ls_db1, doc, use_post=False))

    client.verify_docs_present(ls_url, ls_db1, recreated_docs)
    client.verify_docs_present(sg_url_admin, sg_db, recreated_docs)
    client.verify_docs_present(ls_url, ls_db2, recreated_docs)

    purged_docs = client.purge_docs(ls_url, ls_db1, recreated_docs)
    log_info(purged_docs)
    client.verify_docs_deleted(ls_url,
                               ls_db1,
                               recreated_docs,
                               reason="missing")
Пример #3
0
def test_document_resurrection(params_from_base_test_setup, sg_conf_name, deletion_type):
    """
    Scenarios:

    Doc meta mode / tombstone
    - Create docs (set A) via Sync Gateway
    - Delete docs (set A) via Sync Gateway
    - Verify docs (set A) are deleted via Sync Gateway
    - Create docs (set A) via Sync Gateway
    - Verify revs (set A) are generation 3 via Sync Gateway

    Doc meta mode / purge
    - Create docs (set A) via Sync Gateway
    - Purge docs (set A) via Sync Gateway
    - Verify docs (set A) are deleted via Sync Gateway
    - Create docs (set A) via Sync Gateway
    - Verify revs (set A) are generation 1 via Sync Gateway

    XATTRs / tombstone
    - Create docs (set A) via Sync Gateway
    - Create docs (set B) via SDK
    - Delete SDK docs (set B) via Sync Gateway
    - Delete SG docs (set A) via SDK
    - Verify docs (set B) are deleted via Sync Gateway
    - Verify docs (set B) are deleted via SDK
    - Verify docs (set A) are deleted via Sync Gateway
    - Verify docs (set A) are deleted via SDK
    - Create docs (set A) via Sync Gateway
    - Create docs (set B) via SDK
    - Verify revs (set A, B) are generation 3 via Sync Gateway

    XATTRs / purge
    - Create docs (set A) via Sync Gateway
    - Create docs (set B) via SDK
    - Purge SDK docs (set B) via Sync Gateway
    - Delete SG docs (set A) via SDK
    - Verify docs (set B) are deleted via Sync Gateway
    - Verify docs (set B) are deleted via SDK
    - Verify docs (set A) are deleted via Sync Gateway
    - Verify docs (set A) are deleted via SDK
    - Create docs (set A) via Sync Gateway
    - Create docs (set B) via SDK
    - Verify revs (set A, B) are generation 1 via Sync Gateway

    """
    cluster_conf = params_from_base_test_setup['cluster_config']
    cluster_topology = params_from_base_test_setup['cluster_topology']
    mode = params_from_base_test_setup['mode']
    xattrs_enabled = params_from_base_test_setup['xattrs_enabled']

    cbs_url = cluster_topology['couchbase_servers'][0]
    sg_admin_url = cluster_topology['sync_gateways'][0]['admin']
    sg_url = cluster_topology['sync_gateways'][0]['public']

    bucket_name = 'data-bucket'
    sg_db = 'db'
    cbs_host = host_for_url(cbs_url)

    num_docs_per_client = 10

    # Reset cluster
    sg_conf = sync_gateway_config_path_for_mode(sg_conf_name, mode)
    cluster = Cluster(config=cluster_conf)
    cluster.reset(sg_config_path=sg_conf)

    # Initialize clients
    sg_client = MobileRestClient()
    sdk_client = Bucket('couchbase://{}/{}'.format(cbs_host, bucket_name), password='******')

    # Create Sync Gateway user
    sg_user_channels = ['NASA', 'NATGEO']
    sg_client.create_user(url=sg_admin_url, db=sg_db, name='seth', password='******', channels=sg_user_channels)
    sg_user_auth = sg_client.create_session(url=sg_admin_url, db=sg_db, name='seth', password='******')

    # Create / Add docs from SG
    sg_doc_bodies = document.create_docs(
        doc_id_prefix='sg_doc',
        number=num_docs_per_client,
        content={'foo': 'bar'},
        channels=sg_user_channels,
        attachments_generator=attachment.generate_2_png_10_10
    )
    sg_doc_ids = [doc['_id'] for doc in sg_doc_bodies]

    sg_bulk_docs_resp = sg_client.add_bulk_docs(url=sg_url, db=sg_db, docs=sg_doc_bodies, auth=sg_user_auth)
    assert len(sg_bulk_docs_resp) == num_docs_per_client

    all_doc_ids = sg_doc_ids
    assert len(all_doc_ids) == num_docs_per_client

    if xattrs_enabled:
        #  Create / Add docs from sdk
        log_info('Adding docs via SDK')
        sdk_doc_bodies = document.create_docs(
            doc_id_prefix='sdk_doc',
            number=num_docs_per_client,
            content={'foo': 'bar'},
            channels=sg_user_channels,
        )
        sdk_docs = {doc['_id']: doc for doc in sdk_doc_bodies}
        sdk_doc_ids = [doc['_id'] for doc in sdk_doc_bodies]

        log_info('Creating SDK docs')
        sdk_client.upsert_multi(sdk_docs)

        all_doc_ids = sg_doc_ids + sdk_doc_ids
        assert len(all_doc_ids) == num_docs_per_client * 2

    if deletion_type == 'tombstone':
        # Set the target docs.
        # Doc meta mode: Delete Sync Gateway docs via Sync Gateway
        # XATTR mode: Delete SDK docs via Sync Gateway
        sg_doc_ids_to_delete = sg_doc_ids
        if xattrs_enabled:
            sg_doc_ids_to_delete = sdk_doc_ids

        # SG delete target docs
        for doc_id in sg_doc_ids_to_delete:
            doc = sg_client.get_doc(url=sg_url, db=sg_db, doc_id=doc_id, auth=sg_user_auth)
            deleted = sg_client.delete_doc(url=sg_url, db=sg_db, doc_id=doc_id, rev=doc['_rev'], auth=sg_user_auth)
            log_info(deleted)

        if xattrs_enabled:
            log_info('Deleting SG docs via SDK')
            sdk_client.remove_multi(sg_doc_ids)

    elif deletion_type == 'purge':
        # SG Purge all docs
        all_docs, errors = sg_client.get_bulk_docs(url=sg_url, db=sg_db, doc_ids=all_doc_ids, auth=sg_user_auth)
        if xattrs_enabled:
            assert len(all_docs) == num_docs_per_client * 2
            assert len(errors) == 0
        else:
            assert len(all_docs) == num_docs_per_client
            assert len(errors) == 0
        log_info('Purging docs via Sync Gateway')
        sg_client.purge_docs(url=sg_admin_url, db=sg_db, docs=all_docs)

    else:
        raise ValueError('Invalid test parameters')

    # Verify deletes via Sync Gateway
    deleted_docs_to_verify = sg_doc_ids
    assert len(deleted_docs_to_verify) == num_docs_per_client

    # If running is xattr mode, make sure to verify SG + SDK docs
    if xattrs_enabled:
        deleted_docs_to_verify = sg_doc_ids + sdk_doc_ids
        assert len(deleted_docs_to_verify) == num_docs_per_client * 2

    if xattrs_enabled and deletion_type == 'tombstone':

        # Verify SDK + SG docs are deleted from Sync Gateway
        verify_sg_deletes(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth)

        # Verify SDK + SG docs are deleted from SDK
        verify_sdk_deletes(sdk_client, deleted_docs_to_verify)

    elif xattrs_enabled and deletion_type == 'purge':

        # Verify SDK + SG docs are purged from Sync Gateway
        verify_sg_purges(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth)

        # Verify SDK + SG docs are deleted from SDK
        verify_sdk_deletes(sdk_client, deleted_docs_to_verify)

    elif not xattrs_enabled and deletion_type == 'tombstone':

        # Doc meta: Verify SG docs are all deleted via SG
        verify_sg_deletes(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth)

    elif not xattrs_enabled and deletion_type == 'purge':

        # Doc meta: Verify SG docs are all deleted via SG
        verify_sg_purges(sg_client, sg_url, sg_db, deleted_docs_to_verify, sg_user_auth)

    else:
        raise ValueError('Invalid test parameters')

    # Recreate deleted docs from Sync Gateway
    sg_bulk_docs_resp = sg_client.add_bulk_docs(url=sg_url, db=sg_db, docs=sg_doc_bodies, auth=sg_user_auth)
    assert len(sg_bulk_docs_resp) == num_docs_per_client

    if xattrs_enabled:
        log_info('Recreating SDK docs')
        # Recreate deleted docs from SDK
        sdk_client.upsert_multi(sdk_docs)

    # Get docs via Sync Gateway
    doc_ids_to_get = sg_doc_ids
    if xattrs_enabled:
        doc_ids_to_get = sg_doc_ids + sdk_doc_ids
    docs, errors = sg_client.get_bulk_docs(
        url=sg_url,
        db=sg_db,
        doc_ids=doc_ids_to_get,
        auth=sg_user_auth,
        validate=False
    )
    if xattrs_enabled:
        assert len(docs) == num_docs_per_client * 2
        assert len(errors) == 0
    else:
        assert len(docs) == num_docs_per_client
        assert len(errors) == 0

    if xattrs_enabled:

        # Get SDK docs and makes sure all docs were recreated
        all_docs_from_sdk = sdk_client.get_multi(doc_ids_to_get)
        assert len(all_docs_from_sdk) == num_docs_per_client * 2
        log_info('Found: {} recreated docs via SDK'.format(len(all_docs_from_sdk)))

        # Make sure we are able to get recreated docs via SDK
        doc_ids_to_get_scratch = list(doc_ids_to_get)
        assert len(doc_ids_to_get_scratch) == num_docs_per_client * 2
        for doc_id in all_docs_from_sdk:
            doc_ids_to_get_scratch.remove(doc_id)
        assert len(doc_ids_to_get_scratch) == 0

    # Make sure we are able to get recreated docs via SDK
    doc_ids_to_get_scratch = list(doc_ids_to_get)
    if xattrs_enabled:
        # SG + SDK docs
        assert len(doc_ids_to_get_scratch) == num_docs_per_client * 2
    else:
        # SG docs
        assert len(doc_ids_to_get_scratch) == num_docs_per_client

    for doc in docs:
        # Verify expected document revisions
        if xattrs_enabled:
            if deletion_type == 'purge':
                # SG purges SG docs and recreates them, expecting 1- rev
                # SDK removes SDK docs and recreates them, expecting 1- rev
                assert doc['_rev'].startswith('1-')
            else:
                # SG tombstones SG docs and recreates them, expecting 3- rev
                # SDK removes SDK docs and recreates them, expecting 1- rev
                if doc['_id'].startswith('sg_'):
                    assert doc['_rev'].startswith('3-')
                else:
                    assert doc['_rev'].startswith('1-')
        else:
            if deletion_type == 'purge':
                # SG purges SG docs and recreates them, expecting 1- rev
                assert doc['_rev'].startswith('1-')
            else:
                # SG tombstones SG docs and recreates them, expecting 3- rev
                assert doc['_rev'].startswith('3-')

        doc_ids_to_get_scratch.remove(doc['_id'])

    # Make sure all docs were found
    assert len(doc_ids_to_get_scratch) == 0