예제 #1
0
    def test_04_sync_many_accounts_many_containers_and_obj_meta(self):
        """Many accounts with many containers and some object
        """
        index = {}
        index_container = {}
        # Create account
        self.created = filler.create_swift_account(self.o_ks_client,
                                                   self.pile,
                                                   1, 1, index)

        for account, account_id, username in \
                self.extract_created_a_u_iter(self.created):
            tenant_cnx = sclient.Connection(self.o_st,
                                            "%s:%s" % (account, username),
                                            self.default_user_password,
                                            auth_version=2)
            acc = (account, account_id)
            filler.create_containers(tenant_cnx, acc, 1, index_container)
            filler.create_objects(tenant_cnx, acc, 1, 2048, index_container)

        # Start sync process
        self.swsync.process()

        # Now verify dest
        for account, account_id, username in \
                self.extract_created_a_u_iter(self.created):
            # Verify container listing
            olo = self.list_objects_in_containers(account_id,
                                                  self.o_admin_token, 'orig')
            old = self.list_objects_in_containers(account_id,
                                                  self.d_admin_token, 'dest')

            # Verify we have the same amount of container
            self.assertListEqual(olo.keys(), old.keys())
            # For each container
            for c, objs in olo.items():
                for obj in objs:
                    # Verify first object detail returned by container
                    # server
                    match = [od for od in old[c] if od['name'] == obj['name']]
                    self.assertEqual(len(match), 1)
                    obj_d = match[0]
                    self.assertDictEqual(obj, obj_d)
                # Verify object details from object server
                obj_names = [d['name'] for d in olo[c]]
                for obj_name in obj_names:
                    objd_o = self.get_object_detail(account_id,
                                                    self.o_admin_token, 'orig',
                                                    c, obj_name)
                    objd_d = self.get_object_detail(account_id,
                                                    self.d_admin_token, 'dest',
                                                    c, obj_name)
                    self.verify_aco_diff(objd_o, objd_d)
                    # Verify content
                    self.assertEqual(objd_o[1], objd_d[1])
예제 #2
0
    def test_08_sync_containers_with_last_modified(self):
        """Containers with last-modified middleware."""
        index = {}
        index_container = {}
        # Create account
        self.created = filler.create_swift_account(self.o_ks_client,
                                                   self.pile,
                                                   1, 1, index)

        # Create container and store new account && container
        account_dest, container_dest = None, None
        for account, account_id, username in (
                self.extract_created_a_u_iter(self.created)):
            tenant_cnx = sclient.Connection(self.o_st,
                                            "%s:%s" % (account, username),
                                            self.default_user_password,
                                            auth_version=2)
            acc = (account, account_id)
            filler.create_containers(tenant_cnx, acc, 1, index_container)
            filler.create_objects(tenant_cnx, acc, 1, 2048, index_container)
            cld = self.list_containers(account_id,
                                       self.d_admin_token, 'orig')
            account_dest = account_id
            container_dest = cld[0]['name']
            break

        # Start sync process
        self.swsync.process()

        # Update dest
        self.put_object(account_dest, self.d_admin_token, 'dest',
                        container_dest, 'lm-test', 'lm-data')

        # Get timestamp
        cdd = self.get_container_detail(account_dest, self.d_admin_token,
                                        'dest', container_dest)
        try:
            dest_lm = cdd[0]['x-container-meta-last-modified']
        except(KeyError):
            # Last-modified middleware is not present
            return

        # Restart sync process
        self.swsync.process()

        # Check if dest timestamp have not been updated
        cdd = self.get_container_detail(account_dest, self.d_admin_token,
                                        'dest', container_dest)
        self.assertEqual(dest_lm, cdd[0]['x-container-meta-last-modified'])

        # Check if new object is still present in dest
        obj_detail = self.get_object_detail(account_dest, self.d_admin_token,
                                            'dest', container_dest, 'lm-test')
        self.assertEqual('lm-data', obj_detail[1])
예제 #3
0
    def test_08_sync_containers_with_last_modified(self):
        """Containers with last-modified middleware
        """
        index = {}
        index_container = {}
        # Create account
        self.created = filler.create_swift_account(self.o_ks_client, self.pile,
                                                   1, 1, index)

        # Create container and store new account && container
        account_dest, container_dest = None, None
        for account, account_id, username in \
                self.extract_created_a_u_iter(self.created):
            tenant_cnx = sclient.Connection(self.o_st,
                                            "%s:%s" % (account, username),
                                            self.default_user_password,
                                            auth_version=2)
            acc = (account, account_id)
            filler.create_containers(tenant_cnx, acc, 1, index_container)
            filler.create_objects(tenant_cnx, acc, 1, 2048, index_container)
            cld = self.list_containers(account_id, self.d_admin_token, 'orig')
            account_dest = account_id
            container_dest = cld[0]['name']
            break

        # Start sync process
        self.swsync.process()

        # Update dest
        self.put_object(account_dest, self.d_admin_token, 'dest',
                        container_dest, 'lm-test', 'lm-data')

        # Get timestamp
        cdd = self.get_container_detail(account_dest, self.d_admin_token,
                                        'dest', container_dest)
        try:
            dest_lm = cdd[0]['x-container-meta-last-modified']
        except (KeyError):
            # Last-modified middleware is not present
            return

        # Restart sync process
        self.swsync.process()

        # Check if dest timestamp have not been updated
        cdd = self.get_container_detail(account_dest, self.d_admin_token,
                                        'dest', container_dest)
        self.assertEqual(dest_lm, cdd[0]['x-container-meta-last-modified'])

        # Check if new object is still present in dest
        obj_detail = self.get_object_detail(account_dest, self.d_admin_token,
                                            'dest', container_dest, 'lm-test')
        self.assertEqual('lm-data', obj_detail[1])
예제 #4
0
    def test_create_containers(self):
        get_containers_created = []
        return_dict_ref = {}

        def put_container(*args, **kwargs):
            get_containers_created.append(args[1])

        self.stubs.Set(FakeSWConnection, 'put_container', put_container)
        cnx = self.get_connection()
        filler.create_containers(cnx, 'test', 3, return_dict_ref)
        self.assertEqual(len(get_containers_created), 3)
        self.assertEqual(get_containers_created[0].split('_')[0], 'container')
        meta_amount = len(return_dict_ref['test'].values())
        self.assertEqual(meta_amount, 3)
예제 #5
0
    def test_create_containers(self):
        get_containers_created = []
        return_dict_ref = {}

        def put_container(*args, **kwargs):
            get_containers_created.append(args[1])

        self.stubs.Set(FakeSWConnection, 'put_container', put_container)
        cnx = self.get_connection()
        filler.create_containers(cnx, 'test', 3, return_dict_ref)
        self.assertEqual(len(get_containers_created), 3)
        self.assertEqual(get_containers_created[0].split('_')[0],
                         'container')
        meta_amount = len(return_dict_ref['test'].values())
        self.assertEqual(meta_amount, 3)
예제 #6
0
    def test_create_containers_fail(self):
        get_containers_created = []
        return_dict_ref = {}
        self.attempts = 0

        def put_container(*args, **kwargs):
            if self.attempts == 0:
                self.attempts += 1
                raise swiftclient.client.ClientException('Fake err msg')
            else:
                self.attempts += 1
                get_containers_created.append(args[1])

        self.stubs.Set(FakeSWConnection, 'put_container', put_container)
        cnx = self.get_connection()
        filler.create_containers(cnx, 'test', 3, return_dict_ref)

        self.assertEqual(len(get_containers_created), 2)
예제 #7
0
    def test_create_containers_fail(self):
        get_containers_created = []
        return_dict_ref = {}
        self.attempts = 0

        def put_container(*args, **kwargs):
            if self.attempts == 0:
                self.attempts += 1
                raise swiftclient.client.ClientException('Fake err msg')
            else:
                self.attempts += 1
                get_containers_created.append(args[1])

        self.stubs.Set(FakeSWConnection, 'put_container', put_container)
        cnx = self.get_connection()
        filler.create_containers(cnx, 'test', 3, return_dict_ref)

        self.assertEqual(len(get_containers_created), 2)
예제 #8
0
    def test_03_sync_many_accounts_with_many_containers_meta(self):
        """Many accounts with many containers and container meta data
        """
        index = {}
        index_container = {}
        # Create account
        self.created = filler.create_swift_account(self.o_ks_client,
                                                   self.pile,
                                                   3, 1, index)

        for account, account_id, username in \
                self.extract_created_a_u_iter(self.created):
            tenant_cnx = sclient.Connection(self.o_st,
                                            "%s:%s" % (account, username),
                                            self.default_user_password,
                                            auth_version=2)
            acc = (account, account_id)
            filler.create_containers(tenant_cnx, acc, 3, index_container)

        # Start sync process
        self.swsync.process()

        # Now verify dest
        for account, account_id, username in \
                self.extract_created_a_u_iter(self.created):
            # Verify container listing
            clo = self.list_containers(account_id,
                                       self.o_admin_token, 'orig')
            cld = self.list_containers(account_id,
                                       self.d_admin_token, 'dest')
            self.assertEqual(len(clo), len(cld))
            for do in clo:
                match = [dd for dd in cld if dd['name'] == do['name']]
                self.assertEqual(len(match), 1)
                self.assertDictEqual(do, match[0])
            # Verify container details
            clo_c_names = [d['name'] for d in clo]
            for c_name in clo_c_names:
                cdo = self.get_container_detail(account_id, self.o_admin_token,
                                                'orig', c_name)
                cdd = self.get_container_detail(account_id, self.d_admin_token,
                                                'dest', c_name)
            self.verify_aco_diff(cdo, cdd)
예제 #9
0
    def test_07_object_two_passes(self):
        """Objects modified two passes."""
        index = {}
        index_container = {}
        # Create account
        self.created = filler.create_swift_account(self.o_ks_client,
                                                   self.pile,
                                                   1, 1, index)

        for account, account_id, username in (
                self.extract_created_a_u_iter(self.created)):
            tenant_cnx = sclient.Connection(self.o_st,
                                            "%s:%s" % (account, username),
                                            self.default_user_password,
                                            auth_version=2)
            acc = (account, account_id)
            filler.create_containers(tenant_cnx, acc, 1, index_container)
            filler.create_objects(tenant_cnx, acc, 3, 2048, index_container)

        # Start sync process
        self.swsync.process()

        # Modify objects in containers
        for account, account_id, username in (
                self.extract_created_a_u_iter(self.created)):
            tenant_cnx = sclient.Connection(self.o_st,
                                            "%s:%s" % (account, username),
                                            self.default_user_password,
                                            auth_version=2)

            token = tenant_cnx.get_auth()[1]
            c_o = self.list_objects_in_containers(account_id, token, 'orig')
            for cont, objs in c_o.iteritems():
                for obj in objs:
                    # Modify object meta
                    obj_d, data = self.get_object_detail(account_id,
                                                         token, 'orig',
                                                         cont, obj['name'])
                    meta = {k: v for k, v in obj_d.iteritems()
                            if k.startswith('x-object-meta')}
                    meta_k_names = [k.split('-')[-1] for k in meta]
                    headers = {}
                    headers['X-Object-Meta-a1'] = 'b1'
                    headers["X-Remove-Object-Meta-%s" % meta_k_names[0]] = 'x'
                    headers["X-Object-Meta-%s" % meta_k_names[1]] = 'b2'
                    self.post_object(account_id, token,
                                     'orig',
                                     headers=headers,
                                     container=cont,
                                     name=obj['name'])
                # Create an object
                self.put_object(account_id, token, 'orig',
                                cont, 'foofoo', 'barbarbar')
                self.put_object(account_id, token, 'orig',
                                cont, 'foofoo1', 'barbarbar')
                self.put_object(account_id, token, 'orig',
                                cont, 'foofoo2', 'barbarbar')

                o_names = [o['name'] for o in objs]
                # Delete an object
                name = o_names[0]
                self.delete_object(account_id, token, 'orig',
                                   cont, name)

        # Start sync process
        self.swsync.process()

        # Now verify dest
        for account, account_id, username in (
                self.extract_created_a_u_iter(self.created)):
            # Verify container listing
            olo = self.list_objects_in_containers(account_id,
                                                  self.o_admin_token, 'orig')
            old = self.list_objects_in_containers(account_id,
                                                  self.d_admin_token, 'dest')

            # Verify we have the same amount of container
            self.assertListEqual(olo.keys(), old.keys())
            # For each container
            for c, objs in olo.items():
                for obj in objs:
                    # Verify first object detail returned by container server
                    match = [od for od in old[c] if od['name'] == obj['name']]
                    self.assertEqual(len(match), 1)
                    obj_d = match[0]
                    a = obj.copy()
                    b = obj_d.copy()
                    del a['last_modified']
                    del b['last_modified']
                    self.assertDictEqual(a, b)
                # Verify object details from object server
                obj_names = [d['name'] for d in olo[c]]
                for obj_name in obj_names:
                    objd_o = self.get_object_detail(account_id,
                                                    self.o_admin_token, 'orig',
                                                    c, obj_name)
                    objd_d = self.get_object_detail(account_id,
                                                    self.d_admin_token, 'dest',
                                                    c, obj_name)
                    self.verify_aco_diff(objd_o, objd_d)
                    # Verify content
                    self.assertEqual(objd_o[1], objd_d[1])
예제 #10
0
    def test_06_container_two_passes(self):
        """Containers modified two sync passes."""
        index = {}
        index_container = {}
        # Create account
        self.created = filler.create_swift_account(self.o_ks_client,
                                                   self.pile,
                                                   3, 1, index)

        for account, account_id, username in (
                self.extract_created_a_u_iter(self.created)):
            tenant_cnx = sclient.Connection(self.o_st,
                                            "%s:%s" % (account, username),
                                            self.default_user_password,
                                            auth_version=2)
            acc = (account, account_id)
            filler.create_containers(tenant_cnx, acc, 3, index_container)

        # Start sync process
        self.swsync.process()

        # Modify container in account
        for account, account_id, username in (
                self.extract_created_a_u_iter(self.created)):
            tenant_cnx = sclient.Connection(self.o_st,
                                            "%s:%s" % (account, username),
                                            self.default_user_password,
                                            auth_version=2)

            token = tenant_cnx.get_auth()[1]
            # Modify an existing container meta
            clo = self.list_containers(account_id,
                                       token, 'orig')
            co_name = clo[0]['name']
            c_meta = self.get_container_meta(account_id,
                                             token, 'orig',
                                             co_name)
            c_meta_k_names = [k.split('-')[-1] for k in c_meta]
            headers = {}
            headers['X-Container-Meta-a1'] = 'b1'
            headers["X-Remove-Container-Meta-%s" % c_meta_k_names[0]] = 'x'
            headers["X-Container-Meta-%s" % c_meta_k_names[1]] = 'b2'
            self.post_container(account_id, token,
                                'orig',
                                headers=headers,
                                container=co_name)
            # Add a some more container
            self.put_container(account_id, token, 'orig', 'foobar')
            self.put_container(account_id, token, 'orig', 'foobar1')
            self.put_container(account_id, token, 'orig', 'foobar2')
            # Delete one container
            co_name = clo[1]['name']
            self.delete_container(account_id, token, 'orig', co_name)

        # Re - Start sync process
        self.swsync.process()

        # Now verify dest
        for account, account_id, username in (
                self.extract_created_a_u_iter(self.created)):
            # Verify container listing
            clo = self.list_containers(account_id,
                                       self.o_admin_token, 'orig')
            cld = self.list_containers(account_id,
                                       self.d_admin_token, 'dest')
            self.assertEqual(len(clo), len(cld))
            for do in clo:
                match = [dd for dd in cld if dd['name'] == do['name']]
                self.assertEqual(len(match), 1)
                self.assertDictEqual(do, match[0])
            # Verify container details
            clo_c_names = [d['name'] for d in clo]
            for c_name in clo_c_names:
                cdo = self.get_container_detail(account_id, self.o_admin_token,
                                                'orig', c_name)
                cdd = self.get_container_detail(account_id, self.d_admin_token,
                                                'dest', c_name)
                self.verify_aco_diff(cdo, cdd)
예제 #11
0
    def test_07_object_two_passes(self):
        """Objects modified two passes
        """
        index = {}
        index_container = {}
        # Create account
        self.created = filler.create_swift_account(self.o_ks_client,
                                                   self.pile,
                                                   1, 1, index)

        for account, account_id, username in \
                self.extract_created_a_u_iter(self.created):
            tenant_cnx = sclient.Connection(self.o_st,
                                            "%s:%s" % (account, username),
                                            self.default_user_password,
                                            auth_version=2)
            acc = (account, account_id)
            filler.create_containers(tenant_cnx, acc, 1, index_container)
            filler.create_objects(tenant_cnx, acc, 3, 2048, index_container)

        # Start sync process
        self.swsync.process()

        # Modify objects in containers
        for account, account_id, username in \
                self.extract_created_a_u_iter(self.created):
            tenant_cnx = sclient.Connection(self.o_st,
                                            "%s:%s" % (account, username),
                                            self.default_user_password,
                                            auth_version=2)

            token = tenant_cnx.get_auth()[1]
            c_o = self.list_objects_in_containers(account_id, token, 'orig')
            for cont, objs in c_o.iteritems():
                for obj in objs:
                    # Modify object meta
                    obj_d, data = self.get_object_detail(account_id,
                                                         token, 'orig',
                                                         cont, obj['name'])
                    meta = {k: v for k, v in obj_d.iteritems()
                            if k.startswith('x-object-meta')}
                    meta_k_names = [k.split('-')[-1] for k in meta]
                    headers = {}
                    headers['X-Object-Meta-a1'] = 'b1'
                    headers["X-Remove-Object-Meta-%s" % meta_k_names[0]] = 'x'
                    headers["X-Object-Meta-%s" % meta_k_names[1]] = 'b2'
                    self.post_object(account_id, token,
                                     'orig',
                                     headers=headers,
                                     container=cont,
                                     name=obj['name'])
                # Create an object
                self.put_object(account_id, token, 'orig',
                                cont, 'foofoo', 'barbarbar')
                self.put_object(account_id, token, 'orig',
                                cont, 'foofoo1', 'barbarbar')
                self.put_object(account_id, token, 'orig',
                                cont, 'foofoo2', 'barbarbar')

                o_names = [o['name'] for o in objs]
                # Delete an object
                name = o_names[0]
                self.delete_object(account_id, token, 'orig',
                                   cont, name)

        # Start sync process
        self.swsync.process()

        # Now verify dest
        for account, account_id, username in \
                self.extract_created_a_u_iter(self.created):
            # Verify container listing
            olo = self.list_objects_in_containers(account_id,
                                                  self.o_admin_token, 'orig')
            old = self.list_objects_in_containers(account_id,
                                                  self.d_admin_token, 'dest')

            # Verify we have the same amount of container
            self.assertListEqual(olo.keys(), old.keys())
            # For each container
            for c, objs in olo.items():
                for obj in objs:
                    # Verify first object detail returned by container server
                    match = [od for od in old[c] if od['name'] == obj['name']]
                    self.assertEqual(len(match), 1)
                    obj_d = match[0]
                    a = obj.copy()
                    b = obj_d.copy()
                    del a['last_modified']
                    del b['last_modified']
                    self.assertDictEqual(a, b)
                # Verify object details from object server
                obj_names = [d['name'] for d in olo[c]]
                for obj_name in obj_names:
                    objd_o = self.get_object_detail(account_id,
                                                    self.o_admin_token, 'orig',
                                                    c, obj_name)
                    objd_d = self.get_object_detail(account_id,
                                                    self.d_admin_token, 'dest',
                                                    c, obj_name)
                    self.verify_aco_diff(objd_o, objd_d)
                    # Verify content
                    self.assertEqual(objd_o[1], objd_d[1])
예제 #12
0
    def test_06_container_two_passes(self):
        """Containers modified two sync passes
        """
        index = {}
        index_container = {}
        # Create account
        self.created = filler.create_swift_account(self.o_ks_client,
                                                   self.pile,
                                                   3, 1, index)

        for account, account_id, username in \
                self.extract_created_a_u_iter(self.created):
            tenant_cnx = sclient.Connection(self.o_st,
                                            "%s:%s" % (account, username),
                                            self.default_user_password,
                                            auth_version=2)
            acc = (account, account_id)
            filler.create_containers(tenant_cnx, acc, 3, index_container)

        # Start sync process
        self.swsync.process()

        # Modify container in account
        for account, account_id, username in \
                self.extract_created_a_u_iter(self.created):
            tenant_cnx = sclient.Connection(self.o_st,
                                            "%s:%s" % (account, username),
                                            self.default_user_password,
                                            auth_version=2)

            token = tenant_cnx.get_auth()[1]
            # Modify an existing container meta
            clo = self.list_containers(account_id,
                                       token, 'orig')
            co_name = clo[0]['name']
            c_meta = self.get_container_meta(account_id,
                                             token, 'orig',
                                             co_name)
            c_meta_k_names = [k.split('-')[-1] for k in c_meta]
            headers = {}
            headers['X-Container-Meta-a1'] = 'b1'
            headers["X-Remove-Container-Meta-%s" % c_meta_k_names[0]] = 'x'
            headers["X-Container-Meta-%s" % c_meta_k_names[1]] = 'b2'
            self.post_container(account_id, token,
                                'orig',
                                headers=headers,
                                container=co_name)
            # Add a some more container
            self.put_container(account_id, token, 'orig', 'foobar')
            self.put_container(account_id, token, 'orig', 'foobar1')
            self.put_container(account_id, token, 'orig', 'foobar2')
            # Delete one container
            co_name = clo[1]['name']
            self.delete_container(account_id, token, 'orig', co_name)

        # Re - Start sync process
        self.swsync.process()

        # Now verify dest
        for account, account_id, username in \
                self.extract_created_a_u_iter(self.created):
            # Verify container listing
            clo = self.list_containers(account_id,
                                       self.o_admin_token, 'orig')
            cld = self.list_containers(account_id,
                                       self.d_admin_token, 'dest')
            self.assertEqual(len(clo), len(cld))
            for do in clo:
                match = [dd for dd in cld if dd['name'] == do['name']]
                self.assertEqual(len(match), 1)
                self.assertDictEqual(do, match[0])
            # Verify container details
            clo_c_names = [d['name'] for d in clo]
            for c_name in clo_c_names:
                cdo = self.get_container_detail(account_id, self.o_admin_token,
                                                'orig', c_name)
                cdd = self.get_container_detail(account_id, self.d_admin_token,
                                                'dest', c_name)
                self.verify_aco_diff(cdo, cdd)