def test_04_sync_many_accounts_many_containers_and_obj_meta(self): """Many accounts with many containers and some object """ index = {} index_container = {} # Create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 1, 1, index) for account, account_id, username in \ self.extract_created_a_u_iter(self.created): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) acc = (account, account_id) filler.create_containers(tenant_cnx, acc, 1, index_container) filler.create_objects(tenant_cnx, acc, 1, 2048, index_container) # Start sync process self.swsync.process() # Now verify dest for account, account_id, username in \ self.extract_created_a_u_iter(self.created): # Verify container listing olo = self.list_objects_in_containers(account_id, self.o_admin_token, 'orig') old = self.list_objects_in_containers(account_id, self.d_admin_token, 'dest') # Verify we have the same amount of container self.assertListEqual(olo.keys(), old.keys()) # For each container for c, objs in olo.items(): for obj in objs: # Verify first object detail returned by container # server match = [od for od in old[c] if od['name'] == obj['name']] self.assertEqual(len(match), 1) obj_d = match[0] self.assertDictEqual(obj, obj_d) # Verify object details from object server obj_names = [d['name'] for d in olo[c]] for obj_name in obj_names: objd_o = self.get_object_detail(account_id, self.o_admin_token, 'orig', c, obj_name) objd_d = self.get_object_detail(account_id, self.d_admin_token, 'dest', c, obj_name) self.verify_aco_diff(objd_o, objd_d) # Verify content self.assertEqual(objd_o[1], objd_d[1])
def test_08_sync_containers_with_last_modified(self): """Containers with last-modified middleware.""" index = {} index_container = {} # Create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 1, 1, index) # Create container and store new account && container account_dest, container_dest = None, None for account, account_id, username in ( self.extract_created_a_u_iter(self.created)): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) acc = (account, account_id) filler.create_containers(tenant_cnx, acc, 1, index_container) filler.create_objects(tenant_cnx, acc, 1, 2048, index_container) cld = self.list_containers(account_id, self.d_admin_token, 'orig') account_dest = account_id container_dest = cld[0]['name'] break # Start sync process self.swsync.process() # Update dest self.put_object(account_dest, self.d_admin_token, 'dest', container_dest, 'lm-test', 'lm-data') # Get timestamp cdd = self.get_container_detail(account_dest, self.d_admin_token, 'dest', container_dest) try: dest_lm = cdd[0]['x-container-meta-last-modified'] except(KeyError): # Last-modified middleware is not present return # Restart sync process self.swsync.process() # Check if dest timestamp have not been updated cdd = self.get_container_detail(account_dest, self.d_admin_token, 'dest', container_dest) self.assertEqual(dest_lm, cdd[0]['x-container-meta-last-modified']) # Check if new object is still present in dest obj_detail = self.get_object_detail(account_dest, self.d_admin_token, 'dest', container_dest, 'lm-test') self.assertEqual('lm-data', obj_detail[1])
def test_08_sync_containers_with_last_modified(self): """Containers with last-modified middleware """ index = {} index_container = {} # Create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 1, 1, index) # Create container and store new account && container account_dest, container_dest = None, None for account, account_id, username in \ self.extract_created_a_u_iter(self.created): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) acc = (account, account_id) filler.create_containers(tenant_cnx, acc, 1, index_container) filler.create_objects(tenant_cnx, acc, 1, 2048, index_container) cld = self.list_containers(account_id, self.d_admin_token, 'orig') account_dest = account_id container_dest = cld[0]['name'] break # Start sync process self.swsync.process() # Update dest self.put_object(account_dest, self.d_admin_token, 'dest', container_dest, 'lm-test', 'lm-data') # Get timestamp cdd = self.get_container_detail(account_dest, self.d_admin_token, 'dest', container_dest) try: dest_lm = cdd[0]['x-container-meta-last-modified'] except (KeyError): # Last-modified middleware is not present return # Restart sync process self.swsync.process() # Check if dest timestamp have not been updated cdd = self.get_container_detail(account_dest, self.d_admin_token, 'dest', container_dest) self.assertEqual(dest_lm, cdd[0]['x-container-meta-last-modified']) # Check if new object is still present in dest obj_detail = self.get_object_detail(account_dest, self.d_admin_token, 'dest', container_dest, 'lm-test') self.assertEqual('lm-data', obj_detail[1])
def test_create_objects(self): get_object_created = [] return_dict_ref = {'test': {'container_a': {'objects': []}, 'container_b': {'objects': []}}} def put_object(*args, **kwargs): get_object_created.append(args[1:]) self.stubs.Set(FakeSWConnection, 'put_object', put_object) cnx = self.get_connection() filler.create_objects(cnx, 'test', 2, 2048, return_dict_ref) objects_ca = return_dict_ref['test']['container_a']['objects'] objects_cb = return_dict_ref['test']['container_b']['objects'] self.assertEqual(len(objects_ca), 2) self.assertEqual(len(objects_cb), 2)
def test_create_objects_fail(self): get_object_created = [] return_dict_ref = {'test': {'container_a': {'objects': []}}} self.attempts = 0 def put_object(*args, **kwargs): if self.attempts == 0: self.attempts += 1 raise swiftclient.client.ClientException('Fake err msg') else: self.attempts += 1 get_object_created.append(args[1:]) self.stubs.Set(FakeSWConnection, 'put_object', put_object) cnx = self.get_connection() filler.create_objects(cnx, 'test', 2, 2048, return_dict_ref) objects_ca = return_dict_ref['test']['container_a']['objects'] self.assertEqual(len(objects_ca), 1)
def test_create_objects_fail(self): get_object_created = [] return_dict_ref = {'test': {'container_a': {'objects': []}}} self.attempts = 0 def put_object(*args, **kwargs): if self.attempts == 0: self.attempts += 1 raise swiftclient.client.ClientException('Fake err msg') else: self.attempts += 1 get_object_created.append(args[1:]) self.stubs.Set(FakeSWConnection, 'put_object', put_object) cnx = self.get_connection() filler.create_objects(cnx, 'test', 2, 2048, return_dict_ref) objects_ca = return_dict_ref['test']['container_a']['objects'] self.assertEqual(len(objects_ca), 1)
def test_create_objects(self): get_object_created = [] return_dict_ref = { 'test': { 'container_a': { 'objects': [] }, 'container_b': { 'objects': [] } } } def put_object(*args, **kwargs): get_object_created.append(args[1:]) self.stubs.Set(FakeSWConnection, 'put_object', put_object) cnx = self.get_connection() filler.create_objects(cnx, 'test', 2, 2048, return_dict_ref) objects_ca = return_dict_ref['test']['container_a']['objects'] objects_cb = return_dict_ref['test']['container_b']['objects'] self.assertEqual(len(objects_ca), 2) self.assertEqual(len(objects_cb), 2)
def test_07_object_two_passes(self): """Objects modified two passes.""" index = {} index_container = {} # Create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 1, 1, index) for account, account_id, username in ( self.extract_created_a_u_iter(self.created)): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) acc = (account, account_id) filler.create_containers(tenant_cnx, acc, 1, index_container) filler.create_objects(tenant_cnx, acc, 3, 2048, index_container) # Start sync process self.swsync.process() # Modify objects in containers for account, account_id, username in ( self.extract_created_a_u_iter(self.created)): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) token = tenant_cnx.get_auth()[1] c_o = self.list_objects_in_containers(account_id, token, 'orig') for cont, objs in c_o.iteritems(): for obj in objs: # Modify object meta obj_d, data = self.get_object_detail(account_id, token, 'orig', cont, obj['name']) meta = {k: v for k, v in obj_d.iteritems() if k.startswith('x-object-meta')} meta_k_names = [k.split('-')[-1] for k in meta] headers = {} headers['X-Object-Meta-a1'] = 'b1' headers["X-Remove-Object-Meta-%s" % meta_k_names[0]] = 'x' headers["X-Object-Meta-%s" % meta_k_names[1]] = 'b2' self.post_object(account_id, token, 'orig', headers=headers, container=cont, name=obj['name']) # Create an object self.put_object(account_id, token, 'orig', cont, 'foofoo', 'barbarbar') self.put_object(account_id, token, 'orig', cont, 'foofoo1', 'barbarbar') self.put_object(account_id, token, 'orig', cont, 'foofoo2', 'barbarbar') o_names = [o['name'] for o in objs] # Delete an object name = o_names[0] self.delete_object(account_id, token, 'orig', cont, name) # Start sync process self.swsync.process() # Now verify dest for account, account_id, username in ( self.extract_created_a_u_iter(self.created)): # Verify container listing olo = self.list_objects_in_containers(account_id, self.o_admin_token, 'orig') old = self.list_objects_in_containers(account_id, self.d_admin_token, 'dest') # Verify we have the same amount of container self.assertListEqual(olo.keys(), old.keys()) # For each container for c, objs in olo.items(): for obj in objs: # Verify first object detail returned by container server match = [od for od in old[c] if od['name'] == obj['name']] self.assertEqual(len(match), 1) obj_d = match[0] a = obj.copy() b = obj_d.copy() del a['last_modified'] del b['last_modified'] self.assertDictEqual(a, b) # Verify object details from object server obj_names = [d['name'] for d in olo[c]] for obj_name in obj_names: objd_o = self.get_object_detail(account_id, self.o_admin_token, 'orig', c, obj_name) objd_d = self.get_object_detail(account_id, self.d_admin_token, 'dest', c, obj_name) self.verify_aco_diff(objd_o, objd_d) # Verify content self.assertEqual(objd_o[1], objd_d[1])
def test_07_object_two_passes(self): """Objects modified two passes """ index = {} index_container = {} # Create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 1, 1, index) for account, account_id, username in \ self.extract_created_a_u_iter(self.created): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) acc = (account, account_id) filler.create_containers(tenant_cnx, acc, 1, index_container) filler.create_objects(tenant_cnx, acc, 3, 2048, index_container) # Start sync process self.swsync.process() # Modify objects in containers for account, account_id, username in \ self.extract_created_a_u_iter(self.created): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) token = tenant_cnx.get_auth()[1] c_o = self.list_objects_in_containers(account_id, token, 'orig') for cont, objs in c_o.iteritems(): for obj in objs: # Modify object meta obj_d, data = self.get_object_detail(account_id, token, 'orig', cont, obj['name']) meta = {k: v for k, v in obj_d.iteritems() if k.startswith('x-object-meta')} meta_k_names = [k.split('-')[-1] for k in meta] headers = {} headers['X-Object-Meta-a1'] = 'b1' headers["X-Remove-Object-Meta-%s" % meta_k_names[0]] = 'x' headers["X-Object-Meta-%s" % meta_k_names[1]] = 'b2' self.post_object(account_id, token, 'orig', headers=headers, container=cont, name=obj['name']) # Create an object self.put_object(account_id, token, 'orig', cont, 'foofoo', 'barbarbar') self.put_object(account_id, token, 'orig', cont, 'foofoo1', 'barbarbar') self.put_object(account_id, token, 'orig', cont, 'foofoo2', 'barbarbar') o_names = [o['name'] for o in objs] # Delete an object name = o_names[0] self.delete_object(account_id, token, 'orig', cont, name) # Start sync process self.swsync.process() # Now verify dest for account, account_id, username in \ self.extract_created_a_u_iter(self.created): # Verify container listing olo = self.list_objects_in_containers(account_id, self.o_admin_token, 'orig') old = self.list_objects_in_containers(account_id, self.d_admin_token, 'dest') # Verify we have the same amount of container self.assertListEqual(olo.keys(), old.keys()) # For each container for c, objs in olo.items(): for obj in objs: # Verify first object detail returned by container server match = [od for od in old[c] if od['name'] == obj['name']] self.assertEqual(len(match), 1) obj_d = match[0] a = obj.copy() b = obj_d.copy() del a['last_modified'] del b['last_modified'] self.assertDictEqual(a, b) # Verify object details from object server obj_names = [d['name'] for d in olo[c]] for obj_name in obj_names: objd_o = self.get_object_detail(account_id, self.o_admin_token, 'orig', c, obj_name) objd_d = self.get_object_detail(account_id, self.d_admin_token, 'dest', c, obj_name) self.verify_aco_diff(objd_o, objd_d) # Verify content self.assertEqual(objd_o[1], objd_d[1])