def test_create_swift_account_fail(self): self.ret_index = {} self.pa = 0 def create_tenant(*args): if self.pa == 0: self.pa += 1 raise KSClientException('Fake msg') else: self.pa += 1 return FakeKSTenant('foo1') def create_swift_user(*args): pass client = FakeKSClient() self.stubs.Set(client.tenants, 'create', create_tenant) self.stubs.Set(filler, 'create_swift_user', create_swift_user) concurrency = int(utils.get_config('concurrency', 'filler_keystone_client_concurrency')) pile = eventlet.GreenPile(concurrency) filler.create_swift_account(client, pile, 3, 1, self.ret_index) self.assertEqual(len(self.ret_index.keys()), 2)
def test_02_sync_many_empty_account(self): """Many empty account with meta data.""" index = {} # Create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 3, 1, index) for account, account_id, username in ( self.extract_created_a_u_iter(self.created)): # Post meta data on account tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) filler.create_account_meta(tenant_cnx) # Start sync process self.swsync.process() # Now verify dest for account, account_id, username in ( self.extract_created_a_u_iter(self.created)): alo = self.get_account_detail(account_id, self.o_admin_token, 'orig') ald = self.get_account_detail(account_id, self.d_admin_token, 'dest') self.verify_aco_diff(alo, ald)
def test_create_swift_account(self): self.ret_index = {} self.user_cnt = 0 def create_swift_user(*args): self.user_cnt += 1 self.stubs.Set(filler, 'create_swift_user', create_swift_user) concurrency = int(utils.get_config('concurrency', 'filler_keystone_client_concurrency')) pile = eventlet.GreenPile(concurrency) client = FakeKSClient() filler.create_swift_account(client, pile, 1, 1, self.ret_index) self.assertEqual(self.user_cnt, 1) self.assertEqual(len(self.ret_index.keys()), 1)
def test_04_sync_many_accounts_many_containers_and_obj_meta(self): """Many accounts with many containers and some object """ index = {} index_container = {} # Create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 1, 1, index) for account, account_id, username in \ self.extract_created_a_u_iter(self.created): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) acc = (account, account_id) filler.create_containers(tenant_cnx, acc, 1, index_container) filler.create_objects(tenant_cnx, acc, 1, 2048, index_container) # Start sync process self.swsync.process() # Now verify dest for account, account_id, username in \ self.extract_created_a_u_iter(self.created): # Verify container listing olo = self.list_objects_in_containers(account_id, self.o_admin_token, 'orig') old = self.list_objects_in_containers(account_id, self.d_admin_token, 'dest') # Verify we have the same amount of container self.assertListEqual(olo.keys(), old.keys()) # For each container for c, objs in olo.items(): for obj in objs: # Verify first object detail returned by container # server match = [od for od in old[c] if od['name'] == obj['name']] self.assertEqual(len(match), 1) obj_d = match[0] self.assertDictEqual(obj, obj_d) # Verify object details from object server obj_names = [d['name'] for d in olo[c]] for obj_name in obj_names: objd_o = self.get_object_detail(account_id, self.o_admin_token, 'orig', c, obj_name) objd_d = self.get_object_detail(account_id, self.d_admin_token, 'dest', c, obj_name) self.verify_aco_diff(objd_o, objd_d) # Verify content self.assertEqual(objd_o[1], objd_d[1])
def test_08_sync_containers_with_last_modified(self): """Containers with last-modified middleware.""" index = {} index_container = {} # Create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 1, 1, index) # Create container and store new account && container account_dest, container_dest = None, None for account, account_id, username in ( self.extract_created_a_u_iter(self.created)): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) acc = (account, account_id) filler.create_containers(tenant_cnx, acc, 1, index_container) filler.create_objects(tenant_cnx, acc, 1, 2048, index_container) cld = self.list_containers(account_id, self.d_admin_token, 'orig') account_dest = account_id container_dest = cld[0]['name'] break # Start sync process self.swsync.process() # Update dest self.put_object(account_dest, self.d_admin_token, 'dest', container_dest, 'lm-test', 'lm-data') # Get timestamp cdd = self.get_container_detail(account_dest, self.d_admin_token, 'dest', container_dest) try: dest_lm = cdd[0]['x-container-meta-last-modified'] except(KeyError): # Last-modified middleware is not present return # Restart sync process self.swsync.process() # Check if dest timestamp have not been updated cdd = self.get_container_detail(account_dest, self.d_admin_token, 'dest', container_dest) self.assertEqual(dest_lm, cdd[0]['x-container-meta-last-modified']) # Check if new object is still present in dest obj_detail = self.get_object_detail(account_dest, self.d_admin_token, 'dest', container_dest, 'lm-test') self.assertEqual('lm-data', obj_detail[1])
def test_05_account_two_passes(self): """Account modified two sync passes """ index = {} # create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 3, 1, index) for account, account_id, username in \ self.extract_created_a_u_iter(self.created): # post meta data on account tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) filler.create_account_meta(tenant_cnx) # start sync process self.swsync.process() # Add more meta to account for account, account_id, username in \ self.extract_created_a_u_iter(self.created): # Modify meta data on account tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) token = tenant_cnx.get_auth()[1] # Remove one, modify one, and add one meta a_meta = self.get_account_meta(account_id, token, 'orig') a_meta_k_names = [k.split('-')[-1] for k in a_meta] headers = {} headers['X-Account-Meta-a1'] = 'b1' headers["X-Remove-Account-Meta-%s" % a_meta_k_names[0]] = 'x' headers["X-Account-Meta-%s" % a_meta_k_names[1]] = 'b2' self.post_account(account_id, token, 'orig', headers=headers) # Re - start sync process self.swsync.process() # Now verify dest for account, account_id, username in \ self.extract_created_a_u_iter(self.created): alo = self.get_account_detail(account_id, self.o_admin_token, 'orig') ald = self.get_account_detail(account_id, self.d_admin_token, 'dest') self.verify_aco_diff(alo, ald)
def test_03_sync_many_accounts_with_many_containers_meta(self): """Many accounts with many containers and container meta data """ index = {} index_container = {} # Create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 3, 1, index) for account, account_id, username in \ self.extract_created_a_u_iter(self.created): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) acc = (account, account_id) filler.create_containers(tenant_cnx, acc, 3, index_container) # Start sync process self.swsync.process() # Now verify dest for account, account_id, username in \ self.extract_created_a_u_iter(self.created): # Verify container listing clo = self.list_containers(account_id, self.o_admin_token, 'orig') cld = self.list_containers(account_id, self.d_admin_token, 'dest') self.assertEqual(len(clo), len(cld)) for do in clo: match = [dd for dd in cld if dd['name'] == do['name']] self.assertEqual(len(match), 1) self.assertDictEqual(do, match[0]) # Verify container details clo_c_names = [d['name'] for d in clo] for c_name in clo_c_names: cdo = self.get_container_detail(account_id, self.o_admin_token, 'orig', c_name) cdd = self.get_container_detail(account_id, self.d_admin_token, 'dest', c_name) self.verify_aco_diff(cdo, cdd)
def test_01_sync_one_of_two_empty_accounts(self): """create two empty accounts, Sync only one """ index = {} test_account_name = "account_test" # create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 2, 1, index) for account, account_id, username in \ self.extract_created_a_u_iter(self.created): # post meta data on account tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) filler.create_account_meta(tenant_cnx) # select random account and write it in the filter file t_account, t_account_id, t_username = random.choice(list( self.extract_created_a_u_iter(self.created))) with open(self.filter_filename, "w") as filterlist: filterlist.write(t_account + "\n") # start sync process self.swsync.process() # Now verify dest for account, account_id, username in \ self.extract_created_a_u_iter(self.created): alo = self.get_account_detail(account_id, self.o_admin_token, 'orig') ald = self.get_account_detail(account_id, self.d_admin_token, 'dest') if account == t_account: self.verify_aco_diff(alo, ald)
def test_07_object_two_passes(self): """Objects modified two passes.""" index = {} index_container = {} # Create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 1, 1, index) for account, account_id, username in ( self.extract_created_a_u_iter(self.created)): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) acc = (account, account_id) filler.create_containers(tenant_cnx, acc, 1, index_container) filler.create_objects(tenant_cnx, acc, 3, 2048, index_container) # Start sync process self.swsync.process() # Modify objects in containers for account, account_id, username in ( self.extract_created_a_u_iter(self.created)): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) token = tenant_cnx.get_auth()[1] c_o = self.list_objects_in_containers(account_id, token, 'orig') for cont, objs in c_o.iteritems(): for obj in objs: # Modify object meta obj_d, data = self.get_object_detail(account_id, token, 'orig', cont, obj['name']) meta = {k: v for k, v in obj_d.iteritems() if k.startswith('x-object-meta')} meta_k_names = [k.split('-')[-1] for k in meta] headers = {} headers['X-Object-Meta-a1'] = 'b1' headers["X-Remove-Object-Meta-%s" % meta_k_names[0]] = 'x' headers["X-Object-Meta-%s" % meta_k_names[1]] = 'b2' self.post_object(account_id, token, 'orig', headers=headers, container=cont, name=obj['name']) # Create an object self.put_object(account_id, token, 'orig', cont, 'foofoo', 'barbarbar') self.put_object(account_id, token, 'orig', cont, 'foofoo1', 'barbarbar') self.put_object(account_id, token, 'orig', cont, 'foofoo2', 'barbarbar') o_names = [o['name'] for o in objs] # Delete an object name = o_names[0] self.delete_object(account_id, token, 'orig', cont, name) # Start sync process self.swsync.process() # Now verify dest for account, account_id, username in ( self.extract_created_a_u_iter(self.created)): # Verify container listing olo = self.list_objects_in_containers(account_id, self.o_admin_token, 'orig') old = self.list_objects_in_containers(account_id, self.d_admin_token, 'dest') # Verify we have the same amount of container self.assertListEqual(olo.keys(), old.keys()) # For each container for c, objs in olo.items(): for obj in objs: # Verify first object detail returned by container server match = [od for od in old[c] if od['name'] == obj['name']] self.assertEqual(len(match), 1) obj_d = match[0] a = obj.copy() b = obj_d.copy() del a['last_modified'] del b['last_modified'] self.assertDictEqual(a, b) # Verify object details from object server obj_names = [d['name'] for d in olo[c]] for obj_name in obj_names: objd_o = self.get_object_detail(account_id, self.o_admin_token, 'orig', c, obj_name) objd_d = self.get_object_detail(account_id, self.d_admin_token, 'dest', c, obj_name) self.verify_aco_diff(objd_o, objd_d) # Verify content self.assertEqual(objd_o[1], objd_d[1])
def test_06_container_two_passes(self): """Containers modified two sync passes.""" index = {} index_container = {} # Create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 3, 1, index) for account, account_id, username in ( self.extract_created_a_u_iter(self.created)): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) acc = (account, account_id) filler.create_containers(tenant_cnx, acc, 3, index_container) # Start sync process self.swsync.process() # Modify container in account for account, account_id, username in ( self.extract_created_a_u_iter(self.created)): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) token = tenant_cnx.get_auth()[1] # Modify an existing container meta clo = self.list_containers(account_id, token, 'orig') co_name = clo[0]['name'] c_meta = self.get_container_meta(account_id, token, 'orig', co_name) c_meta_k_names = [k.split('-')[-1] for k in c_meta] headers = {} headers['X-Container-Meta-a1'] = 'b1' headers["X-Remove-Container-Meta-%s" % c_meta_k_names[0]] = 'x' headers["X-Container-Meta-%s" % c_meta_k_names[1]] = 'b2' self.post_container(account_id, token, 'orig', headers=headers, container=co_name) # Add a some more container self.put_container(account_id, token, 'orig', 'foobar') self.put_container(account_id, token, 'orig', 'foobar1') self.put_container(account_id, token, 'orig', 'foobar2') # Delete one container co_name = clo[1]['name'] self.delete_container(account_id, token, 'orig', co_name) # Re - Start sync process self.swsync.process() # Now verify dest for account, account_id, username in ( self.extract_created_a_u_iter(self.created)): # Verify container listing clo = self.list_containers(account_id, self.o_admin_token, 'orig') cld = self.list_containers(account_id, self.d_admin_token, 'dest') self.assertEqual(len(clo), len(cld)) for do in clo: match = [dd for dd in cld if dd['name'] == do['name']] self.assertEqual(len(match), 1) self.assertDictEqual(do, match[0]) # Verify container details clo_c_names = [d['name'] for d in clo] for c_name in clo_c_names: cdo = self.get_container_detail(account_id, self.o_admin_token, 'orig', c_name) cdd = self.get_container_detail(account_id, self.d_admin_token, 'dest', c_name) self.verify_aco_diff(cdo, cdd)
def test_07_object_two_passes(self): """Objects modified two passes """ index = {} index_container = {} # Create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 1, 1, index) for account, account_id, username in \ self.extract_created_a_u_iter(self.created): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) acc = (account, account_id) filler.create_containers(tenant_cnx, acc, 1, index_container) filler.create_objects(tenant_cnx, acc, 3, 2048, index_container) # Start sync process self.swsync.process() # Modify objects in containers for account, account_id, username in \ self.extract_created_a_u_iter(self.created): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) token = tenant_cnx.get_auth()[1] c_o = self.list_objects_in_containers(account_id, token, 'orig') for cont, objs in c_o.iteritems(): for obj in objs: # Modify object meta obj_d, data = self.get_object_detail( account_id, token, 'orig', cont, obj['name']) meta = { k: v for k, v in obj_d.iteritems() if k.startswith('x-object-meta') } meta_k_names = [k.split('-')[-1] for k in meta] headers = {} headers['X-Object-Meta-a1'] = 'b1' headers["X-Remove-Object-Meta-%s" % meta_k_names[0]] = 'x' headers["X-Object-Meta-%s" % meta_k_names[1]] = 'b2' self.post_object(account_id, token, 'orig', headers=headers, container=cont, name=obj['name']) # Create an object self.put_object(account_id, token, 'orig', cont, 'foofoo', 'barbarbar') self.put_object(account_id, token, 'orig', cont, 'foofoo1', 'barbarbar') self.put_object(account_id, token, 'orig', cont, 'foofoo2', 'barbarbar') o_names = [o['name'] for o in objs] # Delete an object name = o_names[0] self.delete_object(account_id, token, 'orig', cont, name) # Start sync process self.swsync.process() # Now verify dest for account, account_id, username in \ self.extract_created_a_u_iter(self.created): # Verify container listing olo = self.list_objects_in_containers(account_id, self.o_admin_token, 'orig') old = self.list_objects_in_containers(account_id, self.d_admin_token, 'dest') # Verify we have the same amount of container self.assertListEqual(olo.keys(), old.keys()) # For each container for c, objs in olo.items(): for obj in objs: # Verify first object detail returned by container server match = [od for od in old[c] if od['name'] == obj['name']] self.assertEqual(len(match), 1) obj_d = match[0] a = obj.copy() b = obj_d.copy() del a['last_modified'] del b['last_modified'] self.assertDictEqual(a, b) # Verify object details from object server obj_names = [d['name'] for d in olo[c]] for obj_name in obj_names: objd_o = self.get_object_detail(account_id, self.o_admin_token, 'orig', c, obj_name) objd_d = self.get_object_detail(account_id, self.d_admin_token, 'dest', c, obj_name) self.verify_aco_diff(objd_o, objd_d) # Verify content self.assertEqual(objd_o[1], objd_d[1])
def test_06_container_two_passes(self): """Containers modified two sync passes """ index = {} index_container = {} # Create account self.created = filler.create_swift_account(self.o_ks_client, self.pile, 3, 1, index) for account, account_id, username in \ self.extract_created_a_u_iter(self.created): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) acc = (account, account_id) filler.create_containers(tenant_cnx, acc, 3, index_container) # Start sync process self.swsync.process() # Modify container in account for account, account_id, username in \ self.extract_created_a_u_iter(self.created): tenant_cnx = sclient.Connection(self.o_st, "%s:%s" % (account, username), self.default_user_password, auth_version=2) token = tenant_cnx.get_auth()[1] # Modify an existing container meta clo = self.list_containers(account_id, token, 'orig') co_name = clo[0]['name'] c_meta = self.get_container_meta(account_id, token, 'orig', co_name) c_meta_k_names = [k.split('-')[-1] for k in c_meta] headers = {} headers['X-Container-Meta-a1'] = 'b1' headers["X-Remove-Container-Meta-%s" % c_meta_k_names[0]] = 'x' headers["X-Container-Meta-%s" % c_meta_k_names[1]] = 'b2' self.post_container(account_id, token, 'orig', headers=headers, container=co_name) # Add a some more container self.put_container(account_id, token, 'orig', 'foobar') self.put_container(account_id, token, 'orig', 'foobar1') self.put_container(account_id, token, 'orig', 'foobar2') # Delete one container co_name = clo[1]['name'] self.delete_container(account_id, token, 'orig', co_name) # Re - Start sync process self.swsync.process() # Now verify dest for account, account_id, username in \ self.extract_created_a_u_iter(self.created): # Verify container listing clo = self.list_containers(account_id, self.o_admin_token, 'orig') cld = self.list_containers(account_id, self.d_admin_token, 'dest') self.assertEqual(len(clo), len(cld)) for do in clo: match = [dd for dd in cld if dd['name'] == do['name']] self.assertEqual(len(match), 1) self.assertDictEqual(do, match[0]) # Verify container details clo_c_names = [d['name'] for d in clo] for c_name in clo_c_names: cdo = self.get_container_detail(account_id, self.o_admin_token, 'orig', c_name) cdd = self.get_container_detail(account_id, self.d_admin_token, 'dest', c_name) self.verify_aco_diff(cdo, cdd)