def test_create_swift_account_fail(self):
self.ret_index = {}
self.pa = 0
def create_tenant(*args):
if self.pa == 0:
self.pa += 1
raise KSClientException('Fake msg')
else:
self.pa += 1
return FakeKSTenant('foo1')
def create_swift_user(*args):
pass
client = FakeKSClient()
self.stubs.Set(client.tenants, 'create', create_tenant)
self.stubs.Set(filler, 'create_swift_user', create_swift_user)
concurrency = int(utils.get_config('concurrency',
'filler_keystone_client_concurrency'))
pile = eventlet.GreenPile(concurrency)
filler.create_swift_account(client, pile, 3, 1, self.ret_index)
self.assertEqual(len(self.ret_index.keys()), 2)
def test_02_sync_many_empty_account(self):
"""Many empty account with meta data."""
index = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
3, 1, index)
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# Post meta data on account
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
filler.create_account_meta(tenant_cnx)
# Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
alo = self.get_account_detail(account_id,
self.o_admin_token, 'orig')
ald = self.get_account_detail(account_id,
self.d_admin_token, 'dest')
self.verify_aco_diff(alo, ald)
def test_create_swift_account(self):
self.ret_index = {}
self.user_cnt = 0
def create_swift_user(*args):
self.user_cnt += 1
self.stubs.Set(filler, 'create_swift_user', create_swift_user)
concurrency = int(utils.get_config('concurrency',
'filler_keystone_client_concurrency'))
pile = eventlet.GreenPile(concurrency)
client = FakeKSClient()
filler.create_swift_account(client, pile, 1, 1, self.ret_index)
self.assertEqual(self.user_cnt, 1)
self.assertEqual(len(self.ret_index.keys()), 1)
def test_04_sync_many_accounts_many_containers_and_obj_meta(self):
"""Many accounts with many containers and some object
"""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
1, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 1, index_container)
filler.create_objects(tenant_cnx, acc, 1, 2048, index_container)
# Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
# Verify container listing
olo = self.list_objects_in_containers(account_id,
self.o_admin_token, 'orig')
old = self.list_objects_in_containers(account_id,
self.d_admin_token, 'dest')
# Verify we have the same amount of container
self.assertListEqual(olo.keys(), old.keys())
# For each container
for c, objs in olo.items():
for obj in objs:
# Verify first object detail returned by container
# server
match = [od for od in old[c] if od['name'] == obj['name']]
self.assertEqual(len(match), 1)
obj_d = match[0]
self.assertDictEqual(obj, obj_d)
# Verify object details from object server
obj_names = [d['name'] for d in olo[c]]
for obj_name in obj_names:
objd_o = self.get_object_detail(account_id,
self.o_admin_token, 'orig',
c, obj_name)
objd_d = self.get_object_detail(account_id,
self.d_admin_token, 'dest',
c, obj_name)
self.verify_aco_diff(objd_o, objd_d)
# Verify content
self.assertEqual(objd_o[1], objd_d[1])
def test_08_sync_containers_with_last_modified(self):
"""Containers with last-modified middleware."""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
1, 1, index)
# Create container and store new account && container
account_dest, container_dest = None, None
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 1, index_container)
filler.create_objects(tenant_cnx, acc, 1, 2048, index_container)
cld = self.list_containers(account_id,
self.d_admin_token, 'orig')
account_dest = account_id
container_dest = cld[0]['name']
break
# Start sync process
self.swsync.process()
# Update dest
self.put_object(account_dest, self.d_admin_token, 'dest',
container_dest, 'lm-test', 'lm-data')
# Get timestamp
cdd = self.get_container_detail(account_dest, self.d_admin_token,
'dest', container_dest)
try:
dest_lm = cdd[0]['x-container-meta-last-modified']
except(KeyError):
# Last-modified middleware is not present
return
# Restart sync process
self.swsync.process()
# Check if dest timestamp have not been updated
cdd = self.get_container_detail(account_dest, self.d_admin_token,
'dest', container_dest)
self.assertEqual(dest_lm, cdd[0]['x-container-meta-last-modified'])
# Check if new object is still present in dest
obj_detail = self.get_object_detail(account_dest, self.d_admin_token,
'dest', container_dest, 'lm-test')
self.assertEqual('lm-data', obj_detail[1])
def test_05_account_two_passes(self):
"""Account modified two sync passes
"""
index = {}
# create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
3, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
# post meta data on account
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
filler.create_account_meta(tenant_cnx)
# start sync process
self.swsync.process()
# Add more meta to account
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
# Modify meta data on account
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
token = tenant_cnx.get_auth()[1]
# Remove one, modify one, and add one meta
a_meta = self.get_account_meta(account_id,
token,
'orig')
a_meta_k_names = [k.split('-')[-1] for k in a_meta]
headers = {}
headers['X-Account-Meta-a1'] = 'b1'
headers["X-Remove-Account-Meta-%s" % a_meta_k_names[0]] = 'x'
headers["X-Account-Meta-%s" % a_meta_k_names[1]] = 'b2'
self.post_account(account_id, token,
'orig', headers=headers)
# Re - start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
alo = self.get_account_detail(account_id,
self.o_admin_token, 'orig')
ald = self.get_account_detail(account_id,
self.d_admin_token, 'dest')
self.verify_aco_diff(alo, ald)
def test_03_sync_many_accounts_with_many_containers_meta(self):
"""Many accounts with many containers and container meta data
"""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
3, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 3, index_container)
# Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
# Verify container listing
clo = self.list_containers(account_id,
self.o_admin_token, 'orig')
cld = self.list_containers(account_id,
self.d_admin_token, 'dest')
self.assertEqual(len(clo), len(cld))
for do in clo:
match = [dd for dd in cld if dd['name'] == do['name']]
self.assertEqual(len(match), 1)
self.assertDictEqual(do, match[0])
# Verify container details
clo_c_names = [d['name'] for d in clo]
for c_name in clo_c_names:
cdo = self.get_container_detail(account_id, self.o_admin_token,
'orig', c_name)
cdd = self.get_container_detail(account_id, self.d_admin_token,
'dest', c_name)
self.verify_aco_diff(cdo, cdd)
def test_01_sync_one_of_two_empty_accounts(self):
"""create two empty accounts, Sync only one
"""
index = {}
test_account_name = "account_test"
# create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
2, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
# post meta data on account
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
filler.create_account_meta(tenant_cnx)
# select random account and write it in the filter file
t_account, t_account_id, t_username = random.choice(list(
self.extract_created_a_u_iter(self.created)))
with open(self.filter_filename, "w") as filterlist:
filterlist.write(t_account + "\n")
# start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
alo = self.get_account_detail(account_id,
self.o_admin_token, 'orig')
ald = self.get_account_detail(account_id,
self.d_admin_token, 'dest')
if account == t_account:
self.verify_aco_diff(alo, ald)
def test_07_object_two_passes(self):
"""Objects modified two passes."""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
1, 1, index)
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 1, index_container)
filler.create_objects(tenant_cnx, acc, 3, 2048, index_container)
# Start sync process
self.swsync.process()
# Modify objects in containers
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
token = tenant_cnx.get_auth()[1]
c_o = self.list_objects_in_containers(account_id, token, 'orig')
for cont, objs in c_o.iteritems():
for obj in objs:
# Modify object meta
obj_d, data = self.get_object_detail(account_id,
token, 'orig',
cont, obj['name'])
meta = {k: v for k, v in obj_d.iteritems()
if k.startswith('x-object-meta')}
meta_k_names = [k.split('-')[-1] for k in meta]
headers = {}
headers['X-Object-Meta-a1'] = 'b1'
headers["X-Remove-Object-Meta-%s" % meta_k_names[0]] = 'x'
headers["X-Object-Meta-%s" % meta_k_names[1]] = 'b2'
self.post_object(account_id, token,
'orig',
headers=headers,
container=cont,
name=obj['name'])
# Create an object
self.put_object(account_id, token, 'orig',
cont, 'foofoo', 'barbarbar')
self.put_object(account_id, token, 'orig',
cont, 'foofoo1', 'barbarbar')
self.put_object(account_id, token, 'orig',
cont, 'foofoo2', 'barbarbar')
o_names = [o['name'] for o in objs]
# Delete an object
name = o_names[0]
self.delete_object(account_id, token, 'orig',
cont, name)
# Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# Verify container listing
olo = self.list_objects_in_containers(account_id,
self.o_admin_token, 'orig')
old = self.list_objects_in_containers(account_id,
self.d_admin_token, 'dest')
# Verify we have the same amount of container
self.assertListEqual(olo.keys(), old.keys())
# For each container
for c, objs in olo.items():
for obj in objs:
# Verify first object detail returned by container server
match = [od for od in old[c] if od['name'] == obj['name']]
self.assertEqual(len(match), 1)
obj_d = match[0]
a = obj.copy()
b = obj_d.copy()
del a['last_modified']
del b['last_modified']
self.assertDictEqual(a, b)
# Verify object details from object server
obj_names = [d['name'] for d in olo[c]]
for obj_name in obj_names:
objd_o = self.get_object_detail(account_id,
self.o_admin_token, 'orig',
c, obj_name)
objd_d = self.get_object_detail(account_id,
self.d_admin_token, 'dest',
c, obj_name)
self.verify_aco_diff(objd_o, objd_d)
# Verify content
self.assertEqual(objd_o[1], objd_d[1])
def test_06_container_two_passes(self):
"""Containers modified two sync passes."""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
3, 1, index)
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 3, index_container)
# Start sync process
self.swsync.process()
# Modify container in account
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
token = tenant_cnx.get_auth()[1]
# Modify an existing container meta
clo = self.list_containers(account_id,
token, 'orig')
co_name = clo[0]['name']
c_meta = self.get_container_meta(account_id,
token, 'orig',
co_name)
c_meta_k_names = [k.split('-')[-1] for k in c_meta]
headers = {}
headers['X-Container-Meta-a1'] = 'b1'
headers["X-Remove-Container-Meta-%s" % c_meta_k_names[0]] = 'x'
headers["X-Container-Meta-%s" % c_meta_k_names[1]] = 'b2'
self.post_container(account_id, token,
'orig',
headers=headers,
container=co_name)
# Add a some more container
self.put_container(account_id, token, 'orig', 'foobar')
self.put_container(account_id, token, 'orig', 'foobar1')
self.put_container(account_id, token, 'orig', 'foobar2')
# Delete one container
co_name = clo[1]['name']
self.delete_container(account_id, token, 'orig', co_name)
# Re - Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# Verify container listing
clo = self.list_containers(account_id,
self.o_admin_token, 'orig')
cld = self.list_containers(account_id,
self.d_admin_token, 'dest')
self.assertEqual(len(clo), len(cld))
for do in clo:
match = [dd for dd in cld if dd['name'] == do['name']]
self.assertEqual(len(match), 1)
self.assertDictEqual(do, match[0])
# Verify container details
clo_c_names = [d['name'] for d in clo]
for c_name in clo_c_names:
cdo = self.get_container_detail(account_id, self.o_admin_token,
'orig', c_name)
cdd = self.get_container_detail(account_id, self.d_admin_token,
'dest', c_name)
self.verify_aco_diff(cdo, cdd)
def test_07_object_two_passes(self):
"""Objects modified two passes
"""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client, self.pile,
1, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 1, index_container)
filler.create_objects(tenant_cnx, acc, 3, 2048, index_container)
# Start sync process
self.swsync.process()
# Modify objects in containers
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
token = tenant_cnx.get_auth()[1]
c_o = self.list_objects_in_containers(account_id, token, 'orig')
for cont, objs in c_o.iteritems():
for obj in objs:
# Modify object meta
obj_d, data = self.get_object_detail(
account_id, token, 'orig', cont, obj['name'])
meta = {
k: v
for k, v in obj_d.iteritems()
if k.startswith('x-object-meta')
}
meta_k_names = [k.split('-')[-1] for k in meta]
headers = {}
headers['X-Object-Meta-a1'] = 'b1'
headers["X-Remove-Object-Meta-%s" % meta_k_names[0]] = 'x'
headers["X-Object-Meta-%s" % meta_k_names[1]] = 'b2'
self.post_object(account_id,
token,
'orig',
headers=headers,
container=cont,
name=obj['name'])
# Create an object
self.put_object(account_id, token, 'orig', cont, 'foofoo',
'barbarbar')
self.put_object(account_id, token, 'orig', cont, 'foofoo1',
'barbarbar')
self.put_object(account_id, token, 'orig', cont, 'foofoo2',
'barbarbar')
o_names = [o['name'] for o in objs]
# Delete an object
name = o_names[0]
self.delete_object(account_id, token, 'orig', cont, name)
# Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
# Verify container listing
olo = self.list_objects_in_containers(account_id,
self.o_admin_token, 'orig')
old = self.list_objects_in_containers(account_id,
self.d_admin_token, 'dest')
# Verify we have the same amount of container
self.assertListEqual(olo.keys(), old.keys())
# For each container
for c, objs in olo.items():
for obj in objs:
# Verify first object detail returned by container server
match = [od for od in old[c] if od['name'] == obj['name']]
self.assertEqual(len(match), 1)
obj_d = match[0]
a = obj.copy()
b = obj_d.copy()
del a['last_modified']
del b['last_modified']
self.assertDictEqual(a, b)
# Verify object details from object server
obj_names = [d['name'] for d in olo[c]]
for obj_name in obj_names:
objd_o = self.get_object_detail(account_id,
self.o_admin_token, 'orig',
c, obj_name)
objd_d = self.get_object_detail(account_id,
self.d_admin_token, 'dest',
c, obj_name)
self.verify_aco_diff(objd_o, objd_d)
# Verify content
self.assertEqual(objd_o[1], objd_d[1])
def test_06_container_two_passes(self):
"""Containers modified two sync passes
"""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client, self.pile,
3, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 3, index_container)
# Start sync process
self.swsync.process()
# Modify container in account
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
token = tenant_cnx.get_auth()[1]
# Modify an existing container meta
clo = self.list_containers(account_id, token, 'orig')
co_name = clo[0]['name']
c_meta = self.get_container_meta(account_id, token, 'orig',
co_name)
c_meta_k_names = [k.split('-')[-1] for k in c_meta]
headers = {}
headers['X-Container-Meta-a1'] = 'b1'
headers["X-Remove-Container-Meta-%s" % c_meta_k_names[0]] = 'x'
headers["X-Container-Meta-%s" % c_meta_k_names[1]] = 'b2'
self.post_container(account_id,
token,
'orig',
headers=headers,
container=co_name)
# Add a some more container
self.put_container(account_id, token, 'orig', 'foobar')
self.put_container(account_id, token, 'orig', 'foobar1')
self.put_container(account_id, token, 'orig', 'foobar2')
# Delete one container
co_name = clo[1]['name']
self.delete_container(account_id, token, 'orig', co_name)
# Re - Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
# Verify container listing
clo = self.list_containers(account_id, self.o_admin_token, 'orig')
cld = self.list_containers(account_id, self.d_admin_token, 'dest')
self.assertEqual(len(clo), len(cld))
for do in clo:
match = [dd for dd in cld if dd['name'] == do['name']]
self.assertEqual(len(match), 1)
self.assertDictEqual(do, match[0])
# Verify container details
clo_c_names = [d['name'] for d in clo]
for c_name in clo_c_names:
cdo = self.get_container_detail(account_id, self.o_admin_token,
'orig', c_name)
cdd = self.get_container_detail(account_id, self.d_admin_token,
'dest', c_name)
self.verify_aco_diff(cdo, cdd)