def test_04_sync_many_accounts_many_containers_and_obj_meta(self):
"""Many accounts with many containers and some object
"""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
1, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 1, index_container)
filler.create_objects(tenant_cnx, acc, 1, 2048, index_container)
# Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
# Verify container listing
olo = self.list_objects_in_containers(account_id,
self.o_admin_token, 'orig')
old = self.list_objects_in_containers(account_id,
self.d_admin_token, 'dest')
# Verify we have the same amount of container
self.assertListEqual(olo.keys(), old.keys())
# For each container
for c, objs in olo.items():
for obj in objs:
# Verify first object detail returned by container
# server
match = [od for od in old[c] if od['name'] == obj['name']]
self.assertEqual(len(match), 1)
obj_d = match[0]
self.assertDictEqual(obj, obj_d)
# Verify object details from object server
obj_names = [d['name'] for d in olo[c]]
for obj_name in obj_names:
objd_o = self.get_object_detail(account_id,
self.o_admin_token, 'orig',
c, obj_name)
objd_d = self.get_object_detail(account_id,
self.d_admin_token, 'dest',
c, obj_name)
self.verify_aco_diff(objd_o, objd_d)
# Verify content
self.assertEqual(objd_o[1], objd_d[1])
def test_08_sync_containers_with_last_modified(self):
"""Containers with last-modified middleware."""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
1, 1, index)
# Create container and store new account && container
account_dest, container_dest = None, None
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 1, index_container)
filler.create_objects(tenant_cnx, acc, 1, 2048, index_container)
cld = self.list_containers(account_id,
self.d_admin_token, 'orig')
account_dest = account_id
container_dest = cld[0]['name']
break
# Start sync process
self.swsync.process()
# Update dest
self.put_object(account_dest, self.d_admin_token, 'dest',
container_dest, 'lm-test', 'lm-data')
# Get timestamp
cdd = self.get_container_detail(account_dest, self.d_admin_token,
'dest', container_dest)
try:
dest_lm = cdd[0]['x-container-meta-last-modified']
except(KeyError):
# Last-modified middleware is not present
return
# Restart sync process
self.swsync.process()
# Check if dest timestamp have not been updated
cdd = self.get_container_detail(account_dest, self.d_admin_token,
'dest', container_dest)
self.assertEqual(dest_lm, cdd[0]['x-container-meta-last-modified'])
# Check if new object is still present in dest
obj_detail = self.get_object_detail(account_dest, self.d_admin_token,
'dest', container_dest, 'lm-test')
self.assertEqual('lm-data', obj_detail[1])
def test_08_sync_containers_with_last_modified(self):
"""Containers with last-modified middleware
"""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client, self.pile,
1, 1, index)
# Create container and store new account && container
account_dest, container_dest = None, None
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 1, index_container)
filler.create_objects(tenant_cnx, acc, 1, 2048, index_container)
cld = self.list_containers(account_id, self.d_admin_token, 'orig')
account_dest = account_id
container_dest = cld[0]['name']
break
# Start sync process
self.swsync.process()
# Update dest
self.put_object(account_dest, self.d_admin_token, 'dest',
container_dest, 'lm-test', 'lm-data')
# Get timestamp
cdd = self.get_container_detail(account_dest, self.d_admin_token,
'dest', container_dest)
try:
dest_lm = cdd[0]['x-container-meta-last-modified']
except (KeyError):
# Last-modified middleware is not present
return
# Restart sync process
self.swsync.process()
# Check if dest timestamp have not been updated
cdd = self.get_container_detail(account_dest, self.d_admin_token,
'dest', container_dest)
self.assertEqual(dest_lm, cdd[0]['x-container-meta-last-modified'])
# Check if new object is still present in dest
obj_detail = self.get_object_detail(account_dest, self.d_admin_token,
'dest', container_dest, 'lm-test')
self.assertEqual('lm-data', obj_detail[1])
def test_create_containers(self):
get_containers_created = []
return_dict_ref = {}
def put_container(*args, **kwargs):
get_containers_created.append(args[1])
self.stubs.Set(FakeSWConnection, 'put_container', put_container)
cnx = self.get_connection()
filler.create_containers(cnx, 'test', 3, return_dict_ref)
self.assertEqual(len(get_containers_created), 3)
self.assertEqual(get_containers_created[0].split('_')[0], 'container')
meta_amount = len(return_dict_ref['test'].values())
self.assertEqual(meta_amount, 3)
def test_create_containers(self):
get_containers_created = []
return_dict_ref = {}
def put_container(*args, **kwargs):
get_containers_created.append(args[1])
self.stubs.Set(FakeSWConnection, 'put_container', put_container)
cnx = self.get_connection()
filler.create_containers(cnx, 'test', 3, return_dict_ref)
self.assertEqual(len(get_containers_created), 3)
self.assertEqual(get_containers_created[0].split('_')[0],
'container')
meta_amount = len(return_dict_ref['test'].values())
self.assertEqual(meta_amount, 3)
def test_create_containers_fail(self):
get_containers_created = []
return_dict_ref = {}
self.attempts = 0
def put_container(*args, **kwargs):
if self.attempts == 0:
self.attempts += 1
raise swiftclient.client.ClientException('Fake err msg')
else:
self.attempts += 1
get_containers_created.append(args[1])
self.stubs.Set(FakeSWConnection, 'put_container', put_container)
cnx = self.get_connection()
filler.create_containers(cnx, 'test', 3, return_dict_ref)
self.assertEqual(len(get_containers_created), 2)
def test_create_containers_fail(self):
get_containers_created = []
return_dict_ref = {}
self.attempts = 0
def put_container(*args, **kwargs):
if self.attempts == 0:
self.attempts += 1
raise swiftclient.client.ClientException('Fake err msg')
else:
self.attempts += 1
get_containers_created.append(args[1])
self.stubs.Set(FakeSWConnection, 'put_container', put_container)
cnx = self.get_connection()
filler.create_containers(cnx, 'test', 3, return_dict_ref)
self.assertEqual(len(get_containers_created), 2)
def test_03_sync_many_accounts_with_many_containers_meta(self):
"""Many accounts with many containers and container meta data
"""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
3, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 3, index_container)
# Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
# Verify container listing
clo = self.list_containers(account_id,
self.o_admin_token, 'orig')
cld = self.list_containers(account_id,
self.d_admin_token, 'dest')
self.assertEqual(len(clo), len(cld))
for do in clo:
match = [dd for dd in cld if dd['name'] == do['name']]
self.assertEqual(len(match), 1)
self.assertDictEqual(do, match[0])
# Verify container details
clo_c_names = [d['name'] for d in clo]
for c_name in clo_c_names:
cdo = self.get_container_detail(account_id, self.o_admin_token,
'orig', c_name)
cdd = self.get_container_detail(account_id, self.d_admin_token,
'dest', c_name)
self.verify_aco_diff(cdo, cdd)
def test_07_object_two_passes(self):
"""Objects modified two passes."""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
1, 1, index)
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 1, index_container)
filler.create_objects(tenant_cnx, acc, 3, 2048, index_container)
# Start sync process
self.swsync.process()
# Modify objects in containers
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
token = tenant_cnx.get_auth()[1]
c_o = self.list_objects_in_containers(account_id, token, 'orig')
for cont, objs in c_o.iteritems():
for obj in objs:
# Modify object meta
obj_d, data = self.get_object_detail(account_id,
token, 'orig',
cont, obj['name'])
meta = {k: v for k, v in obj_d.iteritems()
if k.startswith('x-object-meta')}
meta_k_names = [k.split('-')[-1] for k in meta]
headers = {}
headers['X-Object-Meta-a1'] = 'b1'
headers["X-Remove-Object-Meta-%s" % meta_k_names[0]] = 'x'
headers["X-Object-Meta-%s" % meta_k_names[1]] = 'b2'
self.post_object(account_id, token,
'orig',
headers=headers,
container=cont,
name=obj['name'])
# Create an object
self.put_object(account_id, token, 'orig',
cont, 'foofoo', 'barbarbar')
self.put_object(account_id, token, 'orig',
cont, 'foofoo1', 'barbarbar')
self.put_object(account_id, token, 'orig',
cont, 'foofoo2', 'barbarbar')
o_names = [o['name'] for o in objs]
# Delete an object
name = o_names[0]
self.delete_object(account_id, token, 'orig',
cont, name)
# Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# Verify container listing
olo = self.list_objects_in_containers(account_id,
self.o_admin_token, 'orig')
old = self.list_objects_in_containers(account_id,
self.d_admin_token, 'dest')
# Verify we have the same amount of container
self.assertListEqual(olo.keys(), old.keys())
# For each container
for c, objs in olo.items():
for obj in objs:
# Verify first object detail returned by container server
match = [od for od in old[c] if od['name'] == obj['name']]
self.assertEqual(len(match), 1)
obj_d = match[0]
a = obj.copy()
b = obj_d.copy()
del a['last_modified']
del b['last_modified']
self.assertDictEqual(a, b)
# Verify object details from object server
obj_names = [d['name'] for d in olo[c]]
for obj_name in obj_names:
objd_o = self.get_object_detail(account_id,
self.o_admin_token, 'orig',
c, obj_name)
objd_d = self.get_object_detail(account_id,
self.d_admin_token, 'dest',
c, obj_name)
self.verify_aco_diff(objd_o, objd_d)
# Verify content
self.assertEqual(objd_o[1], objd_d[1])
def test_06_container_two_passes(self):
"""Containers modified two sync passes."""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
3, 1, index)
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 3, index_container)
# Start sync process
self.swsync.process()
# Modify container in account
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
token = tenant_cnx.get_auth()[1]
# Modify an existing container meta
clo = self.list_containers(account_id,
token, 'orig')
co_name = clo[0]['name']
c_meta = self.get_container_meta(account_id,
token, 'orig',
co_name)
c_meta_k_names = [k.split('-')[-1] for k in c_meta]
headers = {}
headers['X-Container-Meta-a1'] = 'b1'
headers["X-Remove-Container-Meta-%s" % c_meta_k_names[0]] = 'x'
headers["X-Container-Meta-%s" % c_meta_k_names[1]] = 'b2'
self.post_container(account_id, token,
'orig',
headers=headers,
container=co_name)
# Add a some more container
self.put_container(account_id, token, 'orig', 'foobar')
self.put_container(account_id, token, 'orig', 'foobar1')
self.put_container(account_id, token, 'orig', 'foobar2')
# Delete one container
co_name = clo[1]['name']
self.delete_container(account_id, token, 'orig', co_name)
# Re - Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# Verify container listing
clo = self.list_containers(account_id,
self.o_admin_token, 'orig')
cld = self.list_containers(account_id,
self.d_admin_token, 'dest')
self.assertEqual(len(clo), len(cld))
for do in clo:
match = [dd for dd in cld if dd['name'] == do['name']]
self.assertEqual(len(match), 1)
self.assertDictEqual(do, match[0])
# Verify container details
clo_c_names = [d['name'] for d in clo]
for c_name in clo_c_names:
cdo = self.get_container_detail(account_id, self.o_admin_token,
'orig', c_name)
cdd = self.get_container_detail(account_id, self.d_admin_token,
'dest', c_name)
self.verify_aco_diff(cdo, cdd)
def test_07_object_two_passes(self):
"""Objects modified two passes
"""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
1, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 1, index_container)
filler.create_objects(tenant_cnx, acc, 3, 2048, index_container)
# Start sync process
self.swsync.process()
# Modify objects in containers
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
token = tenant_cnx.get_auth()[1]
c_o = self.list_objects_in_containers(account_id, token, 'orig')
for cont, objs in c_o.iteritems():
for obj in objs:
# Modify object meta
obj_d, data = self.get_object_detail(account_id,
token, 'orig',
cont, obj['name'])
meta = {k: v for k, v in obj_d.iteritems()
if k.startswith('x-object-meta')}
meta_k_names = [k.split('-')[-1] for k in meta]
headers = {}
headers['X-Object-Meta-a1'] = 'b1'
headers["X-Remove-Object-Meta-%s" % meta_k_names[0]] = 'x'
headers["X-Object-Meta-%s" % meta_k_names[1]] = 'b2'
self.post_object(account_id, token,
'orig',
headers=headers,
container=cont,
name=obj['name'])
# Create an object
self.put_object(account_id, token, 'orig',
cont, 'foofoo', 'barbarbar')
self.put_object(account_id, token, 'orig',
cont, 'foofoo1', 'barbarbar')
self.put_object(account_id, token, 'orig',
cont, 'foofoo2', 'barbarbar')
o_names = [o['name'] for o in objs]
# Delete an object
name = o_names[0]
self.delete_object(account_id, token, 'orig',
cont, name)
# Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
# Verify container listing
olo = self.list_objects_in_containers(account_id,
self.o_admin_token, 'orig')
old = self.list_objects_in_containers(account_id,
self.d_admin_token, 'dest')
# Verify we have the same amount of container
self.assertListEqual(olo.keys(), old.keys())
# For each container
for c, objs in olo.items():
for obj in objs:
# Verify first object detail returned by container server
match = [od for od in old[c] if od['name'] == obj['name']]
self.assertEqual(len(match), 1)
obj_d = match[0]
a = obj.copy()
b = obj_d.copy()
del a['last_modified']
del b['last_modified']
self.assertDictEqual(a, b)
# Verify object details from object server
obj_names = [d['name'] for d in olo[c]]
for obj_name in obj_names:
objd_o = self.get_object_detail(account_id,
self.o_admin_token, 'orig',
c, obj_name)
objd_d = self.get_object_detail(account_id,
self.d_admin_token, 'dest',
c, obj_name)
self.verify_aco_diff(objd_o, objd_d)
# Verify content
self.assertEqual(objd_o[1], objd_d[1])
def test_06_container_two_passes(self):
"""Containers modified two sync passes
"""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
3, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 3, index_container)
# Start sync process
self.swsync.process()
# Modify container in account
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
token = tenant_cnx.get_auth()[1]
# Modify an existing container meta
clo = self.list_containers(account_id,
token, 'orig')
co_name = clo[0]['name']
c_meta = self.get_container_meta(account_id,
token, 'orig',
co_name)
c_meta_k_names = [k.split('-')[-1] for k in c_meta]
headers = {}
headers['X-Container-Meta-a1'] = 'b1'
headers["X-Remove-Container-Meta-%s" % c_meta_k_names[0]] = 'x'
headers["X-Container-Meta-%s" % c_meta_k_names[1]] = 'b2'
self.post_container(account_id, token,
'orig',
headers=headers,
container=co_name)
# Add a some more container
self.put_container(account_id, token, 'orig', 'foobar')
self.put_container(account_id, token, 'orig', 'foobar1')
self.put_container(account_id, token, 'orig', 'foobar2')
# Delete one container
co_name = clo[1]['name']
self.delete_container(account_id, token, 'orig', co_name)
# Re - Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
# Verify container listing
clo = self.list_containers(account_id,
self.o_admin_token, 'orig')
cld = self.list_containers(account_id,
self.d_admin_token, 'dest')
self.assertEqual(len(clo), len(cld))
for do in clo:
match = [dd for dd in cld if dd['name'] == do['name']]
self.assertEqual(len(match), 1)
self.assertDictEqual(do, match[0])
# Verify container details
clo_c_names = [d['name'] for d in clo]
for c_name in clo_c_names:
cdo = self.get_container_detail(account_id, self.o_admin_token,
'orig', c_name)
cdd = self.get_container_detail(account_id, self.d_admin_token,
'dest', c_name)
self.verify_aco_diff(cdo, cdd)