def test_04_sync_many_accounts_many_containers_and_obj_meta(self):
"""Many accounts with many containers and some object
"""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
1, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 1, index_container)
filler.create_objects(tenant_cnx, acc, 1, 2048, index_container)
# Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
# Verify container listing
olo = self.list_objects_in_containers(account_id,
self.o_admin_token, 'orig')
old = self.list_objects_in_containers(account_id,
self.d_admin_token, 'dest')
# Verify we have the same amount of container
self.assertListEqual(olo.keys(), old.keys())
# For each container
for c, objs in olo.items():
for obj in objs:
# Verify first object detail returned by container
# server
match = [od for od in old[c] if od['name'] == obj['name']]
self.assertEqual(len(match), 1)
obj_d = match[0]
self.assertDictEqual(obj, obj_d)
# Verify object details from object server
obj_names = [d['name'] for d in olo[c]]
for obj_name in obj_names:
objd_o = self.get_object_detail(account_id,
self.o_admin_token, 'orig',
c, obj_name)
objd_d = self.get_object_detail(account_id,
self.d_admin_token, 'dest',
c, obj_name)
self.verify_aco_diff(objd_o, objd_d)
# Verify content
self.assertEqual(objd_o[1], objd_d[1])
def test_08_sync_containers_with_last_modified(self):
"""Containers with last-modified middleware."""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
1, 1, index)
# Create container and store new account && container
account_dest, container_dest = None, None
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 1, index_container)
filler.create_objects(tenant_cnx, acc, 1, 2048, index_container)
cld = self.list_containers(account_id,
self.d_admin_token, 'orig')
account_dest = account_id
container_dest = cld[0]['name']
break
# Start sync process
self.swsync.process()
# Update dest
self.put_object(account_dest, self.d_admin_token, 'dest',
container_dest, 'lm-test', 'lm-data')
# Get timestamp
cdd = self.get_container_detail(account_dest, self.d_admin_token,
'dest', container_dest)
try:
dest_lm = cdd[0]['x-container-meta-last-modified']
except(KeyError):
# Last-modified middleware is not present
return
# Restart sync process
self.swsync.process()
# Check if dest timestamp have not been updated
cdd = self.get_container_detail(account_dest, self.d_admin_token,
'dest', container_dest)
self.assertEqual(dest_lm, cdd[0]['x-container-meta-last-modified'])
# Check if new object is still present in dest
obj_detail = self.get_object_detail(account_dest, self.d_admin_token,
'dest', container_dest, 'lm-test')
self.assertEqual('lm-data', obj_detail[1])
def test_08_sync_containers_with_last_modified(self):
"""Containers with last-modified middleware
"""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client, self.pile,
1, 1, index)
# Create container and store new account && container
account_dest, container_dest = None, None
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 1, index_container)
filler.create_objects(tenant_cnx, acc, 1, 2048, index_container)
cld = self.list_containers(account_id, self.d_admin_token, 'orig')
account_dest = account_id
container_dest = cld[0]['name']
break
# Start sync process
self.swsync.process()
# Update dest
self.put_object(account_dest, self.d_admin_token, 'dest',
container_dest, 'lm-test', 'lm-data')
# Get timestamp
cdd = self.get_container_detail(account_dest, self.d_admin_token,
'dest', container_dest)
try:
dest_lm = cdd[0]['x-container-meta-last-modified']
except (KeyError):
# Last-modified middleware is not present
return
# Restart sync process
self.swsync.process()
# Check if dest timestamp have not been updated
cdd = self.get_container_detail(account_dest, self.d_admin_token,
'dest', container_dest)
self.assertEqual(dest_lm, cdd[0]['x-container-meta-last-modified'])
# Check if new object is still present in dest
obj_detail = self.get_object_detail(account_dest, self.d_admin_token,
'dest', container_dest, 'lm-test')
self.assertEqual('lm-data', obj_detail[1])
def test_create_objects(self):
get_object_created = []
return_dict_ref = {'test': {'container_a': {'objects': []},
'container_b': {'objects': []}}}
def put_object(*args, **kwargs):
get_object_created.append(args[1:])
self.stubs.Set(FakeSWConnection,
'put_object',
put_object)
cnx = self.get_connection()
filler.create_objects(cnx, 'test', 2, 2048, return_dict_ref)
objects_ca = return_dict_ref['test']['container_a']['objects']
objects_cb = return_dict_ref['test']['container_b']['objects']
self.assertEqual(len(objects_ca), 2)
self.assertEqual(len(objects_cb), 2)
def test_create_objects_fail(self):
get_object_created = []
return_dict_ref = {'test': {'container_a': {'objects': []}}}
self.attempts = 0
def put_object(*args, **kwargs):
if self.attempts == 0:
self.attempts += 1
raise swiftclient.client.ClientException('Fake err msg')
else:
self.attempts += 1
get_object_created.append(args[1:])
self.stubs.Set(FakeSWConnection, 'put_object', put_object)
cnx = self.get_connection()
filler.create_objects(cnx, 'test', 2, 2048, return_dict_ref)
objects_ca = return_dict_ref['test']['container_a']['objects']
self.assertEqual(len(objects_ca), 1)
def test_create_objects_fail(self):
get_object_created = []
return_dict_ref = {'test': {'container_a': {'objects': []}}}
self.attempts = 0
def put_object(*args, **kwargs):
if self.attempts == 0:
self.attempts += 1
raise swiftclient.client.ClientException('Fake err msg')
else:
self.attempts += 1
get_object_created.append(args[1:])
self.stubs.Set(FakeSWConnection,
'put_object',
put_object)
cnx = self.get_connection()
filler.create_objects(cnx, 'test', 2, 2048, return_dict_ref)
objects_ca = return_dict_ref['test']['container_a']['objects']
self.assertEqual(len(objects_ca), 1)
def test_create_objects(self):
get_object_created = []
return_dict_ref = {
'test': {
'container_a': {
'objects': []
},
'container_b': {
'objects': []
}
}
}
def put_object(*args, **kwargs):
get_object_created.append(args[1:])
self.stubs.Set(FakeSWConnection, 'put_object', put_object)
cnx = self.get_connection()
filler.create_objects(cnx, 'test', 2, 2048, return_dict_ref)
objects_ca = return_dict_ref['test']['container_a']['objects']
objects_cb = return_dict_ref['test']['container_b']['objects']
self.assertEqual(len(objects_ca), 2)
self.assertEqual(len(objects_cb), 2)
def test_07_object_two_passes(self):
"""Objects modified two passes."""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
1, 1, index)
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 1, index_container)
filler.create_objects(tenant_cnx, acc, 3, 2048, index_container)
# Start sync process
self.swsync.process()
# Modify objects in containers
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
token = tenant_cnx.get_auth()[1]
c_o = self.list_objects_in_containers(account_id, token, 'orig')
for cont, objs in c_o.iteritems():
for obj in objs:
# Modify object meta
obj_d, data = self.get_object_detail(account_id,
token, 'orig',
cont, obj['name'])
meta = {k: v for k, v in obj_d.iteritems()
if k.startswith('x-object-meta')}
meta_k_names = [k.split('-')[-1] for k in meta]
headers = {}
headers['X-Object-Meta-a1'] = 'b1'
headers["X-Remove-Object-Meta-%s" % meta_k_names[0]] = 'x'
headers["X-Object-Meta-%s" % meta_k_names[1]] = 'b2'
self.post_object(account_id, token,
'orig',
headers=headers,
container=cont,
name=obj['name'])
# Create an object
self.put_object(account_id, token, 'orig',
cont, 'foofoo', 'barbarbar')
self.put_object(account_id, token, 'orig',
cont, 'foofoo1', 'barbarbar')
self.put_object(account_id, token, 'orig',
cont, 'foofoo2', 'barbarbar')
o_names = [o['name'] for o in objs]
# Delete an object
name = o_names[0]
self.delete_object(account_id, token, 'orig',
cont, name)
# Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in (
self.extract_created_a_u_iter(self.created)):
# Verify container listing
olo = self.list_objects_in_containers(account_id,
self.o_admin_token, 'orig')
old = self.list_objects_in_containers(account_id,
self.d_admin_token, 'dest')
# Verify we have the same amount of container
self.assertListEqual(olo.keys(), old.keys())
# For each container
for c, objs in olo.items():
for obj in objs:
# Verify first object detail returned by container server
match = [od for od in old[c] if od['name'] == obj['name']]
self.assertEqual(len(match), 1)
obj_d = match[0]
a = obj.copy()
b = obj_d.copy()
del a['last_modified']
del b['last_modified']
self.assertDictEqual(a, b)
# Verify object details from object server
obj_names = [d['name'] for d in olo[c]]
for obj_name in obj_names:
objd_o = self.get_object_detail(account_id,
self.o_admin_token, 'orig',
c, obj_name)
objd_d = self.get_object_detail(account_id,
self.d_admin_token, 'dest',
c, obj_name)
self.verify_aco_diff(objd_o, objd_d)
# Verify content
self.assertEqual(objd_o[1], objd_d[1])
def test_07_object_two_passes(self):
"""Objects modified two passes
"""
index = {}
index_container = {}
# Create account
self.created = filler.create_swift_account(self.o_ks_client,
self.pile,
1, 1, index)
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
acc = (account, account_id)
filler.create_containers(tenant_cnx, acc, 1, index_container)
filler.create_objects(tenant_cnx, acc, 3, 2048, index_container)
# Start sync process
self.swsync.process()
# Modify objects in containers
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
tenant_cnx = sclient.Connection(self.o_st,
"%s:%s" % (account, username),
self.default_user_password,
auth_version=2)
token = tenant_cnx.get_auth()[1]
c_o = self.list_objects_in_containers(account_id, token, 'orig')
for cont, objs in c_o.iteritems():
for obj in objs:
# Modify object meta
obj_d, data = self.get_object_detail(account_id,
token, 'orig',
cont, obj['name'])
meta = {k: v for k, v in obj_d.iteritems()
if k.startswith('x-object-meta')}
meta_k_names = [k.split('-')[-1] for k in meta]
headers = {}
headers['X-Object-Meta-a1'] = 'b1'
headers["X-Remove-Object-Meta-%s" % meta_k_names[0]] = 'x'
headers["X-Object-Meta-%s" % meta_k_names[1]] = 'b2'
self.post_object(account_id, token,
'orig',
headers=headers,
container=cont,
name=obj['name'])
# Create an object
self.put_object(account_id, token, 'orig',
cont, 'foofoo', 'barbarbar')
self.put_object(account_id, token, 'orig',
cont, 'foofoo1', 'barbarbar')
self.put_object(account_id, token, 'orig',
cont, 'foofoo2', 'barbarbar')
o_names = [o['name'] for o in objs]
# Delete an object
name = o_names[0]
self.delete_object(account_id, token, 'orig',
cont, name)
# Start sync process
self.swsync.process()
# Now verify dest
for account, account_id, username in \
self.extract_created_a_u_iter(self.created):
# Verify container listing
olo = self.list_objects_in_containers(account_id,
self.o_admin_token, 'orig')
old = self.list_objects_in_containers(account_id,
self.d_admin_token, 'dest')
# Verify we have the same amount of container
self.assertListEqual(olo.keys(), old.keys())
# For each container
for c, objs in olo.items():
for obj in objs:
# Verify first object detail returned by container server
match = [od for od in old[c] if od['name'] == obj['name']]
self.assertEqual(len(match), 1)
obj_d = match[0]
a = obj.copy()
b = obj_d.copy()
del a['last_modified']
del b['last_modified']
self.assertDictEqual(a, b)
# Verify object details from object server
obj_names = [d['name'] for d in olo[c]]
for obj_name in obj_names:
objd_o = self.get_object_detail(account_id,
self.o_admin_token, 'orig',
c, obj_name)
objd_d = self.get_object_detail(account_id,
self.d_admin_token, 'dest',
c, obj_name)
self.verify_aco_diff(objd_o, objd_d)
# Verify content
self.assertEqual(objd_o[1], objd_d[1])
