def test_sync_lazy_dkey(self):
# Create synced containers, but with no key at dest
source_container, dest_container =\
self._setup_synced_containers(dest_overrides={'sync_key': None})
# upload to source
object_name = 'object-%s' % uuid.uuid4()
client.put_object(self.url, self.token, source_container, object_name,
'test-body')
# cycle container-sync, nothing should happen
Manager(['container-sync']).once()
with self.assertRaises(ClientException) as err:
_junk, body = client.get_object(self.url, self.token,
dest_container, object_name)
self.assertEqual(err.exception.http_status, HTTP_NOT_FOUND)
# amend dest key
dest_headers = {'X-Container-Sync-Key': 'secret'}
client.put_container(self.url, self.token, dest_container,
headers=dest_headers)
# cycle container-sync, should replicate
Manager(['container-sync']).once()
_junk, body = client.get_object(self.url, self.token,
dest_container, object_name)
self.assertEqual(body, b'test-body')
def test_sync_static_symlink_different_container(self):
source_container, dest_container = self._setup_synced_containers()
symlink_cont = 'symlink-container-%s' % uuid.uuid4()
client.put_container(self.url, self.token, symlink_cont)
# upload a target to symlink container
target_name = 'target-%s' % uuid.uuid4()
target_body = b'target body'
etag = client.put_object(
self.url, self.token, symlink_cont, target_name,
target_body)
# upload a regular object
regular_name = 'regular-%s' % uuid.uuid4()
regular_body = b'regular body'
client.put_object(
self.url, self.token, source_container, regular_name,
regular_body)
# static symlink
target_path = '%s/%s' % (symlink_cont, target_name)
symlink_name = 'symlink-%s' % uuid.uuid4()
put_headers = {'X-Symlink-Target': target_path,
'X-Symlink-Target-Etag': etag}
# upload the symlink
client.put_object(
self.url, self.token, source_container, symlink_name,
'', headers=put_headers)
# verify object is a symlink
resp_headers, symlink_body = client.get_object(
self.url, self.token, source_container, symlink_name,
query_string='symlink=get')
self.assertEqual(b'', symlink_body)
self.assertIn('x-symlink-target', resp_headers)
self.assertIn('x-symlink-target-etag', resp_headers)
# verify symlink behavior
resp_headers, actual_target_body = client.get_object(
self.url, self.token, source_container, symlink_name)
self.assertEqual(target_body, actual_target_body)
self.assertIn('content-location', resp_headers)
content_location = resp_headers['content-location']
# cycle container-sync
Manager(['container-sync']).once()
# regular object should have synced
resp_headers, actual_target_body = client.get_object(
self.url, self.token, dest_container, regular_name)
self.assertEqual(regular_body, actual_target_body)
# static symlink gets synced, too
resp_headers, actual_target_body = client.get_object(
self.url, self.token, dest_container, symlink_name)
self.assertEqual(target_body, actual_target_body)
self.assertIn('content-location', resp_headers)
self.assertEqual(content_location, resp_headers['content-location'])
def test_storlet_acl_get_success(self):
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
exc_pattern = '^.*403 Forbidden.*$'
with self.assertRaisesRegexp(ClientException, exc_pattern):
swift_client.get_object(self.member_url,
self.member_token,
self.container,
'test_object',
headers=headers)
headers = {
'X-Storlet-Container-Read': self.conf.member_user,
'X-Storlet-Name': self.storlet_name
}
swift_client.post_container(self.url, self.token, self.container,
headers)
swift_client.head_container(self.url, self.token, self.container)
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
resp_dict = dict()
swift_client.get_object(self.member_url,
self.member_token,
self.container,
'test_object',
response_dict=resp_dict,
headers=headers)
self.assertEqual(200, resp_dict['status'])
def test_storlet_acl_get_success(self):
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
exc_pattern = '^.*403 Forbidden.*$'
with self.assertRaisesRegexp(ClientException, exc_pattern):
swift_client.get_object(self.member_url, self.member_token,
'myobjects', 'test_object',
headers=headers)
headers = {'X-Storlet-Container-Read': self.conf.member_user,
'X-Storlet-Name': self.storlet_name}
swift_client.post_container(self.url,
self.token,
'myobjects',
headers)
swift_client.head_container(self.url,
self.token,
'myobjects')
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
resp_dict = dict()
swift_client.get_object(self.member_url,
self.member_token,
'myobjects', 'test_object',
response_dict=resp_dict,
headers=headers)
self.assertEqual(resp_dict['status'], 200)
def invokeTestStorlet(self, op, withlog=False):
headers = {'X-Run-Storlet': self.storlet_name}
if withlog is True:
headers['X-Storlet-Generate-Log'] = 'True'
params = 'op={0}¶m2=val2'.format(op)
resp_dict = dict()
try:
resp_headers, gf = c.get_object(self.url, self.token, 'myobjects',
'test_object', None, None, params,
resp_dict, headers)
get_text = gf
get_response_status = resp_dict.get('status')
if withlog is True:
resp_headers, gf = c.get_object(self.url, self.token,
'storletlog', 'test.log',
None, None, None, None,
headers)
self.assertEqual(resp_headers.get('status'), 200)
gf.read()
self.assertEqual(resp_headers.get('status') == 200)
if op == 'print':
self.assertEqual(get_response_status, 200)
self.assertTrue('op' in get_text)
self.assertTrue('print' in get_text)
self.assertTrue('param2' in get_text)
self.assertTrue('val2' in get_text)
except Exception:
get_response_status = resp_dict.get('status')
if op == 'crash':
self.assertTrue(get_response_status >= 500 or
get_response_status == 404)
def invoke_storlet(self, op, withlog=False):
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
if withlog is True:
headers['X-Storlet-Generate-Log'] = 'True'
params = 'op={0}¶m2=val2'.format(op)
resp_dict = dict()
try:
resp_headers, get_text = swift_client.get_object(
self.url, self.token, self.container, 'test_object', None,
None, params, resp_dict, headers)
get_response_status = resp_dict.get('status')
except Exception:
get_response_status = resp_dict.get('status')
if op == 'crash':
self.assertTrue(get_response_status >= 500
or get_response_status == 404)
if withlog:
resp_headers, get_text = swift_client.get_object(
self.url, self.token, 'storletlog', 'test.log', None, None,
None, None, headers)
self.assertEqual(200, resp_headers.get('status'))
self.assertEqual('aaa', get_text.read())
if op == 'print':
self.assertEqual(200, get_response_status)
self.assertIn('op print', get_text)
self.assertIn('param2 val2', get_text)
def _do_test(self, overwrite_contents):
self.brain.put_container()
self.brain.stop_primary_half()
# put object to only 1 of 3 primaries
self.brain.put_object(contents=b'VERIFY')
self.brain.start_primary_half()
# Restart services and attempt to overwrite
with self.assertRaises(client.ClientException) as exc_mgr:
self.brain.put_object(headers={'If-None-Match': '*'},
contents=overwrite_contents)
self.assertEqual(exc_mgr.exception.http_status, 412)
# make sure we're GETting from the servers that missed the original PUT
self.brain.stop_handoff_half()
# verify the PUT did not complete
with self.assertRaises(client.ClientException) as exc_mgr:
client.get_object(self.url, self.token, self.container_name,
self.object_name)
self.assertEqual(exc_mgr.exception.http_status, 404)
# for completeness, run replicators...
Manager(['object-replicator']).once()
# ...and verify the object was not overwritten
_headers, body = client.get_object(self.url, self.token,
self.container_name,
self.object_name)
self.assertEqual(body, b'VERIFY')
def test_symlink_target(self):
if 'symlink' not in self.cluster_info:
raise unittest.SkipTest("Symlink not enabled in proxy; can't test "
"symlink to reserved name")
int_client = self.make_internal_client()
# create link container first, ensure account gets created too
client.put_container(self.url, self.token, 'c1')
# Create reserve named container
tgt_cont = get_reserved_name('container-%s' % uuid4())
int_client.create_container(self.account, tgt_cont)
# sanity, user can't get to reserved name
with self.assertRaises(ClientException) as cm:
client.head_container(self.url, self.token, tgt_cont)
self.assertEqual(412, cm.exception.http_status)
tgt_obj = get_reserved_name('obj-%s' % uuid4())
int_client.upload_object(BytesIO(b'target object'), self.account,
tgt_cont, tgt_obj)
metadata = int_client.get_object_metadata(self.account, tgt_cont,
tgt_obj)
etag = metadata['etag']
# users can write a dynamic symlink that targets a reserved
# name object
client.put_object(self.url,
self.token,
'c1',
'symlink',
headers={
'X-Symlink-Target':
'%s/%s' % (tgt_cont, tgt_obj),
'Content-Type': 'application/symlink',
})
# but can't read the symlink
with self.assertRaises(ClientException) as cm:
client.get_object(self.url, self.token, 'c1', 'symlink')
self.assertEqual(412, cm.exception.http_status)
# user's can't create static symlink to reserved name
with self.assertRaises(ClientException) as cm:
client.put_object(self.url,
self.token,
'c1',
'static-symlink',
headers={
'X-Symlink-Target':
'%s/%s' % (tgt_cont, tgt_obj),
'X-Symlink-Target-Etag': etag,
'Content-Type': 'application/symlink',
})
self.assertEqual(412, cm.exception.http_status)
# clean-up
client.delete_object(self.url, self.token, 'c1', 'symlink')
int_client.delete_object(self.account, tgt_cont, tgt_obj)
int_client.delete_container(self.account, tgt_cont)
def _work(self, op):
running_ops = self.outstanding
self.outstanding += 1
t1 = time()
try:
# client = Connection(authurl='http://192.168.16.12:8080/auth/v1.0', user='******', key='testing')
# client.put_object(self.container,'%s-%s' % (self.base, op), self.data)
get_object(self.storage_url, token=self.token, container=self.container, name='%s-%s' % (self.base, op))
t2 = time()
elapsed = (t2-t1) * 1000
self.outstanding -= 1
LOG.info("Operation #%d took %.2f ms (%.2f MB/s, %d ops outstanding on arrival)" % (op, elapsed, (self.data_size / (1024.*1024)) / (t2 - t1) , running_ops))
entry = { "Operation": op,
"Arrival-time": t1,
"Completion-time": t2,
"Elapsed": elapsed,
"Outstanding-on-arrival": running_ops,
"Outstanding-on-completion": self.outstanding }
print '%d, %f, %f, %.2f, %d, %d' % (op, t1, t2, elapsed, running_ops, self.outstanding)
return entry
except KeyboardInterrupt:
self.outstanding -= 1
self.running = False
return None
except Exception:
self.errors += 1
raise
def test_sync_lazy_dkey(self):
# Create synced containers, but with no key at dest
source_container, dest_container =\
self._setup_synced_containers('secret', None)
# upload to source
object_name = 'object-%s' % uuid.uuid4()
client.put_object(self.url, self.token, source_container, object_name,
'test-body')
# cycle container-sync, nothing should happen
Manager(['container-sync']).once()
with self.assertRaises(ClientException) as err:
_junk, body = client.get_object(self.url, self.token,
dest_container, object_name)
self.assertEqual(err.exception.http_status, HTTP_NOT_FOUND)
# amend dest key
dest_headers = {'X-Container-Sync-Key': 'secret'}
client.put_container(self.url, self.token, dest_container,
headers=dest_headers)
# cycle container-sync, should replicate
Manager(['container-sync']).once()
_junk, body = client.get_object(self.url, self.token,
dest_container, object_name)
self.assertEqual(body, 'test-body')
def test_put(self):
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
querystring = "action=compress"
# simply set 1KB string data to compress
data = b'A' * 1024
response = dict()
c.put_object(self.url, self.token, self.container, self.storlet_file,
data, None, None, None, "application/octet-stream",
headers, None, None, querystring, response)
querystring = "action=uncompress"
original_headers, original_content = \
c.get_object(self.url, self.token, self.container,
self.storlet_file,
response_dict=dict())
object_length = int(original_headers['content-length'])
self.assertLess(object_length, len(data))
processed_headers, returned_content = \
c.get_object(self.url, self.token, self.container,
self.storlet_file,
query_string=querystring, response_dict=dict(),
headers=headers, resp_chunk_size=object_length)
processed_content = b''
for chunk in returned_content:
if chunk:
processed_content += chunk
self.assertEqual(data, processed_content)
def _do_test(self, overwrite_contents):
self.brain.put_container()
self.brain.stop_primary_half()
# put object to only 1 of 3 primaries
self.brain.put_object(contents='VERIFY')
self.brain.start_primary_half()
# Restart services and attempt to overwrite
with self.assertRaises(client.ClientException) as exc_mgr:
self.brain.put_object(headers={'If-None-Match': '*'},
contents=overwrite_contents)
self.assertEqual(exc_mgr.exception.http_status, 412)
# make sure we're GETting from the servers that missed the original PUT
self.brain.stop_handoff_half()
# verify the PUT did not complete
with self.assertRaises(client.ClientException) as exc_mgr:
client.get_object(
self.url, self.token, self.container_name, self.object_name)
self.assertEqual(exc_mgr.exception.http_status, 404)
# for completeness, run replicators...
Manager(['object-replicator']).once()
# ...and verify the object was not overwritten
_headers, body = client.get_object(
self.url, self.token, self.container_name, self.object_name)
self.assertEqual(body, 'VERIFY')
def delete_given_folder(request, container, foldername):
""" Recursively delete all the objects in the given folder and then
delete the folder itself. """
storage_url = request.session.get('storage_url', '')
auth_token = request.session.get('auth_token', '')
# Get all objects within folder.
meta, objects = client.get_container(
storage_url, auth_token, container,
headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')},
delimiter='/', prefix=foldername)
# Recursive call to delete subfolders.
pseudofolders, objs = pseudofolder_object_list(objects, foldername)
for folder in pseudofolders:
delete_given_folder(request, container, folder[0])
# Delete all objects.
for obj in objs:
delete_given_object(request, container, obj["name"])
# Delete the folder itself.
try:
client.get_object(
storage_url, auth_token, container, foldername,
headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')})
delete_given_object(request, container, foldername)
except:
# If folder does not exist, pass
pass
def test_storlet_acl_get_fail(self):
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
exc_pattern = '^.*403 Forbidden.*$'
with self.assertRaisesRegexp(ClientException, exc_pattern):
swift_client.get_object(self.member_url, self.member_token,
self.container, 'test_object',
headers=headers)
def test_storlet_acl_get_fail(self):
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
exc_pattern = '^.*403 Forbidden.*$'
with self.assertRaisesRegexp(ClientException, exc_pattern):
swift_client.get_object(self.member_url, self.member_token,
'myobjects', 'test_object',
headers=headers)
def test_query_string(self):
c.http_connection = self.fake_http_connection(200,
query_string="hello=20")
c.get_object('http://www.test.com',
'asdf',
'asdf',
'asdf',
query_string="hello=20")
def test_missing_primaries(self):
# Create container
container = 'container-%s' % uuid4()
client.put_container(self.url,
self.token,
container,
headers={'X-Storage-Policy': self.policy.name})
# Create container/obj (goes to all three primaries)
obj = 'object-%s' % uuid4()
client.put_object(self.url, self.token, container, obj, b'VERIFY')
odata = client.get_object(self.url, self.token, container, obj)[-1]
if odata != b'VERIFY':
raise Exception('Object GET did not return VERIFY, instead it '
'returned: %s' % repr(odata))
# Kill all primaries obj server
obj = 'object-%s' % uuid4()
opart, onodes = self.object_ring.get_nodes(self.account, container,
obj)
for onode in onodes:
kill_server((onode['ip'], onode['port']), self.ipport2server)
# Indirectly (i.e., through proxy) try to GET object, it should return
# a 503, since all primaries will Timeout and handoffs return a 404.
try:
client.get_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 503)
else:
self.fail("Expected ClientException but didn't get it")
# Restart the first container/obj primary server again
onode = onodes[0]
start_server((onode['ip'], onode['port']), self.ipport2server)
# Send a delete that will reach first primary and handoff.
# Sure, the DELETE will return a 404 since the handoff doesn't
# have a .data file, but object server will still write a
# Tombstone in the handoff node!
try:
client.delete_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
# kill the first container/obj primary server again
kill_server((onode['ip'], onode['port']), self.ipport2server)
# a new GET should return a 404, since all primaries will Timeout
# and the handoff will return a 404 but this time with a tombstone
try:
client.get_object(self.url, self.token, container, obj)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def test_skip_sync_when_misconfigured(self):
source_container, dest_container = self._setup_synced_containers()
container_name = 'versioned-%s' % uuid.uuid4()
version_hdr = {'X-Versions-Enabled': 'true'}
client.put_container(self.url, self.token, container_name,
headers=version_hdr)
# some sanity checks
object_name = 'object-%s' % uuid.uuid4()
client.put_object(self.url, self.token, container_name, object_name,
'version1')
client.put_object(self.url, self.token, container_name, object_name,
'version2')
resp_headers, listing = client.get_container(
self.url, self.token, container_name,
query_string='versions')
self.assertEqual(2, len(listing))
sync_headers = {}
sync_to = '//%s/%s/%s/%s' % (self.realm, self.cluster, self.account,
dest_container)
sync_headers['X-Container-Sync-To'] = sync_to
sync_headers['X-Container-Sync-Key'] = 'secret'
# use internal client to set container-sync headers
# since it doesn't have container_sync middleware in pipeline
# allowing us to bypass checks
int_client = self.make_internal_client()
# TODO: what a terrible hack, maybe we need to extend internal
# client to allow caller to become a swift_owner??
int_client.app.app.app.app.swift_owner_headers = []
int_client.set_container_metadata(self.account, container_name,
metadata=sync_headers)
headers = client.head_container(self.url, self.token,
container_name)
# This should never happen, but if it does because of eventual
# consistency or a messed up pipeline, container-sync should
# skip syncing container.
self.assertEqual('True', headers.get('x-versions-enabled'))
self.assertEqual('secret', headers.get('x-container-sync-key'))
self.assertEqual(sync_to, headers.get('x-container-sync-to'))
# cycle container-sync
Manager(['container-sync']).once()
with self.assertRaises(ClientException) as cm:
client.get_object(
self.url, self.token, dest_container, object_name)
self.assertEqual(404, cm.exception.http_status) # sanity check
def test_get(self):
resp = dict()
req_headers = {'X-Run-Storlet': self.storlet_name}
with self.assertRaises(ClientException) as cm:
client.get_object(self.url,
self.token,
self.container,
self.storlet_file,
response_dict=resp,
headers=req_headers)
e = cm.exception
self.assertEqual(e.http_status, 503)
def test_ec_primary_timeout(self):
container_name = 'container-%s' % uuid4()
object_name = 'object-%s' % uuid4()
# create EC container
headers = {'X-Storage-Policy': self.policy.name}
client.put_container(self.url,
self.token,
container_name,
headers=headers)
# PUT object, should go to primary nodes
old_contents = Body()
client.put_object(self.url,
self.token,
container_name,
object_name,
contents=old_contents)
# get our node lists
opart, onodes = self.object_ring.get_nodes(self.account,
container_name, object_name)
# shutdown three of the primary data nodes
for i in range(3):
failed_primary = onodes[i]
failed_primary_device_path = self.device_dir(
'object', failed_primary)
self.kill_drive(failed_primary_device_path)
# Indirectly (i.e., through proxy) try to GET object, it should return
# a 503, since all primaries will Timeout and handoffs return a 404.
try:
client.get_object(self.url, self.token, container_name,
object_name)
except client.ClientException as err:
self.assertEqual(err.http_status, 503)
else:
self.fail("Expected ClientException but didn't get it")
# Send a delete to write down tombstones in the handoff nodes
client.delete_object(self.url, self.token, container_name, object_name)
# Now a new GET should return 404 because the handoff nodes
# return a 404 with a Tombstone.
try:
client.get_object(self.url, self.token, container_name,
object_name)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def test_ec_missing_all_durable_fragments(self):
# This tests helps assert the behavior that when
# the proxy has enough fragments to reconstruct the object
# but none are marked as durable, the proxy should return a 404.
container_name = 'container-%s' % uuid4()
object_name = 'object-%s' % uuid4()
# create EC container
headers = {'X-Storage-Policy': self.policy.name}
client.put_container(self.url, self.token, container_name,
headers=headers)
# PUT object, should go to primary nodes
client.put_object(self.url, self.token, container_name,
object_name, contents=b'object contents')
# get our node lists
opart, onodes = self.object_ring.get_nodes(
self.account, container_name, object_name)
# sanity test
odata = client.get_object(self.url, self.token, container_name,
object_name)[-1]
self.assertEqual(b'object contents', odata)
# make all fragments non-durable
for node in onodes:
part_dir = self.storage_dir(node, part=opart)
for dirs, subdirs, files in os.walk(part_dir):
for fname in files:
if fname.endswith('.data'):
non_durable_fname = fname.replace('#d', '')
os.rename(os.path.join(dirs, fname),
os.path.join(dirs, non_durable_fname))
break
headers = direct_client.direct_head_object(
node, opart, self.account, container_name, object_name,
headers={
'X-Backend-Storage-Policy-Index': self.policy.idx,
'X-Backend-Fragment-Preferences': json.dumps([])})
self.assertNotIn('X-Backend-Durable-Timestamp', headers)
# Now a new GET should return *404* because all fragments
# are non-durable, even if they are reconstructable
try:
client.get_object(self.url, self.token, container_name,
object_name)
except client.ClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Expected ClientException but didn't get it")
def invoke_storlet(self, start, end, first_partition, max_record_line,
columns_selection, where_clause):
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
headers['X-Storlet-Range'] = 'bytes=%d-%d' % (start,
end + max_record_line)
headers['X-Storlet-Parameter-1'] = '%s:%s' % ('start', start)
headers['X-Storlet-Parameter-2'] = '%s:%s' % ('end', end)
headers['X-Storlet-Parameter-3'] = '%s:%s' % ('max_record_line',
max_record_line)
headers['X-Storlet-Parameter-4'] = '%s:%s' % ('first_partition',
first_partition)
headers['X-Storlet-Parameter-5'] = '%s:%s' % ('selected_columns',
columns_selection)
headers['X-Storlet-Parameter-6'] = '%s:%s' % ('where_clause',
where_clause)
_, content = c.get_object(self.url,
self.token,
self.container,
self.storlet_file,
response_dict=dict(),
headers=headers)
if not six.PY2:
content = content.decode('utf-8')
return content
def object_info(container,obj):
headers, content = client.get_object(url = storageURL,
token = authtoken,
container = container,
name = obj
)
return headers, content
def test_get_multipart_mime_response(self):
obj = 'small'
obj2 = 'small2'
body = '0123456789abcd'
body2 = 'efghijklmnopqr'
c.put_object(self.url, self.token, self.container, obj, body)
c.put_object(self.url, self.token, self.container, obj2, body2)
headers = {
'X-Run-Storlet': self.storlet_name,
'X-Storlet-Extra-Resources': os.path.join('/' + self.container,
obj2)
}
headers.update(self.additional_headers)
resp_headers, resp_content = c.get_object(self.url,
self.token,
self.container,
obj,
headers=headers)
multipart_prefix = 'multipart/mixed; boundary='
# N.B. swiftclient makes the header key as lower case
self.assertIn('content-type', resp_headers)
self.assertIn(multipart_prefix, resp_headers['content-type'])
boundary = resp_headers['content-type'][len(multipart_prefix):]
self.assertEqual(
'%s\n--%s\n%s\n--%s--' % (body, boundary, body2, boundary),
resp_content)
def test_put(self):
objname = self.storlet_file + '-put'
resp = dict()
req_headers = {'X-Run-Storlet': self.storlet_name}
client.put_object(self.url,
self.token,
self.container,
objname,
self.content,
response_dict=resp,
headers=req_headers)
self.assertEqual(201, resp['status'])
resp = dict()
headers, content = client.get_object(self.url,
self.token,
self.container,
objname,
response_dict=resp)
self.assertEqual(200, resp['status'])
self.assertEqual('simple', headers['x-object-meta-test'])
self.assertEqual(self.content, content)
resp = dict()
client.delete_object(self.url,
self.token,
self.container,
objname,
response_dict=resp)
self.assertEqual(204, resp['status'])
def invoke_storlet(url, token,op, params = None, global_params = None):
if params != None:
querystring=''
for key in params:
querystring += '%s=%s,' % (key, params[key])
querystring = querystring[:-1]
else:
querystring = None
metadata = {'X-Run-Storlet': STORLET_NAME }
if op == 'GET':
# Get original object
original_headers, original_content = c.get_object(url, token,
'myobjects',
SOURCE_FILE,
response_dict=dict(),
headers=metadata)
assert(original_headers['X-Object-Meta-key1'.lower()] == '1')
assert(original_headers['X-Object-Meta-key2'.lower()] == '2')
assert(original_headers['X-Object-Meta-key3'.lower()] == '3')
assert(original_headers['X-Object-Meta-key4'.lower()] == '4')
assert(original_headers['X-Object-Meta-key5'.lower()] == '5')
assert(original_headers['X-Object-Meta-key6'.lower()] == '6')
assert(original_headers['X-Object-Meta-key7'.lower()] == '7')
assert(original_headers['X-Object-Meta-key8'.lower()] == '8')
assert(original_headers['X-Object-Meta-key9'.lower()] == '9')
assert(original_headers['X-Object-Meta-key10'.lower()] == '10')
assert(original_headers['X-Object-Meta-override_key'.lower()] == 'new_value')
def invoke_storlet(url, token, storletName, jarName, objectName, mode):
resp = dict()
if mode == 'PUT':
f = open('../StorletSamples/' + storletName + '/sampleData.txt', 'r')
c.put_object(url,
token,
'myobjects',
objectName,
f,
headers={'X-Run-Storlet': jarName},
response_dict=resp)
f.close()
if mode == 'GET':
resp_headers, saved_content = c.get_object(
url,
token,
'myobjects',
objectName,
headers={'X-Run-Storlet': jarName},
response_dict=resp)
assert (resp['status'] == 200 or resp['status'] == 201)
if mode == 'GET':
return resp_headers, saved_content
def test_metadata_get(self, params=None, global_params=None):
if params is not None:
querystring = ''
for key in params:
querystring += '%s=%s,' % (key, params[key])
querystring = querystring[:-1]
else:
querystring = None
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
original_headers, original_content = \
c.get_object(self.url, self.token,
self.container, self.storlet_file,
response_dict=dict(), headers=headers)
self.assertEqual('1', original_headers['X-Object-Meta-key1'.lower()])
self.assertEqual('2', original_headers['X-Object-Meta-key2'.lower()])
self.assertEqual('3', original_headers['X-Object-Meta-key3'.lower()])
self.assertEqual('4', original_headers['X-Object-Meta-key4'.lower()])
self.assertEqual('5', original_headers['X-Object-Meta-key5'.lower()])
self.assertEqual('6', original_headers['X-Object-Meta-key6'.lower()])
self.assertEqual('7', original_headers['X-Object-Meta-key7'.lower()])
self.assertEqual('8', original_headers['X-Object-Meta-key8'.lower()])
self.assertEqual('9', original_headers['X-Object-Meta-key9'.lower()])
self.assertEqual('10', original_headers['X-Object-Meta-key10'.lower()])
omv = original_headers['X-Object-Meta-override_key'.lower()]
self.assertEqual('new_value', omv)
def test_metadata_get(self, params=None, global_params=None):
if params is not None:
querystring = ''
for key in params:
querystring += '%s=%s,' % (key, params[key])
querystring = querystring[:-1]
else:
querystring = None
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
original_headers, original_content = \
c.get_object(self.url, self.token,
'myobjects', self.storlet_file,
response_dict=dict(), headers=headers)
self.assertEqual(original_headers['X-Object-Meta-key1'.lower()], '1')
self.assertEqual(original_headers['X-Object-Meta-key2'.lower()], '2')
self.assertEqual(original_headers['X-Object-Meta-key3'.lower()], '3')
self.assertEqual(original_headers['X-Object-Meta-key4'.lower()], '4')
self.assertEqual(original_headers['X-Object-Meta-key5'.lower()], '5')
self.assertEqual(original_headers['X-Object-Meta-key6'.lower()], '6')
self.assertEqual(original_headers['X-Object-Meta-key7'.lower()], '7')
self.assertEqual(original_headers['X-Object-Meta-key8'.lower()], '8')
self.assertEqual(original_headers['X-Object-Meta-key9'.lower()], '9')
self.assertEqual(original_headers['X-Object-Meta-key10'.lower()], '10')
omv = original_headers['X-Object-Meta-override_key'.lower()]
self.assertEqual(omv, 'new_value')
def get_object(self, container_name, object_name):
headers, body = client.get_object(
self.url, self.token, container_name, object_name, resp_chunk_size=64 * 2 ** 10
)
resp_checksum = md5()
for chunk in body:
resp_checksum.update(chunk)
return resp_checksum.hexdigest()
def test_request_headers(self):
request_args = {}
def fake_request(method, url, body=None, headers=None):
request_args["method"] = method
request_args["url"] = url
request_args["body"] = body
request_args["headers"] = headers
return
conn = self.fake_http_connection(200)("http://www.test.com/")
conn[1].request = fake_request
headers = {"Range": "bytes=1-2"}
c.get_object("url_is_irrelevant", "TOKEN", "container", "object", http_conn=conn, headers=headers)
self.assertFalse(request_args["headers"] is None, "No headers in the request")
self.assertTrue("Range" in request_args["headers"], "No Range header in the request")
self.assertEqual(request_args["headers"]["Range"], "bytes=1-2")
def download_collection(request, container, prefix=None, non_recursive=False):
""" Download the content of an entire container/pseudofolder
as a Zip file. """
storage_url = request.session.get('storage_url', '')
auth_token = request.session.get('auth_token', '')
delimiter = '/' if non_recursive else None
try:
x, objects = client.get_container(
storage_url,
auth_token,
container,
delimiter=delimiter,
prefix=prefix
)
except client.ClientException:
return HttpResponseForbidden()
x, objs = pseudofolder_object_list(objects, prefix)
# Do not provide download when the folder is empty.
if len(x) + len(objs) == 0:
messages.add_message(
request, messages.ERROR, _("Unable to download, no files found."))
if prefix: # Return user to object view
# remove the last prefix. ex "dir1/dir2/" -> "dir1"
prefix = prefix[0:prefix.rfind('/', 0, prefix.rfind('/'))]
return redirect(swiftbrowser.views.objectview, container=container,
prefix=prefix)
else: # Return user to the container view
return redirect(swiftbrowser.views.containerview)
output = StringIO()
zipf = zipfile.ZipFile(output, 'w')
for o in objs:
name = o['name']
try:
x, content = client.get_object(storage_url, auth_token, container,
name)
except client.ClientException:
return HttpResponseForbidden()
if prefix:
name = name[len(prefix):]
zipf.writestr(name, content)
zipf.close()
if prefix:
filename = prefix.split('/')[-2]
else:
filename = container
response = HttpResponse(output.getvalue(), 'application/zip')
response['Content-Disposition'] = 'attachment; filename="%s.zip"'\
% (filename)
output.close()
return response
def invokeTestStorlet(url, token, op, withlog=False):
headers = {'X-Run-Storlet':TEST_STORLET_NAME}
if withlog == True:
headers['X-Storlet-Generate-Log'] = 'True'
params = 'op={0}¶m2=val2'.format(op)
resp_dict = dict()
try:
resp_headers, gf = c.get_object(url, token, 'myobjects',
'test_object', None, None, params,
resp_dict, headers)
#print resp_dict
get_text = gf
#print get_text
get_response_status = resp_dict.get('status')
if withlog == True:
resp_headers, gf = c.get_object(url, token,
'storletlog', 'test.log',
None, None, None, None, headers)
assert resp_headers.get('status') == 200
text = gf.read()
assert resp_headers.get('status') == 200
#print text
if op == 'print':
assert get_response_status == 200
assert 'op' in get_text
assert 'print' in get_text
assert 'param2' in get_text
assert 'val2' in get_text
except Exception as e:
get_response_status = resp_dict.get('status')
if op == 'crash':
print get_response_status
assert get_response_status >= 500 or get_response_status == 404
if op == 'hold':
#print get_response_status
assert get_response_status >= 500 or get_response_status == 404
if op == 'print':
#print get_response_status
raise e
def get_object(self, container, object):
try:
return client.get_object(self.storage_url,
self.token,
container,
object,
http_conn=self.http_conn)
except:
return None
def proxy_get(self):
# GET object
headers, body = client.get_object(
self.url, self.token, self.container_name, self.object_name, resp_chunk_size=64 * 2 ** 10
)
resp_checksum = md5()
for chunk in body:
resp_checksum.update(chunk)
return resp_checksum.hexdigest()
def get_object(self, container_name, object_name):
headers, body = client.get_object(self.url, self.token,
container_name,
object_name,
resp_chunk_size=64 * 2 ** 10)
resp_checksum = md5()
for chunk in body:
resp_checksum.update(chunk)
return resp_checksum.hexdigest()
def test_get(self):
resp = dict()
req_headers = {'X-Run-Storlet': self.storlet_name}
headers, content = client.get_object(
self.url, self.token, self.container, self.storlet_file,
response_dict=resp, headers=req_headers)
self.assertEqual(200, resp['status'])
self.assertEqual('simple', headers['x-object-meta-test'])
self.assertEqual(self.content, content)
def invokeTestStorlet(url, token, op, withlog=False):
headers = {'X-Run-Storlet': TEST_STORLET_NAME}
if withlog == True:
headers['X-Storlet-Generate-Log'] = 'True'
params = 'op={0}¶m2=val2'.format(op)
resp_dict = dict()
try:
resp_headers, gf = c.get_object(url, token, 'myobjects', 'test_object',
None, None, params, resp_dict, headers)
#print resp_dict
get_text = gf
#print get_text
get_response_status = resp_dict.get('status')
if withlog == True:
resp_headers, gf = c.get_object(url, token, 'storletlog',
'test.log', None, None, None, None,
headers)
assert resp_headers.get('status') == 200
text = gf.read()
assert resp_headers.get('status') == 200
#print text
if op == 'print':
assert get_response_status == 200
assert 'op' in get_text
assert 'print' in get_text
assert 'param2' in get_text
assert 'val2' in get_text
except Exception as e:
get_response_status = resp_dict.get('status')
if op == 'crash':
print get_response_status
assert get_response_status >= 500 or get_response_status == 404
if op == 'hold':
#print get_response_status
assert get_response_status >= 500 or get_response_status == 404
if op == 'print':
#print get_response_status
raise e
def test_sync_unexpired_object_metadata(self):
# verify that metadata can be sync'd to a frag that has missed a POST
# and consequently that frag appears to be expired, when in fact the
# POST removed the x-delete-at header
client.put_container(self.url, self.token, self.container_name,
headers={'x-storage-policy': self.policy.name})
opart, onodes = self.object_ring.get_nodes(
self.account, self.container_name, self.object_name)
delete_at = int(time.time() + 3)
contents = 'body-%s' % uuid.uuid4()
headers = {'x-delete-at': delete_at}
client.put_object(self.url, self.token, self.container_name,
self.object_name, headers=headers, contents=contents)
# fail a primary
post_fail_node = random.choice(onodes)
post_fail_path = self.device_dir('object', post_fail_node)
self.kill_drive(post_fail_path)
# post over w/o x-delete-at
client.post_object(self.url, self.token, self.container_name,
self.object_name, {'content-type': 'something-new'})
# revive failed primary
self.revive_drive(post_fail_path)
# wait for the delete_at to pass, and check that it thinks the object
# is expired
timeout = time.time() + 5
while time.time() < timeout:
try:
direct_client.direct_head_object(
post_fail_node, opart, self.account, self.container_name,
self.object_name, headers={
'X-Backend-Storage-Policy-Index': int(self.policy)})
except direct_client.ClientException as err:
if err.http_status != 404:
raise
break
else:
time.sleep(0.1)
else:
self.fail('Failed to get a 404 from node with expired object')
self.assertEqual(err.http_status, 404)
self.assertIn('X-Backend-Timestamp', err.http_headers)
# but from the proxy we've got the whole story
headers, body = client.get_object(self.url, self.token,
self.container_name,
self.object_name)
self.assertNotIn('X-Delete-At', headers)
self.reconstructor.once()
# ... and all the nodes have the final unexpired state
for node in onodes:
headers = direct_client.direct_head_object(
node, opart, self.account, self.container_name,
self.object_name, headers={
'X-Backend-Storage-Policy-Index': int(self.policy)})
self.assertNotIn('X-Delete-At', headers)
def test_request_headers(self):
request_args = {}
def fake_request(method, url, body=None, headers=None):
request_args['method'] = method
request_args['url'] = url
request_args['body'] = body
request_args['headers'] = headers
return
conn = self.fake_http_connection(200)('http://www.test.com/')
conn[1].request = fake_request
headers = {'Range': 'bytes=1-2'}
c.get_object('url_is_irrelevant', 'TOKEN', 'container', 'object',
http_conn=conn, headers=headers)
self.assertFalse(request_args['headers'] is None,
"No headers in the request")
self.assertTrue('Range' in request_args['headers'],
"No Range header in the request")
self.assertEqual(request_args['headers']['Range'], 'bytes=1-2')
def _test_get_range(self, start, end, expected):
srange = 'bytes=%d-%d' % (start, end)
headers = {'X-Run-Storlet': self.storlet_name,
'X-Storlet-Range': srange}
junk, content = c.get_object(self.url, self.token,
'myobjects',
'small',
headers=headers,
response_dict=dict())
self.assertEqual(content, expected)
def test_request_headers(self):
request_args = {}
def fake_request(method, url, body=None, headers=None):
request_args['method'] = method
request_args['url'] = url
request_args['body'] = body
request_args['headers'] = headers
return
conn = self.fake_http_connection(200)('http://www.test.com/')
conn[1].request = fake_request
headers = {'Range': 'bytes=1-2'}
c.get_object('url_is_irrelevant', 'TOKEN', 'container', 'object',
http_conn=conn, headers=headers)
self.assertFalse(request_args['headers'] is None,
"No headers in the request")
self.assertTrue('Range' in request_args['headers'],
"No Range header in the request")
self.assertEqual(request_args['headers']['Range'], 'bytes=1-2')
def test_extract_face(self):
self.invoke_storlet(
self.container_name, 'eran.jpg',
os.path.join(self.container_name, 'small_eran.jpeg'), 'eran')
headers, body = client.get_object(self.url, self.token,
self.container_name,
'small_eran.jpeg')
self.assertTrue('x-object-meta-rect' in headers)
self.assertTrue(headers['x-object-meta-rect'] == '[498 119 621 242]')
def load(self, credentials):
if credentials is None:
raise BadRequest('Swift backend requires credentials in x-auth-token headers')
log.debug('Loading Swift object %s' % self.context.value)
protocol, schema, host, path = parse_uri(self.context.value)
uri = "%s:%s" % (schema, host)
base_path, objname = path.split('/', 1)
response, contents = client.get_object(uri, credentials, base_path, objname)
return StringIO.StringIO(contents)
def proxy_get(self):
# GET object
headers, body = client.get_object(self.url, self.token,
self.container_name,
self.object_name,
resp_chunk_size=64 * 2 ** 10)
resp_checksum = md5(usedforsecurity=False)
for chunk in body:
resp_checksum.update(chunk)
return headers, resp_checksum.hexdigest()
def invoke_identity_on_get_SLO(url, token):
metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME }
response = dict()
headers, body = c.get_object(url, token,
'myobjects','assembly',
query_string = None,
response_dict=response,
resp_chunk_size=1048576,
headers=metadata)
compare_slo_to_chunks(body)
def test_get_SLO(self):
headers = {'X-Run-Storlet': self.storlet_name}
response = dict()
headers, body = c.get_object(self.url, self.token,
'myobjects', 'assembly',
query_string=None,
response_dict=response,
resp_chunk_size=1048576,
headers=headers)
self.compare_slo_to_chunks(body)
def get_obj(request, container, object_name):
"""Download an object from Swift
:param container: container of swift where object is stored
:param object_name: ID of object
:return: object bytestream
"""
try:
storage_url, auth_token = _get_auth_data(request.session)
try:
obj_headers, obj = client.get_object(storage_url, auth_token, container, object_name)
except swift_exception.ClientException as e:
storage_url, auth_token = _reauthorize(request)
obj_headers, obj = client.get_object(storage_url, auth_token, container, object_name)
filetype = obj_headers['x-object-meta-format'].lower()
filename = obj_headers['x-object-meta-name']
header = obj_headers
# print obj_headers
if 'download' in request.GET:
if request.GET['download']:
extension = ''
if len(filename.split('.')) > 1:
extension = '.' + filename.split('.')[-1]
filename = request.GET['download'] + extension
header['Content-Disposition'] = 'attachment;filename="%s"' % filename
else:
header['Content-Disposition'] = 'inline;filename="%s"' % filename
if filetype == 'pdf':
header['X-Frame-Options'] = ''
return Response(obj, headers=header, content_type='application/pdf')
content_type = ''
if filetype in video_formats:
content_type = 'video/' + filetype
if filetype in image_types:
content_type = 'image/' + filetype
if content_type:
return Response(obj, headers=header, content_type=content_type)
return Response(obj, headers=header)
except swift_exception.ClientException as e:
print e
return Response('Please Contact Your Administrator', status=status.HTTP_401_UNAUTHORIZED)
except Exception as e:
print e
return Response(e.message, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
def read(self, read_bytes=None, offset=None):
''' Read the object from swift if not read_bytes read the whole
object'''
headers = {}
if read_bytes is not None and offset is not None:
headers["Range"] = "bytes=%s-%s" % (offset, offset + read_bytes)
return get_object(self.swift_info["url"], self.token,
self.swift_info["container"], self.swift_info["object"],
headers=headers)[1][:read_bytes]
def test_get_SLO(self):
headers = {'X-Run-Storlet': self.storlet_name}
headers.update(self.additional_headers)
response = dict()
headers, body = client.get_object(self.url, self.token,
self.container, 'assembly',
query_string=None,
response_dict=response,
resp_chunk_size=1024 * 1024,
headers=headers)
self.compare_slo_to_chunks(body)
def test_execdep(self):
headers = {"X-Run-Storlet": self.storlet_name}
headers.update(self.additional_headers)
resp = dict()
resp_headers, gf = c.get_object(
self.url, self.token, "myobjects", self.storlet_file, response_dict=resp, headers=headers
)
self.assertIn("x-object-meta-depend-ret-code", resp_headers)
self.assertEqual("42", resp_headers["x-object-meta-depend-ret-code"])
self.assertEqual(resp["status"], 200)
def get_SLO(self):
response = dict()
headers, body = client.get_object(self.url, self.token, self.container,
'assembly', http_conn=None,
resp_chunk_size=1024 * 1024,
query_string=None,
response_dict=response,
headers=None)
for (i, chunk) in enumerate(body):
self.assertEqual(chunk, self.chunks[i])
def invoke_storlet(url, token):
metadata = {'X-Run-Storlet': EXECDEP_STORLET_NAME }
resp = dict()
resp_headers, gf = c.get_object(url, token,
'myobjects',
EXECDEP_JUNK_FILE,
response_dict=resp,
headers=metadata)
assert 'x-object-meta-depend-ret-code' in resp_headers
assert resp_headers['x-object-meta-depend-ret-code'] == '42'
assert resp['status'] == 200
def create_thumbnail(request, account, original_container_name, container,
objectname):
""" Creates a thumbnail for an image. """
storage_url = request.session.get('storage_url', '')
auth_token = request.session.get('auth_token', '')
try:
client.head_container(storage_url, auth_token,
account)
except client.ClientException:
try:
client.put_container(
storage_url,
auth_token,
account)
except client.ClientException as e:
logger.error("Cannot put container %s. Error: %s "
% (container, str(e)))
return None
try:
headers, content = client.get_object(
storage_url,
auth_token,
container,
objectname)
im = Image.open(StringIO(content))
im.thumbnail(settings.THUMBNAIL_SIZE, Image.ANTIALIAS)
output = StringIO()
mimetype = headers['content-type'].split('/')[-1]
im.save(output, format=mimetype)
content = output.getvalue()
headers = {'X-Delete-After': settings.THUMBNAIL_DURABILITY}
try:
client.put_object(
storage_url,
auth_token,
account,
"%s_%s" % (original_container_name, objectname),
content,
headers=headers)
except client.ClientException as e:
logger.error("Cannot create thumbnail for image %s."
"Could not put thumbnail to storage: %s"
% (objectname, str(e)))
output.close()
except client.ClientException as e:
logger.error("Cannot create thumbnail for image %s."
"Could not retrieve the image from storage: %s"
% (objectname, str(e)))
except IOError as e:
logger.error("Cannot create thumbnail for image %s."
"An IOError occured: %s" % (objectname, e.strerror))
def invoke_identity_on_get_SLO(url, token):
metadata = {'X-Run-Storlet': IDENTITY_STORLET_NAME}
response = dict()
headers, body = c.get_object(url,
token,
'myobjects',
'assembly',
query_string=None,
response_dict=response,
resp_chunk_size=1048576,
headers=metadata)
compare_slo_to_chunks(body)
def invoke_storlet_on_get(self):
headers = {'X-Run-Storlet': self.storlet_name}
resp = dict()
resp_headers, gf = c.get_object(self.url, self.token,
'myobjects',
self.storlet_file,
response_dict=resp,
headers=headers)
with open('/tmp/sample.jpg', 'w') as f:
f.write(gf)
self.assertTrue(resp['status'] in [200, 202])
def test_execdep(self):
headers = {'X-Run-Storlet': self.storlet_name}
resp = dict()
resp_headers, gf = c.get_object(self.url, self.token,
'myobjects',
self.storlet_file,
response_dict=resp,
headers=headers)
self.assertTrue('x-object-meta-depend-ret-code' in resp_headers)
self.assertTrue(resp_headers['x-object-meta-depend-ret-code'] == '42')
self.assertEqual(resp['status'], 200)
def test_get(self):
resp = dict()
req_headers = {'X-Run-Storlet': self.storlet_name}
headers, content = client.get_object(self.url,
self.token,
self.container,
self.storlet_file,
response_dict=resp,
headers=req_headers)
self.assertEqual(200, resp['status'])
self.assertEqual('simple', headers['x-object-meta-test'])
self.assertEqual(self.content, content)
def _test_sync(self, object_post_as_copy):
source_container, dest_container = self._setup_synced_containers()
# upload to source
object_name = 'object-%s' % uuid.uuid4()
put_headers = {'X-Object-Meta-Test': 'put_value'}
client.put_object(self.url, self.token, source_container, object_name,
'test-body', headers=put_headers)
# cycle container-sync
Manager(['container-sync']).once()
resp_headers, body = client.get_object(self.url, self.token,
dest_container, object_name)
self.assertEqual(body, 'test-body')
self.assertIn('x-object-meta-test', resp_headers)
self.assertEqual('put_value', resp_headers['x-object-meta-test'])
# update metadata with a POST, using an internal client so we can
# vary the object_post_as_copy setting - first use post-as-copy
post_headers = {'Content-Type': 'image/jpeg',
'X-Object-Meta-Test': 'post_value'}
int_client = self.make_internal_client(
object_post_as_copy=object_post_as_copy)
int_client.set_object_metadata(self.account, source_container,
object_name, post_headers)
# sanity checks...
resp_headers = client.head_object(
self.url, self.token, source_container, object_name)
self.assertIn('x-object-meta-test', resp_headers)
self.assertEqual('post_value', resp_headers['x-object-meta-test'])
self.assertEqual('image/jpeg', resp_headers['content-type'])
# cycle container-sync
Manager(['container-sync']).once()
# verify that metadata changes were sync'd
resp_headers, body = client.get_object(self.url, self.token,
dest_container, object_name)
self.assertEqual(body, 'test-body')
self.assertIn('x-object-meta-test', resp_headers)
self.assertEqual('post_value', resp_headers['x-object-meta-test'])
self.assertEqual('image/jpeg', resp_headers['content-type'])
# delete the object
client.delete_object(
self.url, self.token, source_container, object_name)
with self.assertRaises(ClientException) as cm:
client.get_object(
self.url, self.token, source_container, object_name)
self.assertEqual(404, cm.exception.http_status) # sanity check
# cycle container-sync
Manager(['container-sync']).once()
# verify delete has been sync'd
with self.assertRaises(ClientException) as cm:
client.get_object(
self.url, self.token, dest_container, object_name)
self.assertEqual(404, cm.exception.http_status) # sanity check
