def test_all_nodes_fail(self):
# Create container1
container1 = 'container-%s' % uuid4()
cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
client.put_container(self.url, self.token, container1)
client.put_object(self.url, self.token, container1, 'obj1', 'data1')
# All primaries go down
for cnode in cnodes:
kill_server((cnode['ip'], cnode['port']), self.ipport2server)
# Can't GET the container
with self.assertRaises(client.ClientException) as caught:
client.get_container(self.url, self.token, container1)
self.assertEqual(caught.exception.http_status, 503)
# But we can still write objects! The old info is still in memcache
client.put_object(self.url, self.token, container1, 'obj2', 'data2')
# Can't POST the container, either
with self.assertRaises(client.ClientException) as caught:
client.post_container(self.url, self.token, container1, {})
self.assertEqual(caught.exception.http_status, 503)
# Though it *does* evict the cache
with self.assertRaises(client.ClientException) as caught:
client.put_object(self.url, self.token, container1, 'obj3', 'x')
self.assertEqual(caught.exception.http_status, 503)
def test_param_end_marker(self):
c.http_connection = self.fake_http_connection(
204, query_string="format=json&end_marker=end_marker")
c.get_container('http://www.test.com',
'asdf',
'asdf',
end_marker='end_marker')
def _get_total_objects(request, container, objectname):
""" Return the total number of objects under a pseudofolder. """
storage_url = request.session.get('storage_url', '')
auth_token = request.session.get('auth_token', '')
prefix = request.session.get('prefix')
if prefix:
prefix += objectname
else:
prefix = objectname
try:
if prefix:
meta, objects = client.get_container(
storage_url, auth_token, container,
prefix=prefix, full_listing=True,
headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')})
else:
meta, objects = client.get_container(
storage_url, auth_token, container,
headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')},
full_listing=True)
except client.ClientException:
return 0
pseudofolders, objs = pseudofolder_object_list(objects, prefix)
return len(objs)
def create_container(request):
""" Creates a container (empty object of type application/directory) """
storage_url = request.session.get("storage_url", "")
auth_token = request.session.get("auth_token", "")
headers = {
"X-Container-Meta-Access-Control-Expose-Headers": "Access-Control-Allow-Origin",
"X-Container-Meta-Access-Control-Allow-Origin": settings.BASE_URL,
}
form = CreateContainerForm(request.POST or None)
if form.is_valid():
container = form.cleaned_data["containername"]
# Check container does not already exist
try:
client.get_container(storage_url, auth_token, container)
messages.add_message(request, messages.ERROR, _("Container {0} already exists.".format(container)))
except:
try:
client.put_container(storage_url, auth_token, container, headers)
messages.add_message(request, messages.INFO, _("Container created."))
except client.ClientException:
messages.add_message(request, messages.ERROR, _("Access denied."))
return redirect(containerview)
return render_to_response(
"create_container.html", {"session": request.session}, context_instance=RequestContext(request)
)
def test_post_trumped_by_prior_delete(self):
# new metadata and content-type posted to subset of nodes should not
# cause object to persist after replication of an earlier delete on
# other nodes.
self.brain.put_container()
# incomplete put
self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(
headers={
'Content-Type': 'oldest',
'X-Object-Sysmeta-Test': 'oldest',
'X-Object-Meta-Test': 'oldest'
})
self.brain.start_primary_half()
self.container_brain.start_primary_half()
# incomplete put then delete
self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._put_object(
headers={
'Content-Type': 'oldest',
'X-Object-Sysmeta-Test': 'oldest',
'X-Object-Meta-Test': 'oldest'
})
self._delete_object()
self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# handoff post
self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._post_object(
headers={
'Content-Type': 'newest',
'X-Object-Sysmeta-Test': 'ignored',
'X-Object-Meta-Test': 'newest'
})
# check object metadata
metadata = self._get_object_metadata()
self.assertEqual(metadata['x-object-sysmeta-test'], 'oldest')
self.assertEqual(metadata['x-object-meta-test'], 'newest')
self.assertEqual(metadata['content-type'], 'newest')
self.brain.start_primary_half()
self.container_brain.start_primary_half()
# delete trumps later post
self.get_to_final_state()
# check object is now deleted
self.assertRaises(UnexpectedResponse, self._get_object_metadata)
container_metadata, objs = client.get_container(
self.url, self.token, self.container_name)
self.assertEqual(0, len(objs))
self._assert_consistent_container_dbs()
self._assert_consistent_deleted_object()
self._assert_consistent_suffix_hashes()
def public_objectview(request, account, container, prefix=None):
""" Returns list of all objects in current container. """
storage_url = settings.STORAGE_URL + account
auth_token = ' '
try:
_meta, objects = client.get_container(storage_url,
auth_token,
container,
delimiter='/',
prefix=prefix)
except client.ClientException:
messages.add_message(request, messages.ERROR, _("Access denied."))
return redirect(containerview)
prefixes = prefix_list(prefix)
pseudofolders, objs = pseudofolder_object_list(objects, prefix)
base_url = get_base_url(request)
account = storage_url.split('/')[-1]
return render_to_response("publicview.html", {
'container': container,
'objects': objs,
'folders': pseudofolders,
'prefix': prefix,
'prefixes': prefixes,
'base_url': base_url,
'storage_url': storage_url,
'account': account
},
context_instance=RequestContext(request))
def delete_given_folder(request, container, foldername):
""" Recursively delete all the objects in the given folder and then
delete the folder itself. """
storage_url = request.session.get('storage_url', '')
auth_token = request.session.get('auth_token', '')
# Get all objects within folder.
meta, objects = client.get_container(
storage_url, auth_token, container,
headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')},
delimiter='/', prefix=foldername)
# Recursive call to delete subfolders.
pseudofolders, objs = pseudofolder_object_list(objects, foldername)
for folder in pseudofolders:
delete_given_folder(request, container, folder[0])
# Delete all objects.
for obj in objs:
delete_given_object(request, container, obj["name"])
# Delete the folder itself.
try:
client.get_object(
storage_url, auth_token, container, foldername,
headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')})
delete_given_object(request, container, foldername)
except:
# If folder does not exist, pass
pass
def objectview(request, container, prefix=None):
""" Returns list of all objects in current container. """
storage_url = get_endpoint(request, 'adminURL')
public_url = get_endpoint(request, 'publicURL') + '/' + container
auth_token = get_token_id(request)
http_conn = client.http_connection(storage_url,
insecure=settings.SWIFT_INSECURE)
page = request.GET.get('page', 1)
try:
_, objects = client.get_container(storage_url, auth_token,
container, delimiter='/',
prefix=prefix,
http_conn=http_conn)
except client.ClientException as err:
log.exception('{}{}'.format(ugettext('Exception:').encode('UTF-8'), err))
messages.add_message(request, messages.ERROR, ugettext('Access denied.'))
return redirect(containerview)
prefixes = prefix_list(prefix)
object_list = pseudofolder_object_list(objects, prefix, public_url)
context = utils.update_default_context(request, {
'container': container,
'objects': utils.generic_pagination(object_list, page),
'prefix': prefix,
'prefixes': prefixes,
})
return render_to_response("objectview.html", context,
context_instance=RequestContext(request))
def delete_container(request, container, force=True):
"""
Deletes a container. If force is True, it will deletes all objects first.
"""
storage_url = get_endpoint(request, 'adminURL')
auth_token = get_token_id(request)
http_conn = client.http_connection(storage_url,
insecure=settings.SWIFT_INSECURE)
if force:
try:
_, objects = client.get_container(storage_url,
auth_token,
container,
http_conn=http_conn)
except client.ClientException as err:
log.exception('{}{}'.format(ugettext('Exception:').encode('UTF-8'), err))
return False
for obj in objects:
delete_object(request=request,
container=container,
objectname=obj['name'])
try:
client.delete_container(storage_url, auth_token,
container, http_conn=http_conn)
actionlog.log(request.user.username, "delete", container)
except client.ClientException as err:
log.exception('{}{}'.format(ugettext('Exception:').encode('UTF-8'), err))
return False
return True
def subdir_proc(storage_url, token, container, subdir=None, nest=0):
#print "DEBUG %s" % subdir
(c_headers, objects) = get_container(storage_url, token,
container,
prefix=subdir,
delimiter='/')
spacer = ' ' * ((nest + 1) * 2)
for obj in objects:
if 'subdir' in obj:
sdir_name = obj['subdir']
l = len(sdir_name)
idx = sdir_name.rfind('/', 0, l - 1)
if idx >= 0:
sdir_name = sdir_name[idx + 1:]
print "%-30s" % (spacer + sdir_name)
if subdir is None:
new_subdir = obj['subdir']
else:
new_subdir = obj['subdir']
subdir_proc(storage_url, token, container, new_subdir, nest + 1)
else:
if subdir is None:
l = 0
else:
l = len(subdir)
objname = obj['name'][l:]
print "%-30s %20s bytes" % (spacer + objname, obj['bytes'])
def getMemberNames(self):
_stat, objects = client.get_container(self.storage_url,
self.auth_token,
container=self.container,
delimiter='/',
prefix=self.prefix,
http_conn=self.http_connection)
self.objects = {}
childs = []
for obj in objects:
name = obj.get('name')
if name and name != self.prefix:
name = name.encode("utf8")
childs.append(name)
self.objects[name] = obj
subdir = obj.get('subdir')
if subdir and subdir != self.prefix:
subdir = subdir.rstrip('/')
subdir = subdir.encode("utf8")
# there might be two entries:
# 1. object with type application/directory and no trailing '/'
# 2. subdir entry with trailing '/'
if subdir not in childs:
childs.append(subdir)
self.objects[subdir] = obj
return childs
def delete_container(request):
auth_token = request.session['auth_token']
storage_url = request.session['storage_url']
container = ''
if 'container' in request.GET.keys():
container = request.GET['container']
else:
return redirect(containers)
try:
http_conn = (urlparse(storage_url),
client.HTTPConnection(storage_url, insecure=settings.SWIFT_SSL_INSECURE))
_m, objects = client.get_container(storage_url, auth_token, container, http_conn=http_conn)
for obj in objects:
logger.info("Deleting object %s in container %s" % (obj['name'], container))
client.delete_object(storage_url, auth_token,
container, obj['name'], http_conn=http_conn)
logger.info("Deleting container %s" % (container))
client.delete_container(storage_url, auth_token, container, http_conn=http_conn)
messages.add_message(request, messages.INFO, "Container deleted.")
except client.ClientException:
messages.add_message(request, messages.ERROR, "Access denied.")
return redirect(containers)
def list(self):
swift_objects = []
for cont in self.list_containers():
objs = [{'container': cont, 'name': obj['name']} for obj in
swift_client.get_container(self.endpoint, self.token, cont)[1]]
swift_objects.extend(objs)
return swift_objects
def get_object_table(request):
""" Returns json object of all objects in current container. """
storage_url = request.session.get('storage_url', '')
auth_token = request.session.get('auth_token', '')
container = request.session.get('container')
prefix = request.session.get('prefix')
try:
meta, objects = client.get_container(
storage_url, auth_token, container, delimiter='/', prefix=prefix,
headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')})
except client.ClientException:
messages.add_message(request, messages.ERROR, _("Access denied."))
return redirect(containerview)
pseudofolders, objs = pseudofolder_object_list(objects, prefix)
# Check for incomplete SLO uploads. The object table will display them
# allowing the user to continue uploading or delete them.
incomplete_slo = check_incomplete_slo(
request, storage_url, auth_token, container, prefix)
return JsonResponse({
'success': True,
"data": {
'container': container,
'objects': objs,
'folders': pseudofolders,
'folder_prefix': prefix,
'incomplete_slo': incomplete_slo
}
})
def public_objectview(request, account, container, prefix=None):
""" Returns list of all objects in current container. """
storage_url = settings.STORAGE_URL + account
auth_token = ' '
try:
_meta, objects = client.get_container(
storage_url, auth_token, container, delimiter='/', prefix=prefix)
except client.ClientException:
traceback.print_exc()
messages.add_message(request, messages.ERROR, _("Access denied."))
return redirect(containerview)
prefixes = prefix_list(prefix)
pseudofolders, objs = pseudofolder_object_list(objects, prefix)
base_url = get_base_url(request)
account = storage_url.split('/')[-1]
return render_to_response("publicview.html", {
'container': container,
'objects': objs,
'folders': pseudofolders,
'prefix': prefix,
'prefixes': prefixes,
'base_url': base_url,
'storage_url': storage_url,
'account': account},
context_instance=RequestContext(request))
def test_async_updates_after_PUT_and_POST(self):
# verify correct update values when PUT update and POST updates are
# missed but then async updates are sent
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url,
self.token,
'c1',
headers={'X-Storage-Policy': self.policy.name})
# PUT and POST to object while one container server is stopped so that
# we force async updates to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url,
self.token,
'c1',
'o1',
contents=content,
content_type='test/ctype')
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# use internal client for POST so we can force fast-post mode
int_client = self.make_internal_client(object_post_as_copy=False)
int_client.set_object_metadata(self.account, 'c1', 'o1',
{'X-Object-Meta-Fruit': 'Tomato'})
self.assertEqual('Tomato',
int_client.get_object_metadata(
self.account, 'c1',
'o1')['x-object-meta-fruit']) # sanity
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(
direct_client.direct_get_container(cnodes[0], cpart, self.account,
'c1')[1])
# Run the object-updaters to send the async pendings
Manager(['object-updater']).once()
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(
cnode, cpart, self.account, 'c1')[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual('test/ctype', listing[0]['content_type'])
listing_etags.add(listing[0]['hash'])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, 'c1')
self.assertEqual(1, len(listing))
self.assertEqual('o1', listing[0]['name'])
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual(meta['etag'], listing[0]['hash'])
self.assertEqual('test/ctype', listing[0]['content_type'])
def public_objectview(request, account, container, prefix=None):
""" Returns list of all objects in current container. """
storage_url = settings.STORAGE_URL + account
auth_token = " "
try:
_meta, objects = client.get_container(storage_url, auth_token, container, delimiter="/", prefix=prefix)
except client.ClientException:
messages.add_message(request, messages.ERROR, _("Access denied."))
return redirect(containerview)
prefixes = prefix_list(prefix)
pseudofolders, objs = pseudofolder_object_list(objects, prefix)
base_url = get_base_url(request)
account = storage_url.split("/")[-1]
return render_to_response(
"publicview.html",
{
"container": container,
"objects": objs,
"folders": pseudofolders,
"prefix": prefix,
"prefixes": prefixes,
"base_url": base_url,
"storage_url": storage_url,
"account": account,
},
context_instance=RequestContext(request),
)
def getMemberNames(self):
_stat, objects = client.get_container(self.storage_url,
self.auth_token,
container=self.container,
delimiter='/',
prefix=self.prefix,
http_conn=self.http_connection)
self.objects = {}
childs = []
for obj in objects:
name = obj.get('name')
if name and name != self.prefix:
name = name.encode("utf8")
childs.append(name)
self.objects[name] = obj
subdir = obj.get('subdir')
if subdir and subdir != self.prefix:
subdir = subdir.rstrip('/')
subdir = subdir.encode("utf8")
# there might be two entries:
# 1. object with type application/directory and no trailing '/'
# 2. subdir entry with trailing '/'
if subdir not in childs:
childs.append(subdir)
self.objects[subdir] = obj
return childs
def get_all(request, container):
""" Returns List of all Objects present in specified container
:param container: Name of Swift Container
:return: meta data and id's of all the objects
"""
try:
response_dict = dict()
storage_url, auth_token = _get_auth_data(request.session)
data_container = client.get_container(storage_url, auth_token, container)
for obj in data_container[1]:
meta_object = client.head_object(storage_url, auth_token, container, obj['name'])
if not meta_object['x-object-meta-deleted']:
if meta_object['x-object-meta-type'] in ['thumbnail', 'original-thumbnail']:
if meta_object['x-object-meta-format'] not in response_dict:
form = meta_object['x-object-meta-format']
response_dict[form] = []
if meta_object['x-object-meta-format'] in response_dict:
new_obj = {'thumbnail_id': obj['name'],
'name': meta_object['x-object-meta-name'],
'type': meta_object['x-object-meta-type'],
'resolution': meta_object['x-object-meta-resolution']}
if meta_object['x-object-meta-format'] is 'thumbnail':
new_obj['original_id'] = meta_object['x-object-meta-original']
response_dict[form].append(new_obj)
return Response(response_dict)
except swift_exception.ClientException as e:
print e
return Response('Please contact your admininstrator', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
except Exception as e:
print e
return Response('Please contact your admininstrator', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
def list_container(self,container_name,prefix=None,delimiter=None):
try:
return client.get_container(self.storage_url,self.token,container_name,prefix=prefix,delimiter=delimiter,http_conn=self.http_conn)
except:
return None
def test_create_swift_client(self):
mock_resp = mock(dict)
when(swiftclient.client.Connection).get_container("bob").thenReturn(["text", mock_resp])
client = remote.create_swift_client(TroveContext(tenant="123"))
headers, container = client.get_container("bob")
self.assertIs(headers, "text")
self.assertIs(container, mock_resp)
def get_container_detail(self, account_id, token, s_type, container):
cnx = self.get_cnx(account_id, s_type)
return sclient.get_container(None,
token,
container,
http_conn=cnx,
full_listing=True)
def get_segment_number(file_name, request, container, prefix=None):
'''Return the segment number a given file should create next. If it is 0,
create a pseudo folder for the file. The folder is created if it doesn't
exist. '''
if prefix:
foldername = prefix + '/' + file_name + '_segments'
else:
foldername = file_name + '_segments'
storage_url = request.session.get('storage_url', '')
auth_token = request.session.get('auth_token', '')
foldername = os.path.normpath(foldername)
foldername = foldername.strip('/')
foldername += '/'
content_type = 'application/directory'
obj = None
client.put_object(storage_url, auth_token, container, foldername, obj,
content_type=content_type)
meta, objects = client.get_container(storage_url, auth_token, container,
delimiter='/', prefix=foldername)
pseudofolders, objs = pseudofolder_object_list(objects, prefix)
return get_first_nonconsecutive(objs)
def get_object_table(request):
""" Returns json object of all objects in current container. """
storage_url = request.session.get("storage_url", "")
auth_token = request.session.get("auth_token", "")
container = request.session.get("container")
prefix = request.session.get("prefix")
try:
meta, objects = client.get_container(storage_url, auth_token, container, delimiter="/", prefix=prefix)
except client.ClientException:
messages.add_message(request, messages.ERROR, _("Access denied."))
return redirect(containerview)
prefixes = prefix_list(prefix)
pseudofolders, objs = pseudofolder_object_list(objects, prefix)
base_url = get_base_url(request)
account = storage_url.split("/")[-1]
return JsonResponse(
{
"success": True,
"data": {"container": container, "objects": objs, "folders": pseudofolders, "folder_prefix": prefix},
}
)
def delete_data(self, context, container_name):
url = self._get_endpoint_url(context)
token = context.auth_token
try:
headers, info = swift_client.get_container(
url, token, container_name
)
for obj in info:
try:
swift_client.delete_object(
url, token, container_name, obj['name']
)
except swift_exceptions.ClientException as e:
LOG.error(e)
if e.http_status != 404:
raise
swift_client.delete_container(url, token, container_name)
except swift_exceptions.ClientException as e:
LOG.error(e)
if e.http_status == 404:
raise exception.ContainerNotExists(
container_name=container_name
)
else:
raise exception.ContainerDeletionError(
container_name=container_name
)
def delete_given_folder(request, container, foldername):
storage_url = request.session.get('storage_url', '')
auth_token = request.session.get('auth_token', '')
# Get all objects within folder.
meta, objects = client.get_container(
storage_url, auth_token, container, delimiter='/', prefix=foldername)
# Recursive call to delete subfolders.
pseudofolders, objs = pseudofolder_object_list(objects, foldername)
for folder in pseudofolders:
delete_given_folder(request, container, folder[0])
# Delete all objects.
for obj in objs:
delete_given_object(request, container, obj["name"])
# Delete the folder itself.
try:
delete_given_object(request, container, foldername)
except:
#Except a failure to delete if the pseudo folder was not created
#manually.
pass
def get_temp_key(storage_url, auth_token, container):
""" Tries to get meta-temp-url key from account.
If not set, generate tempurl and save it to acocunt.
This requires at least account owner rights. """
try:
account = client.get_account(storage_url, auth_token)
except client.ClientException:
#Try to get the container temp url key instead
try:
container = client.get_container(
storage_url, auth_token, container)
return container[0].get('x-container-meta-temp-url-key')
except client.ClientException:
return None
return None
key = account[0].get('x-account-meta-temp-url-key')
if not key:
chars = string.ascii_lowercase + string.digits
key = ''.join(random.choice(chars) for x in range(32))
headers = {'x-account-meta-temp-url-key': key}
try:
client.post_account(storage_url, auth_token, headers)
except client.ClientException:
return None
return key
def object_versioning(request, container, prefix=None):
storage_url = get_endpoint(request, 'adminURL')
auth_token = get_token_id(request)
public_url = get_endpoint(request, 'publicURL') + '/' + container
http_conn = client.http_connection(storage_url,
insecure=settings.SWIFT_INSECURE)
objects = []
page = request.GET.get('page', 1)
if request.method == 'GET':
headers = client.head_container(storage_url,
auth_token,
container,
http_conn=http_conn)
version_location = headers.get('x-versions-location', None)
if version_location:
try:
_, objects = client.get_container(storage_url,
auth_token,
version_location,
prefix=prefix,
delimiter='/',
http_conn=http_conn)
except client.ClientException:
pass
prefixes = prefix_list(prefix)
object_list = pseudofolder_object_list(objects, prefix, public_url)
context = utils.update_default_context(request, {
'container': container,
'objects': utils.generic_pagination(object_list, page),
'version_location': version_location,
'prefix': prefix,
'prefixes': prefixes,
})
return render_to_response('container_versioning.html',
dictionary=context,
context_instance=RequestContext(request))
if request.method == 'POST':
action = request.POST.get('action', None)
if action == 'enable':
enable_versioning(request, container)
actionlog.log(request.user.username, "enable", 'Versioning. Container: %s' % container)
elif action == 'disable':
disable_versioning(request, container)
actionlog.log(request.user.username, "disable", 'Versioning. Container: %s' % container)
else:
messages.add_message(request, messages.ERROR, ugettext('Action is required.'))
return redirect(object_versioning, container=container)
def test_create_swift_client(self):
mock_resp = MagicMock()
swiftclient.client.Connection.get_container = MagicMock(
return_value=["text", mock_resp])
client = remote.create_swift_client(TroveContext(tenant='123'))
headers, container = client.get_container('bob')
self.assertIs(headers, "text")
self.assertIs(container, mock_resp)
def find_container(self, name):
if not self.http_conn:
self.connect()
retval = swift.get_container(url=self.swift_url, token=self.token, http_conn=self.http_conn, container=name)
if self.debug:
print(retval)
return retval
def test_create_swift_client(self):
mock_resp = MagicMock()
swiftclient.client.Connection.get_container = MagicMock(
return_value=["text", mock_resp])
client = remote.create_swift_client(TroveContext(tenant='123'))
headers, container = client.get_container('bob')
self.assertIs(headers, "text")
self.assertIs(container, mock_resp)
def test_first_two_nodes_fail(self):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
object1 = 'object1'
client.put_object(self.url, self.token, container, object1, 'test')
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
cpart, cnodes = self.container_ring.get_nodes(self.account, container)
for x in xrange(2):
kill(self.pids[self.port2server[cnodes[x]['port']]], SIGTERM)
client.delete_object(self.url, self.token, container, object1)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
self.assert_(object1 not in [o['name'] for o in
client.get_container(self.url, self.token, container)[1]])
for x in xrange(2):
self.pids[self.port2server[cnodes[x]['port']]] = \
Popen(['swift-container-server',
'/etc/swift/container-server/%d.conf' %
((cnodes[x]['port'] - 6001) / 10)]).pid
sleep(2)
self.assert_(container in [c['name'] for c in
client.get_account(self.url, self.token)[1]])
# This okay because the first node hasn't got the update that the
# object was deleted yet.
self.assert_(object1 in [o['name'] for o in
direct_client.direct_get_container(cnodes[0], cpart,
self.account, container)[1]])
# This fails because all three nodes have to indicate deletion before
# we tell the user it worked. Since the first node 409s (it hasn't got
# the update that the object was deleted yet), the whole must 503
# (until every is synced up, then the delete would work).
exc = None
try:
client.delete_container(self.url, self.token, container)
except client.ClientException, err:
exc = err
def test_create_swift_client(self):
mock_resp = mock(dict)
when(swiftclient.client.Connection).get_container('bob').thenReturn(
["text", mock_resp])
client = remote.create_swift_client(TroveContext(tenant='123'))
headers, container = client.get_container('bob')
self.assertIs(headers, "text")
self.assertIs(container, mock_resp)
def clean_container(self, container_name):
_, objects = client.get_container(
self.url, self.token, container_name,
full_listing=True)
# delete all objects inside the container
# N.B. this cleanup could run in parallel but currently we have a few
# objects in the user testing container so that, currently this does
# as sequential simply
for obj_dict in objects:
client.delete_object(
self.url, self.token,
container_name, obj_dict['name'])
client.get_container(self.url, self.token, container_name)
# delete the container
client.delete_container(self.url, self.token, container_name)
def test_create_swift_client(self):
mock_resp = mock(dict)
when(swiftclient.client.Connection).get_container('bob').thenReturn(
["text", mock_resp])
client = remote.create_swift_client(ReddwarfContext(tenant='123'))
headers, container = client.get_container('bob')
self.assertIs(headers, "text")
self.assertIs(container, mock_resp)
def download_collection(request, container, prefix=None, non_recursive=False):
""" Download the content of an entire container/pseudofolder
as a Zip file. """
storage_url = request.session.get('storage_url', '')
auth_token = request.session.get('auth_token', '')
delimiter = '/' if non_recursive else None
try:
x, objects = client.get_container(
storage_url,
auth_token,
container,
delimiter=delimiter,
prefix=prefix
)
except client.ClientException:
return HttpResponseForbidden()
x, objs = pseudofolder_object_list(objects, prefix)
# Do not provide download when the folder is empty.
if len(x) + len(objs) == 0:
messages.add_message(
request, messages.ERROR, _("Unable to download, no files found."))
if prefix: # Return user to object view
# remove the last prefix. ex "dir1/dir2/" -> "dir1"
prefix = prefix[0:prefix.rfind('/', 0, prefix.rfind('/'))]
return redirect(swiftbrowser.views.objectview, container=container,
prefix=prefix)
else: # Return user to the container view
return redirect(swiftbrowser.views.containerview)
output = StringIO()
zipf = zipfile.ZipFile(output, 'w')
for o in objs:
name = o['name']
try:
x, content = client.get_object(storage_url, auth_token, container,
name)
except client.ClientException:
return HttpResponseForbidden()
if prefix:
name = name[len(prefix):]
zipf.writestr(name, content)
zipf.close()
if prefix:
filename = prefix.split('/')[-2]
else:
filename = container
response = HttpResponse(output.getvalue(), 'application/zip')
response['Content-Disposition'] = 'attachment; filename="%s.zip"'\
% (filename)
output.close()
return response
def delete_pseudofolder(request, container, pseudofolder):
""" Deletes an empty object, used as a pseudofolder """
storage_url = get_endpoint(request, 'adminURL')
auth_token = get_token_id(request)
http_conn = client.http_connection(storage_url,
insecure=settings.SWIFT_INSECURE)
objects = client.get_container(storage_url,
auth_token,
container,
prefix=pseudofolder,
http_conn=http_conn)[1]
count_deletes = 0
for obj in objects:
try:
client.delete_object(storage_url,
token=auth_token,
container=container,
name=obj['name'],
http_conn=http_conn)
count_deletes += 1
except client.ClientException as err:
log.exception('{}{}'.format(_('Exception:').encode('UTF-8'), err))
# Empty pseudofolder
if count_deletes == 1 and count_deletes == len(objects):
messages.add_message(request, messages.SUCCESS,
_('Pseudofolder deleted.'))
# Non empty pseudofolder
elif count_deletes > 1 and count_deletes == len(objects):
messages.add_message(
request, messages.SUCCESS,
_('Pseudofolder and {0} objects deleted.').format(count_deletes -
1))
elif count_deletes > 0 and count_deletes < len(objects):
messages.add_message(request, messages.SUCCESS,
_('It was not possible to delete all objects.'))
else:
messages.add_message(request, messages.ERROR,
_('Fail to delete pseudofolder.'))
if pseudofolder[-1] == '/': # deleting a pseudofolder, move one level up
pseudofolder = pseudofolder[:-1]
prefix = '/'.join(pseudofolder.split('/')[:-1])
if prefix:
prefix += '/'
actionlog.log(request.user.username, "delete", pseudofolder)
if prefix:
return redirect(objectview, container=container, prefix=prefix)
else:
return redirect(objectview, container=container)
def test_put_ctype_replicated_when_subsequent_post(self):
# primary half handoff half
# ------------ ------------
# t0.data: ctype = foo
# t1.data: ctype = bar
# t2.meta:
#
# ...run replicator and expect...
#
# t1.data: ctype = bar
# t2.meta:
self.brain.put_container(policy_index=0)
# incomplete write
self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._put_object(headers={'Content-Type': 'foo'})
self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# handoff write
self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(headers={'Content-Type': 'bar'})
self.brain.start_primary_half()
self.container_brain.start_primary_half()
# metadata update with newest data unavailable
self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._post_object(headers={'X-Object-Meta-Color': 'Blue'})
self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
self.get_to_final_state()
# check object metadata
metadata = client.head_object(self.url, self.token,
self.container_name,
self.object_name)
# check container listing metadata
container_metadata, objs = client.get_container(self.url, self.token,
self.container_name)
for obj in objs:
if obj['name'] == self.object_name:
break
else:
self.fail('obj not found in container listing')
expected = 'bar'
self.assertEqual(obj['content_type'], expected)
self.assertEqual(metadata['x-object-meta-color'], 'Blue')
self._assert_object_metadata_matches_listing(obj, metadata)
self._assert_consistent_container_dbs()
self._assert_consistent_object_metadata()
self._assert_consistent_suffix_hashes()
def test_put_ctype_replicated_when_subsequent_post(self):
# primary half handoff half
# ------------ ------------
# t0.data: ctype = foo
# t1.data: ctype = bar
# t2.meta:
#
# ...run replicator and expect...
#
# t1.data: ctype = bar
# t2.meta:
self.brain.put_container()
# incomplete write
self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._put_object(headers={'Content-Type': 'foo'})
self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# handoff write
self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(headers={'Content-Type': 'bar'})
self.brain.start_primary_half()
self.container_brain.start_primary_half()
# metadata update with newest data unavailable
self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._post_object(headers={'X-Object-Meta-Color': 'Blue'})
self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
self.get_to_final_state()
# check object metadata
metadata = client.head_object(self.url, self.token,
self.container_name,
self.object_name)
# check container listing metadata
container_metadata, objs = client.get_container(self.url, self.token,
self.container_name)
for obj in objs:
if obj['name'] == self.object_name:
break
else:
self.fail('obj not found in container listing')
expected = 'bar'
self.assertEqual(obj['content_type'], expected)
self.assertEqual(metadata['x-object-meta-color'], 'Blue')
self._assert_object_metadata_matches_listing(obj, metadata)
self._assert_consistent_container_dbs()
self._assert_consistent_object_metadata()
self._assert_consistent_suffix_hashes()
def test_update_during_POST_only(self):
# verify correct update values when PUT update is missed but then a
# POST update succeeds *before* the PUT async pending update is sent
cpart, cnodes = self.container_ring.get_nodes(self.account, "c1")
client.put_container(self.url, self.token, "c1", headers={"X-Storage-Policy": self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
content = u"stuff"
client.put_object(self.url, self.token, "c1", "o1", contents=content, content_type="test/ctype")
meta = client.head_object(self.url, self.token, "c1", "o1")
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
self.assertFalse(direct_client.direct_get_container(cnodes[0], cpart, self.account, "c1")[1])
# use internal client for POST so we can force fast-post mode
int_client = self.make_internal_client(object_post_as_copy=False)
int_client.set_object_metadata(self.account, "c1", "o1", {"X-Object-Meta-Fruit": "Tomato"})
self.assertEqual(
"Tomato", int_client.get_object_metadata(self.account, "c1", "o1")["x-object-meta-fruit"]
) # sanity
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]["bytes"])
self.assertEqual("test/ctype", listing[0]["content_type"])
listing_etags.add(listing[0]["hash"])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, "c1")
self.assertEqual(1, len(listing))
self.assertEqual("o1", listing[0]["name"])
self.assertEqual(len(content), listing[0]["bytes"])
self.assertEqual(meta["etag"], listing[0]["hash"])
self.assertEqual("test/ctype", listing[0]["content_type"])
# Run the object-updaters to send the async pending from the PUT
Manager(["object-updater"]).once()
# check container listing metadata is still correct
for cnode in cnodes:
listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]["bytes"])
self.assertEqual("test/ctype", listing[0]["content_type"])
listing_etags.add(listing[0]["hash"])
self.assertEqual(1, len(listing_etags))
def create_container(request):
""" Creates a container on swift. """
storage_url = request.session.get('storage_url', '')
auth_token = request.session.get('auth_token', '')
# We need the CORS header set to the base url of this instance of
# swiftbrowser. Otherwise posts from the browser to swift will not work.
headers = {
'X-Container-Meta-Access-Control-Expose-Headers':
'Access-Control-Allow-Origin',
'X-Container-Meta-Access-Control-Allow-Origin': settings.BASE_URL
}
form = CreateContainerForm(request.POST or None)
if form.is_valid():
container = form.cleaned_data['containername']
#Check container does not already exist
try:
client.get_container(
storage_url, auth_token, container,
headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')})
messages.add_message(
request,
messages.ERROR,
_("Container {0} already exists.".format(container)))
except:
try:
client.put_container(
storage_url, auth_token, container, headers)
messages.add_message(request, messages.INFO,
_("Container created."))
except client.ClientException:
messages.add_message(
request, messages.ERROR, _("Access denied."))
return redirect(containerview)
return render_to_response(
'create_container.html',
{'session': request.session},
context_instance=RequestContext(request))
def is_subdir(self, name):
"""Checks if given name is a subdir.
This is a workaround for Swift (and other object storages).
There are several possibilites for a given URL in the form /ABC/XYZ.
1. /ABC/XYZ is the full path to an object. In this case there are
more possibilites, for example get the object, check if it exists,
put a new object under this name.
2. /ABC/XYZ should list the contents of container ABC with a prefix XYZ
The latter one will return en empty result set, thus this will be used
to differentiate between the possibilites.
"""
name = name.strip('/')
obj = self.objects.get(name, self.objects.get(name + '/'))
if not obj:
_, objects = client.get_container(self.storage_url,
self.auth_token,
container=self.container,
http_conn=self.http_connection)
for obj in objects:
objname = obj.get('name')
self.objects[objname] = obj
_, objects = client.get_container(self.storage_url,
self.auth_token,
container=self.container,
delimiter='/',
prefix=name,
http_conn=self.http_connection)
for obj in objects:
objname = obj.get('name', obj.get('subdir'))
self.objects[objname] = obj
obj = self.objects.get(name, self.objects.get(name + '/', {}))
if obj.get('subdir') or \
obj.get('content_type') == 'application/directory':
return True
return False
def list_container(self, container_name, prefix=None, delimiter=None):
try:
return client.get_container(self.storage_url,
self.token,
container_name,
prefix=prefix,
delimiter=delimiter,
http_conn=self.http_conn)
except:
return None
def is_subdir(self, name):
"""Checks if given name is a subdir.
This is a workaround for Swift (and other object storages).
There are several possibilites for a given URL in the form /ABC/XYZ.
1. /ABC/XYZ is the full path to an object. In this case there are
more possibilites, for example get the object, check if it exists,
put a new object under this name.
2. /ABC/XYZ should list the contents of container ABC with a prefix XYZ
The latter one will return en empty result set, thus this will be used
to differentiate between the possibilites.
"""
name = name.strip('/')
name = name.replace(self.container + '/', '')
obj = self.objects.get(name, self.objects.get(name + '/'))
if not obj:
_, objects = client.get_container(self.storage_url,
self.auth_token,
container=self.container,
http_conn=self.http_connection)
for obj in objects:
objname = obj.get('name')
self.objects[objname] = obj
_, objects = client.get_container(self.storage_url,
self.auth_token,
container=self.container,
delimiter='/',
prefix=name,
http_conn=self.http_connection)
for obj in objects:
objname = obj.get('name', obj.get('subdir'))
self.objects[objname] = obj
obj = self.objects.get(name, self.objects.get(name + '/', {}))
if obj.get('subdir') or \
obj.get('content_type') == 'application/directory':
return True
return False
def list(self):
swift_objects = []
for cont in self.list_containers():
objs = [{'container': cont, 'name': obj['name']} for obj in
swift_client.get_container(self.endpoint,
self.token,
cont,
http_conn=self.http_conn)[1]]
swift_objects.extend(objs)
return swift_objects
def test_skip_sync_when_misconfigured(self):
source_container, dest_container = self._setup_synced_containers()
container_name = 'versioned-%s' % uuid.uuid4()
version_hdr = {'X-Versions-Enabled': 'true'}
client.put_container(self.url, self.token, container_name,
headers=version_hdr)
# some sanity checks
object_name = 'object-%s' % uuid.uuid4()
client.put_object(self.url, self.token, container_name, object_name,
'version1')
client.put_object(self.url, self.token, container_name, object_name,
'version2')
resp_headers, listing = client.get_container(
self.url, self.token, container_name,
query_string='versions')
self.assertEqual(2, len(listing))
sync_headers = {}
sync_to = '//%s/%s/%s/%s' % (self.realm, self.cluster, self.account,
dest_container)
sync_headers['X-Container-Sync-To'] = sync_to
sync_headers['X-Container-Sync-Key'] = 'secret'
# use internal client to set container-sync headers
# since it doesn't have container_sync middleware in pipeline
# allowing us to bypass checks
int_client = self.make_internal_client()
# TODO: what a terrible hack, maybe we need to extend internal
# client to allow caller to become a swift_owner??
int_client.app.app.app.app.swift_owner_headers = []
int_client.set_container_metadata(self.account, container_name,
metadata=sync_headers)
headers = client.head_container(self.url, self.token,
container_name)
# This should never happen, but if it does because of eventual
# consistency or a messed up pipeline, container-sync should
# skip syncing container.
self.assertEqual('True', headers.get('x-versions-enabled'))
self.assertEqual('secret', headers.get('x-container-sync-key'))
self.assertEqual(sync_to, headers.get('x-container-sync-to'))
# cycle container-sync
Manager(['container-sync']).once()
with self.assertRaises(ClientException) as cm:
client.get_object(
self.url, self.token, dest_container, object_name)
self.assertEqual(404, cm.exception.http_status) # sanity check
def moveRecursive(self, newname):
""" Move/rename a container.
This is only working for empty containers and objects with a size of 0.
Required by some clients because they create a folder "New folder" first.
For all other requests this will simply return HTTP_FORBIDDEN. """
oldname = self.path.lstrip('/')
newname = newname.lstrip('/')
if '/' not in oldname:
try:
# Container deletion will fail if not empty
client.delete_container(self.storage_url,
self.auth_token,
oldname)
client.put_container(self.storage_url,
self.auth_token,
newname)
except client.ClientException:
raise dav_error.DAVError(dav_error.HTTP_FORBIDDEN)
else:
old_container, _, old_object = oldname.partition('/')
new_container, _, new_object = newname.partition('/')
if old_container != new_container:
raise dav_error.DAVError(dav_error.HTTP_FORBIDDEN)
# If it is a pseudofolder, check that it is empty
_, objects = client.get_container(
self.storage_url,
self.auth_token,
container=self.container,
delimiter='/',
prefix=sanitize(old_object).rstrip('/') + '/',
http_conn=self.http_connection)
if len(objects) != 1: # first object is the pseudofolder entry
raise dav_error.DAVError(dav_error.HTTP_FORBIDDEN)
# also check if the object is empty
if objects[0].get('bytes') != 0:
raise dav_error.DAVError(dav_error.HTTP_FORBIDDEN)
# Do a COPY to preserve existing metadata and content-type
client.put_object(self.storage_url,
self.auth_token,
self.container,
sanitize(new_object),
headers = {'X-Copy-From': '/' + oldname + '/'},
http_conn=self.http_connection)
client.delete_object(self.storage_url,
self.auth_token,
self.container,
old_object + '/',
http_conn=self.http_connection)
def test_post_ctype_replicated_when_previous_incomplete_puts(self):
# primary half handoff half
# ------------ ------------
# t0.data: ctype = foo
# t1.data: ctype = bar
# t2.meta: ctype = baz
#
# ...run replicator and expect...
#
# t1.data:
# t2.meta: ctype = baz
self.brain.put_container(policy_index=0)
# incomplete write to primary half
self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._put_object(headers={'Content-Type': 'foo'})
self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
# handoff write
self.brain.stop_primary_half()
self.container_brain.stop_primary_half()
self._put_object(headers={'Content-Type': 'bar'})
self.brain.start_primary_half()
self.container_brain.start_primary_half()
# content-type update to primary half
self.brain.stop_handoff_half()
self.container_brain.stop_handoff_half()
self._post_object(headers={'Content-Type': 'baz'})
self.brain.start_handoff_half()
self.container_brain.start_handoff_half()
self.get_to_final_state()
# check object metadata
metadata = client.head_object(self.url, self.token,
self.container_name,
self.object_name)
# check container listing metadata
container_metadata, objs = client.get_container(self.url, self.token,
self.container_name)
for obj in objs:
if obj['name'] == self.object_name:
break
expected = 'baz'
self.assertEqual(obj['content_type'], expected)
self._assert_object_metadata_matches_listing(obj, metadata)
self._assert_consistent_container_dbs()
self._assert_consistent_object_metadata()
self._assert_consistent_suffix_hashes()
def run_test(num_locks, catch_503):
container = 'container-%s' % uuid4()
client.put_container(self.url, self.token, container)
# Get the container info into memcache (so no stray
# get_container_info calls muck up our timings)
client.get_container(self.url, self.token, container)
db_files = self.get_container_db_files(container)
db_conns = []
for i in range(num_locks):
db_conn = connect(db_files[i])
db_conn.execute('begin exclusive transaction')
db_conns.append(db_conn)
if catch_503:
try:
client.delete_container(self.url, self.token, container)
except client.ClientException as err:
self.assertEqual(err.http_status, 503)
else:
self.fail("Expected ClientException but didn't get it")
else:
client.delete_container(self.url, self.token, container)
def test_create_swift_client(self):
mock_resp = MagicMock()
with patch.object(swiftclient.client.Connection, 'get_container',
MagicMock(return_value=["text", mock_resp])):
service_catalog = [{'endpoints': [{'region': 'RegionOne',
'publicURL': 'example.com'}],
'type': 'object-store'}]
client = remote.create_swift_client(TroveContext(
tenant=uuid.uuid4().hex,
service_catalog=service_catalog))
headers, container = client.get_container('bob')
self.assertIs(headers, "text")
self.assertIs(container, mock_resp)
def get_container_objects(container, storage_url, auth_token):
object_list = []
http_conn = client.http_connection(storage_url, insecure=settings.SWIFT_INSECURE)
_, objects = client.get_container(storage_url,
auth_token,
container,
http_conn=http_conn)
for object in objects:
object_list.append(object['name'])
return object_list
def delete_account_cont(self, account_url, token):
cnx = sclient.http_connection(account_url)
al = sclient.get_account(None, token, http_conn=cnx, full_listing=True)
for container in [c['name'] for c in al[1]]:
ci = sclient.get_container(None,
token,
container,
http_conn=cnx,
full_listing=True)
on = [od['name'] for od in ci[1]]
for obj in on:
sclient.delete_object('', token, container, obj, http_conn=cnx)
sclient.delete_container('', token, container, http_conn=cnx)
def cloudview(request, cloud, prefix=None):
""" Returns list of all objects in a cloud. """
storage_url = request.session.get('storage_url', settings.STORAGE_URL)
auth_token = request.session.get('auth_token', 'demo')
request.session['cloud'] = getcloud(request)
objects = []
read_acl = []
buckets = cloud_to_bucket(cloud)
arrayOfBuckets = buckets.split(",")
newkey = u"{key}".format(key='bucket')
for bucket in arrayOfBuckets:
try:
meta, objects1 = client.get_container(storage_url, auth_token, bucket, delimiter='/', prefix=prefix)
for entry in objects1:
value = u"{value}".format( value=bucket)
entry.update({newkey:value})
objects += objects1
read_acl += meta.get('x-container-read', '').split(',')
except client.ClientException:
messages.add_message(request, messages.ERROR, _("No Access to Bucket %s." % bucket))
#merge all buckets
prefixes = prefix_list(prefix)
pseudofolders, objs = pseudofolder_object_list(objects, prefix)
objs.sort(key=lambda x:x['last_modified'], reverse=True)
del objs[100:]
base_url = get_base_url(request)
account = storage_url.split('/')[-1]
public = False
required_acl = ['.r:*', '.rlistings']
if [x for x in read_acl if x in required_acl]:
public = True
return render_to_response("cloudview.html", {
'container': buckets,
'cloud': cloud,
'objects': objs,
'folders': pseudofolders,
'session': request.session,
'prefix': prefix,
'prefixes': prefixes,
'base_url': base_url,
'account': account,
'public': public},
context_instance=RequestContext(request))
def test_async_update_after_PUT(self):
cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
client.put_container(self.url,
self.token,
'c1',
headers={'X-Storage-Policy': self.policy.name})
# put an object while one container server is stopped so that we force
# an async update to it
kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
content = u'stuff'
client.put_object(self.url,
self.token,
'c1',
'o1',
contents=content,
content_type='test/ctype')
meta = client.head_object(self.url, self.token, 'c1', 'o1')
# re-start the container server and assert that it does not yet know
# about the object
start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
self.assertFalse(
direct_client.direct_get_container(cnodes[0], cpart, self.account,
'c1')[1])
# Run the object-updaters to be sure updates are done
Manager(['object-updater']).once()
# check the re-started container server got same update as others.
# we cannot assert the actual etag value because it may be encrypted
listing_etags = set()
for cnode in cnodes:
listing = direct_client.direct_get_container(
cnode, cpart, self.account, 'c1')[1]
self.assertEqual(1, len(listing))
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual('test/ctype', listing[0]['content_type'])
listing_etags.add(listing[0]['hash'])
self.assertEqual(1, len(listing_etags))
# check that listing meta returned to client is consistent with object
# meta returned to client
hdrs, listing = client.get_container(self.url, self.token, 'c1')
self.assertEqual(1, len(listing))
self.assertEqual('o1', listing[0]['name'])
self.assertEqual(len(content), listing[0]['bytes'])
self.assertEqual(meta['etag'], listing[0]['hash'])
self.assertEqual('test/ctype', listing[0]['content_type'])
def test_create_swift_client(self):
mock_resp = MagicMock()
swiftclient.client.Connection.get_container = MagicMock(
return_value=["text", mock_resp])
service_catalog = [{
'endpoints': [{
'publicURL': 'example.com'
}],
'type': 'object-store'
}]
client = remote.create_swift_client(
TroveContext(tenant='123', service_catalog=service_catalog))
headers, container = client.get_container('bob')
self.assertIs(headers, "text")
self.assertIs(container, mock_resp)
def delete_container(request, container):
""" Deletes a container """
storage_url = request.session.get('storage_url', '')
auth_token = request.session.get('auth_token', '')
try:
_m, objects = client.get_container(storage_url, auth_token, container)
for obj in objects:
client.delete_object(storage_url, auth_token, container,
obj['name'])
client.delete_container(storage_url, auth_token, container)
messages.add_message(request, messages.INFO, _("Container deleted."))
except client.ClientException:
messages.add_message(request, messages.ERROR, _("Access denied."))
return redirect(containerview)
def update_container(request, project_id, container_name):
if request.method == 'PUT':
sp = JSONParser().parse(request)
token = get_token_connection(request)
url = settings.SWIFT_URL + "/AUTH_" + project_id
headers, obj_list = swift_client.get_container(url, token,
container_name)
headers['X-Storage-Policy'] = sp
path_container = settings.SWIFT_CFG_TMP_DIR + "/" + container_name
os.mkdir(path_container)
for obj in obj_list:
file = open(path_container + "/" + obj["name"], "w")
obj_headers, obj_body = swift_client.get_object(
url, token, container_name, obj["name"])
file.write(obj_body)
file.close()
obj["headers"] = obj_headers
swift_client.delete_object(url, token, container_name, obj["name"])
swift_client.delete_container(url, token, container_name)
swift_client.put_container(url, token, container_name, headers)
for obj in obj_list:
obj_path = os.path.join(path_container, obj["name"])
obj_body = open(obj_path, "r")
content_length = os.stat(obj_path).st_size
swift_response = {}
swift_client.put_object(url, token, container_name, obj["name"],
obj_body, content_length, None, None,
obj['content_type'], obj["headers"], None,
None, None, swift_response)
obj_body.close()
os.remove(obj_path)
os.rmdir(path_container)
return JSONResponse("Container Policy updated correctly",
status=status.HTTP_201_CREATED)
return JSONResponse('Method ' + str(request.method) + ' not allowed.',
status=status.HTTP_405_METHOD_NOT_ALLOWED)
def get_all(request, container):
""" Returns List of all Objects present in specified container
:param container: Name of Swift Container
:return: meta data and id's of all the objects
"""
try:
response_dict = dict()
storage_url, auth_token = _get_auth_data(request.session)
data_container = client.get_container(storage_url, auth_token,
container)
for obj in data_container[1]:
meta_object = client.head_object(storage_url, auth_token,
container, obj['name'])
if not meta_object['x-object-meta-deleted']:
if meta_object['x-object-meta-type'] in [
'thumbnail', 'original-thumbnail'
]:
if meta_object[
'x-object-meta-format'] not in response_dict:
form = meta_object['x-object-meta-format']
response_dict[form] = []
if meta_object['x-object-meta-format'] in response_dict:
new_obj = {
'thumbnail_id': obj['name'],
'name': meta_object['x-object-meta-name'],
'type': meta_object['x-object-meta-type'],
'resolution':
meta_object['x-object-meta-resolution']
}
if meta_object['x-object-meta-format'] is 'thumbnail':
new_obj['original_id'] = meta_object[
'x-object-meta-original']
response_dict[form].append(new_obj)
return Response(response_dict)
except swift_exception.ClientException as e:
print e
return Response('Please contact your admininstrator',
status=status.HTTP_501_NOT_IMPLEMENTED)
except Exception as e:
print e
return Response('Please contact your admininstrator',
status=status.HTTP_501_NOT_IMPLEMENTED)
def objectview(request, container, prefix=None):
""" Returns list of all objects in current container. """
storage_url = request.session.get('storage_url', '')
auth_token = request.session.get('auth_token', '')
try:
meta, objects = client.get_container(storage_url,
auth_token,
container,
delimiter='/',
prefix=prefix)
except client.ClientException:
traceback.print_exc()
messages.add_message(request, messages.ERROR, _("Access denied."))
return redirect(containerview)
prefixes = prefix_list(prefix)
pseudofolders, objs = pseudofolder_object_list(objects, prefix)
base_url = get_base_url(request)
account = storage_url.split('/')[-1]
read_acl = meta.get('x-container-read', '').split(',')
public = False
required_acl = ['.r:*', '.rlistings']
if [x for x in read_acl if x in required_acl]:
public = True
return render_to_response("objectview.html", {
'container': container,
'objects': objs,
'folders': pseudofolders,
'session': request.session,
'prefix': prefix,
'prefixes': prefixes,
'base_url': base_url,
'account': account,
'public': public
},
context_instance=RequestContext(request))
def _test_syncing(self, source_container, dest_container):
# test syncing and versioning
object_name = 'object-%s' % uuid.uuid4()
client.put_object(self.url, self.token, source_container, object_name,
'version1')
# cycle container-sync
Manager(['container-sync']).once()
# overwrite source
client.put_object(self.url, self.token, source_container, object_name,
'version2')
# cycle container-sync
Manager(['container-sync']).once()
resp_headers, listing = client.get_container(
self.url, self.token, dest_container,
query_string='versions')
self.assertEqual(2, len(listing))