def wrapper(*args, **kw):
storage_url = args[0].session.get('storage_url', '')
auth_token = args[0].session.get('auth_token', '')
username = args[0].session.get('username', '')
password = args[0].session.get('password', '')
try:
client.head_account(storage_url, auth_token)
return fn(*args, **kw)
except:
#Attempt to get a new auth token
try:
auth_version = settings.SWIFT_AUTH_VERSION or 1
(storage_url, auth_token) = client.get_auth(
settings.SWIFT_AUTH_URL, username, password,
auth_version=auth_version)
args[0].session['auth_token'] = auth_token
args[0].session['storage_url'] = storage_url
return fn(*args, **kw)
except:
messages.error(args[0], _("Session expired."))
return {'errors': 'true'}
def wrapper(*args, **kw):
storage_url = args[0].session.get('storage_url', '')
auth_token = args[0].session.get('auth_token', '')
username = args[0].session.get('username', '')
password = args[0].session.get('password', '')
# If the following variables are available, attempt to get an
# auth token
if (storage_url and auth_token and username and password):
try:
client.head_account(storage_url, auth_token)
return fn(*args, **kw)
except:
#Attempt to get a new auth token
try:
auth_version = settings.SWIFT_AUTH_VERSION or 1
(storage_url, auth_token) = client.get_auth(
settings.SWIFT_AUTH_URL, username, password,
auth_version=auth_version)
args[0].session['auth_token'] = auth_token
args[0].session['storage_url'] = storage_url
return fn(*args, **kw)
except:
# Failure to get an auth token, tell the user the session
# has expired.
messages.error(args[0], _("Session expired."))
return redirect(swiftbrowser.views.login)
def test_server_error(self):
body = "c" * 65
c.http_connection = self.fake_http_connection(500, body=body)
self.assertRaises(c.ClientException, c.head_account, "http://www.tests.com", "asdf")
try:
c.head_account("http://www.tests.com", "asdf")
except c.ClientException as e:
new_body = "[first 60 chars of response] " + body[0:60]
self.assertEqual(e.__str__()[-89:], new_body)
def test_server_error(self):
body = 'c' * 65
c.http_connection = self.fake_http_connection(500, body=body)
self.assertRaises(c.ClientException, c.head_account,
'http://www.tests.com', 'asdf')
try:
c.head_account('http://www.tests.com', 'asdf')
except c.ClientException as e:
new_body = "[first 60 chars of response] " + body[0:60]
self.assertEquals(e.__str__()[-89:], new_body)
def wrapper(*args, **kw):
storage_url = args[0].session.get('storage_url', '')
auth_token = args[0].session.get('auth_token', '')
username = args[0].session.get('username', '')
password = args[0].session.get('password', '')
# If the following variables are available, attempt to get an
# auth token
if (storage_url and auth_token and username and password):
# If the user has no role, head the container to valid the token
if args[0].session.get('norole'):
storage_url = args[0].session.get('default_storage_url', '')
auth_token = args[0].session.get('default_auth_token', '')
#Attempt to get a new auth token
try:
client.head_container(
storage_url, auth_token, args[0].session.get('user'))
return fn(*args, **kw)
except:
# Failure to get an auth token, tell the user the session
# has expiredself.
messages.error(args[0], _("Session expired."))
# A regular user's token is validated by heading the account.
else:
try:
client.head_account(storage_url, auth_token)
return fn(*args, **kw)
except:
#Attempt to get a new auth token
try:
auth_version = settings.SWIFT_AUTH_VERSION or 1
(storage_url, auth_token) = client.get_auth(
settings.SWIFT_AUTH_URL, username, password,
auth_version=auth_version)
args[0].session['auth_token'] = auth_token
args[0].session['storage_url'] = storage_url
return fn(*args, **kw)
except:
# Failure to get an auth token, tell the user the
# session has expired.
messages.error(args[0], _("Session expired."))
return redirect(swiftbrowser.views.login)
def get_temp_key(storage_url, auth_token):
""" Tries to get meta-temp-url key from account.
If not set, generate tempurl and save it to acocunt.
This requires at least account owner rights. """
logging.debug(' in get_temp_key: ' )
try:
account = client.head_account(storage_url, auth_token)
except client.ClientException:
return None
logging.debug(' account in get_temp_key: %s ' % account)
key = account.get('x-account-meta-temp-url-key')
logging.debug(' key in get_temp_key: %s ' % key)
if not key:
chars = string.ascii_lowercase + string.digits
key = ''.join(random.choice(chars) for x in range(32))
headers = {'x-account-meta-temp-url-key': key}
try:
client.post_account(storage_url, auth_token, headers)
except client.ClientException:
return None
return key
def setUp(self):
super(TestAccountReaper, self).setUp()
self.all_objects = []
# upload some containers
body = 'test-body'
for policy in ENABLED_POLICIES:
container = 'container-%s-%s' % (policy.name, uuid.uuid4())
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy': policy.name})
obj = 'object-%s' % uuid.uuid4()
client.put_object(self.url, self.token, container, obj, body)
self.all_objects.append((policy, container, obj))
policy.load_ring('/etc/swift')
Manager(['container-updater']).once()
headers = client.head_account(self.url, self.token)
self.assertEqual(int(headers['x-account-container-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-object-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-bytes-used']),
len(ENABLED_POLICIES) * len(body))
part, nodes = self.account_ring.get_nodes(self.account)
for node in nodes:
direct_delete_account(node, part, self.account)
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
value = c.head_account('http://www.tests.com', 'asdf')
# TODO: Hmm. This doesn't really test too much as it uses a fake that
# always returns the same dict. I guess it "exercises" the code, so
# I'll leave it for now.
self.assertEquals(type(value), dict)
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
value = c.head_account('http://www.tests.com', 'asdf')
# TODO: Hmm. This doesn't really test too much as it uses a fake that
# always returns the same dict. I guess it "exercises" the code, so
# I'll leave it for now.
self.assertEqual(type(value), dict)
def accountview(request, project):
storage_url = get_storage_endpoint(request, 'adminURL')
if not storage_url:
return
auth_token = request.session.get('auth_token')
http_conn = client.http_connection(storage_url,
insecure=settings.SWIFT_INSECURE)
head_acc = {}
try:
head_acc = client.head_account(storage_url,
auth_token,
http_conn=http_conn)
except Exception as err:
log.exception(f'Exception: {err}')
context = {
'cloud': head_acc.get('x-account-meta-cloud'),
'total_containers': head_acc.get('x-account-container-count'),
'total_objects': head_acc.get('x-account-object-count'),
'total_bytes': head_acc.get('x-account-bytes-used'),
'account_domain_id': head_acc.get('x-account-project-domain-id'),
}
return render(request, 'storage/accountview.html',
update_default_context(request, context))
def setUp(self):
super(TestAccountReaper, self).setUp()
self.all_objects = []
# upload some containers
body = 'test-body'
for policy in ENABLED_POLICIES:
container = 'container-%s-%s' % (policy.name, uuid.uuid4())
client.put_container(self.url,
self.token,
container,
headers={'X-Storage-Policy': policy.name})
obj = 'object-%s' % uuid.uuid4()
client.put_object(self.url, self.token, container, obj, body)
self.all_objects.append((policy, container, obj))
policy.load_ring('/etc/swift')
Manager(['container-updater']).once()
headers = client.head_account(self.url, self.token)
self.assertEqual(int(headers['x-account-container-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-object-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-bytes-used']),
len(ENABLED_POLICIES) * len(body))
part, nodes = self.account_ring.get_nodes(self.account)
for node in nodes:
direct_delete_account(node, part, self.account)
def iter_accounts(ksclient):
endpoint = ksclient.service_catalog.url_for(
service_type='object-store',
endpoint_type='adminURL')
base_url = '%s/v1/%s' % (endpoint, cfg.CONF.reseller_prefix)
for t in ksclient.tenants.list():
yield (t.id, swift.head_account('%s%s' % (base_url, t.id),
ksclient.auth_token))
def get_swift_used(sc, tenant_id):
tenant_url, token = get_swift_tenant_connection(sc, tenant_id)
try:
swift_account = swiftclient.head_account(url=tenant_url, token=token)
except sc_exception:
print 'Project %s has no swift quota' % tenant_id
return
return swift_account.get(SWIFT_USED_KEY, -1)
def get_default_temp_time(storage_url, auth_token):
"""Return in seconds the header Default-Temp-Time for the given tenant.
If an exception is caught, return 0."""
try:
cont = client.head_account(storage_url, auth_token)
return cont.get('x-account-meta-default-temp-time', '')
except:
return 0
def iter_accounts(ksclient):
try:
endpoint = ksclient.service_catalog.url_for(service_type="object-store", endpoint_type="adminURL")
except exceptions.EndpointNotFound:
LOG.debug(_("Swift endpoint not found"))
return
for t in ksclient.tenants.list():
yield (t.id, swift.head_account(SwiftPollster._neaten_url(endpoint, t.id), ksclient.auth_token))
def login(request):
""" Tries to login user and sets session data """
request.session.flush()
#Process the form if there is a POST request.
if (request.POST):
form = LoginForm(request.POST)
if form.is_valid():
username = form.cleaned_data['username']
password = form.cleaned_data['password']
try:
auth_version = settings.SWIFT_AUTH_VERSION or 1
(storage_url, auth_token) = client.get_auth(
settings.SWIFT_AUTH_URL, username, password,
auth_version=auth_version)
request.session['auth_token'] = auth_token
request.session['storage_url'] = storage_url
request.session['username'] = username
request.session['password'] = password
tenant_name, user = split_tenant_user_names(username)
request.session['user'] = user
request.session['tenant_name'] = tenant_name
# Upon successful retrieval of a token, if we're unable to
# head the account, then the user is not an admin or
# swiftoperator and has access to only a container.
try:
client.head_account(storage_url, auth_token)
except:
request.session['norole'] = True
return redirect(containerview)
# Specify why the login failed.
except client.ClientException, e:
messages.error(request, _("Login failed: {0}".format(
e)))
# Generic login failure message.
except Exception, e:
print(e)
messages.error(request, _("Login failed."))
def test_sync(self):
all_objects = []
# upload some containers
for policy in ENABLED_POLICIES:
container = 'container-%s-%s' % (policy.name, uuid.uuid4())
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy': policy.name})
obj = 'object-%s' % uuid.uuid4()
body = 'test-body'
client.put_object(self.url, self.token, container, obj, body)
all_objects.append((policy, container, obj))
Manager(['container-updater']).once()
headers = client.head_account(self.url, self.token)
self.assertEqual(int(headers['x-account-container-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-object-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-bytes-used']),
len(ENABLED_POLICIES) * len(body))
part, nodes = self.account_ring.get_nodes(self.account)
for node in nodes:
direct_delete_account(node, part, self.account)
Manager(['account-reaper']).once()
get_to_final_state()
for policy, container, obj in all_objects:
cpart, cnodes = self.container_ring.get_nodes(
self.account, container)
for cnode in cnodes:
try:
direct_head_container(cnode, cpart, self.account,
container)
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail('Found un-reaped /%s/%s on %r' %
(self.account, container, node))
object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/')
part, nodes = object_ring.get_nodes(self.account, container, obj)
for node in nodes:
try:
direct_get_object(node, part, self.account,
container, obj)
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail('Found un-reaped /%s/%s/%s on %r in %s!' %
(self.account, container, obj, node, policy))
def iter_accounts():
ks = ksclient.Client(username=cfg.CONF.os_username,
password=cfg.CONF.os_password,
tenant_name=cfg.CONF.os_tenant_name,
auth_url=cfg.CONF.os_auth_url)
endpoint = ks.service_catalog.url_for(service_type='object-store',
endpoint_type='adminURL')
base_url = '%s/v1/%s' % (endpoint, cfg.CONF.reseller_prefix)
for t in ks.tenants.list():
yield (t.id, swift.head_account('%s%s' % (base_url, t.id),
ks.auth_token))
def get_tempurl_key():
(storage_url, auth_token) = client.get_auth(settings.SWIFT_AUTH_URL, settings.SWIFT_USER, settings.SWIFT_PASSWORD)
meta = client.head_account(storage_url, auth_token)
key = meta.get("x-account-meta-temp-url-key")
if not key:
key = random_key()
headers = {"x-account-meta-temp-url-key": key}
client.post_account(storage_url, auth_token, headers)
return storage_url, key
def test_sync(self):
all_objects = []
# upload some containers
for policy in ENABLED_POLICIES:
container = 'container-%s-%s' % (policy.name, uuid.uuid4())
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy': policy.name})
obj = 'object-%s' % uuid.uuid4()
body = 'test-body'
client.put_object(self.url, self.token, container, obj, body)
all_objects.append((policy, container, obj))
Manager(['container-updater']).once()
headers = client.head_account(self.url, self.token)
self.assertEqual(int(headers['x-account-container-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-object-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-bytes-used']),
len(ENABLED_POLICIES) * len(body))
part, nodes = self.account_ring.get_nodes(self.account)
for node in nodes:
direct_delete_account(node, part, self.account)
Manager(['account-reaper']).once()
get_to_final_state()
for policy, container, obj in all_objects:
cpart, cnodes = self.container_ring.get_nodes(
self.account, container)
for cnode in cnodes:
try:
direct_head_container(cnode, cpart, self.account,
container)
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail('Found un-reaped /%s/%s on %r' %
(self.account, container, node))
object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/')
part, nodes = object_ring.get_nodes(self.account, container, obj)
for node in nodes:
try:
direct_get_object(node, part, self.account,
container, obj)
except ClientException as err:
self.assertEquals(err.http_status, 404)
else:
self.fail('Found un-reaped /%s/%s/%s on %r in %s!' %
(self.account, container, obj, node, policy))
def _get_account_info(self, ksclient, cache):
try:
endpoint = ksclient.service_catalog.url_for(
service_type='object-store',
endpoint_type='adminURL')
except exceptions.EndpointNotFound:
LOG.debug(_("Swift endpoint not found"))
raise StopIteration()
for t in cache['tenants']:
yield (t.id, swift.head_account(self._neaten_url(endpoint, t.id),
ksclient.auth_token))
def _get_account_info(self, ksclient, cache):
try:
endpoint = ksclient.service_catalog.url_for(
service_type='object-store',
endpoint_type=cfg.CONF.service_credentials.os_endpoint_type)
except exceptions.EndpointNotFound:
LOG.debug(_("Swift endpoint not found"))
raise StopIteration()
for t in cache[self.CACHE_KEY_TENANT]:
yield (t.id, swift.head_account(self._neaten_url(endpoint, t.id),
ksclient.auth_token))
def _get_account_info(self, ksclient, cache):
try:
endpoint = ksclient.service_catalog.url_for(
service_type='object-store',
endpoint_type=cfg.CONF.service_credentials.os_endpoint_type)
except exceptions.EndpointNotFound:
LOG.debug(_("Swift endpoint not found"))
raise StopIteration()
for t in cache['tenants']:
yield (t.id,
swift.head_account(self._neaten_url(endpoint, t.id),
ksclient.auth_token))
def iter_accounts(ksclient):
try:
endpoint = ksclient.service_catalog.url_for(
service_type='object-store',
endpoint_type='adminURL')
except exceptions.EndpointNotFound:
LOG.debug(_("Swift endpoint not found"))
return
base_url = '%s/v1/%s' % (endpoint, cfg.CONF.reseller_prefix)
for t in ksclient.tenants.list():
yield (t.id, swift.head_account('%s%s' % (base_url, t.id),
ksclient.auth_token))
def iter_accounts(ksclient):
try:
endpoint = ksclient.service_catalog.url_for(
service_type='object-store', endpoint_type='adminURL')
except exceptions.EndpointNotFound:
LOG.debug(_("Swift endpoint not found"))
return
base_url = '%s/v1/%s' % (endpoint, cfg.CONF.reseller_prefix)
for t in ksclient.tenants.list():
yield (t.id,
swift.head_account('%s%s' % (base_url, t.id),
ksclient.auth_token))
def iter_accounts(ksclient):
try:
endpoint = ksclient.service_catalog.url_for(
service_type='object-store', endpoint_type='adminURL')
except exceptions.EndpointNotFound:
LOG.debug(_("Swift endpoint not found"))
return
for t in ksclient.tenants.list():
yield (t.id,
swift.head_account(
SwiftPollster._neaten_url(endpoint, t.id),
ksclient.auth_token))
def get_swift_usage(tenant_url, token):
usage = {}
usage['gb_allocated'] = usage['gb_used'] = 0
try:
swift_account = swiftclient.head_account(url=tenant_url, token=token)
if SWIFT_QUOTA_KEY in swift_account:
usage['gb_allocated'] = b_to_gb(swift_account[SWIFT_QUOTA_KEY])
usage['gb_used'] = b_to_gb(swift_account['x-account-bytes-used'])
except ClientException:
pass
return usage
def get_fine_grained_temp_key(storage_url, auth_token, container_name=None):
"""
Tries to get meta-temp-url key from account or container.
If not set, generate tempurl and save it.
"""
logging.debug(' in get_fine_grained_temp_key: container_name:%s, \
storage_url:%s ' %
(container_name, storage_url) )
try:
if container_name:
container = client.head_container(storage_url, auth_token,
container_name)
key = container.get('x-container-meta-temp-url-key')
logging.debug(' key in get_fine_grained_temp_key container: %s ' % key)
else:
account = client.head_account(storage_url, auth_token)
key = account.get('x-account-meta-temp-url-key')
logging.debug(' key in get_fine_grained_temp_key account: %s ' % key)
except client.ClientException:
return None
# logging.debug(' account or container in get_temp_key: %s '
# % account or container)
if not key:
chars = string.ascii_lowercase + string.digits
key = ''.join(random.choice(chars) for x in range(32))
if container_name:
headers = {'x-container-meta-temp-url-key': key}
try:
client.post_container(storage_url, auth_token, container_name,
headers)
logging.debug(' post_container')
except client.ClientException:
return None
raise ValueError('cannot get key, have no account rights to \
get account key!')
else:
headers = {'x-account-meta-temp-url-key': key}
try:
client.post_account(storage_url, auth_token, headers)
logging.debug(' post_account')
except client.ClientException:
return None
return key
def get_account(self, deep=True):
if not self.http_conn:
self.connect()
account_info = swift.head_account(url=self.swift_url, token=self.token, http_conn=self.http_conn)
account_head, containers = swift.get_account(url=self.swift_url, token=self.token, http_conn=self.http_conn)
if self.debug:
print(account_info)
print(account_head)
for container in containers:
print(container)
print
if deep:
self.get_containers(containers)
def setUp(self):
super(TestAccountReaper, self).setUp()
self.all_objects = []
int_client = self.make_internal_client()
# upload some containers
body = b'test-body'
for policy in ENABLED_POLICIES:
container = 'container-%s-%s' % (policy.name, uuid.uuid4())
client.put_container(self.url,
self.token,
container,
headers={'X-Storage-Policy': policy.name})
obj = 'object-%s' % uuid.uuid4()
client.put_object(self.url, self.token, container, obj, body)
self.all_objects.append((policy, container, obj))
# Also create some reserved names
container = get_reserved_name('reserved', policy.name,
str(uuid.uuid4()))
int_client.create_container(
self.account,
container,
headers={'X-Storage-Policy': policy.name})
obj = get_reserved_name('object', str(uuid.uuid4()))
int_client.upload_object(BytesIO(body), self.account, container,
obj)
self.all_objects.append((policy, container, obj))
policy.load_ring('/etc/swift')
Manager(['container-updater']).once()
headers = client.head_account(self.url, self.token)
self.assertEqual(int(headers['x-account-container-count']),
len(self.all_objects))
self.assertEqual(int(headers['x-account-object-count']),
len(self.all_objects))
self.assertEqual(int(headers['x-account-bytes-used']),
len(self.all_objects) * len(body))
part, nodes = self.account_ring.get_nodes(self.account)
for node in nodes:
direct_delete_account(node, part, self.account)
def test_sync(self):
all_objects = []
# upload some containers
for policy in ENABLED_POLICIES:
container = 'container-%s-%s' % (policy.name, uuid.uuid4())
client.put_container(self.url, self.token, container,
headers={'X-Storage-Policy': policy.name})
obj = 'object-%s' % uuid.uuid4()
body = 'test-body'
client.put_object(self.url, self.token, container, obj, body)
all_objects.append((policy, container, obj))
Manager(['container-updater']).once()
headers = client.head_account(self.url, self.token)
self.assertEqual(int(headers['x-account-container-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-object-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-bytes-used']),
len(ENABLED_POLICIES) * len(body))
part, nodes = self.account_ring.get_nodes(self.account)
for node in nodes:
direct_delete_account(node, part, self.account)
# run the reaper
Manager(['account-reaper']).once()
for policy, container, obj in all_objects:
# verify that any container deletes were at same timestamp
cpart, cnodes = self.container_ring.get_nodes(
self.account, container)
delete_times = set()
for cnode in cnodes:
try:
direct_head_container(cnode, cpart, self.account,
container)
except ClientException as err:
self.assertEqual(err.http_status, 404)
delete_time = err.http_headers.get(
'X-Backend-DELETE-Timestamp')
# 'X-Backend-DELETE-Timestamp' confirms it was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
# Container replicas may not yet be deleted if we have a
# policy with object replicas < container replicas, so
# ignore successful HEAD. We'll check for all replicas to
# be deleted again after running the replicators.
pass
self.assertEqual(1, len(delete_times), delete_times)
# verify that all object deletes were at same timestamp
object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/')
part, nodes = object_ring.get_nodes(self.account, container, obj)
headers = {'X-Backend-Storage-Policy-Index': int(policy)}
delete_times = set()
for node in nodes:
try:
direct_get_object(node, part, self.account,
container, obj, headers=headers)
except ClientException as err:
self.assertEqual(err.http_status, 404)
delete_time = err.http_headers.get('X-Backend-Timestamp')
# 'X-Backend-Timestamp' confirms obj was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
self.fail('Found un-reaped /%s/%s/%s on %r in %s!' %
(self.account, container, obj, node, policy))
self.assertEqual(1, len(delete_times))
# run replicators and updaters
self.get_to_final_state()
for policy, container, obj in all_objects:
# verify that ALL container replicas are now deleted
cpart, cnodes = self.container_ring.get_nodes(
self.account, container)
delete_times = set()
for cnode in cnodes:
try:
direct_head_container(cnode, cpart, self.account,
container)
except ClientException as err:
self.assertEqual(err.http_status, 404)
delete_time = err.http_headers.get(
'X-Backend-DELETE-Timestamp')
# 'X-Backend-DELETE-Timestamp' confirms it was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
self.fail('Found un-reaped /%s/%s on %r' %
(self.account, container, cnode))
# sanity check that object state is still consistent...
object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/')
part, nodes = object_ring.get_nodes(self.account, container, obj)
headers = {'X-Backend-Storage-Policy-Index': int(policy)}
delete_times = set()
for node in nodes:
try:
direct_get_object(node, part, self.account,
container, obj, headers=headers)
except ClientException as err:
self.assertEqual(err.http_status, 404)
delete_time = err.http_headers.get('X-Backend-Timestamp')
# 'X-Backend-Timestamp' confirms obj was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
self.fail('Found un-reaped /%s/%s/%s on %r in %s!' %
(self.account, container, obj, node, policy))
self.assertEqual(1, len(delete_times))
def get_swift_quota(sc, tenant):
tenant_url, token = get_swift_tenant_connection(sc, tenant.id)
swift_account = swift_client.head_account(url=tenant_url, token=token)
quota = convert_quota_to_gb(int(swift_account.get(SWIFT_QUOTA_KEY, -1)))
return quota
def objectview(request, container, prefix=None):
""" Returns list of all objects in current container. """
storage_url = request.session.get('storage_url', '')
auth_token = request.session.get('auth_token', '')
# Users with no role use container keys
if request.session.get('norole'):
key = request.session.get("keys")[container]
# Regular users use account keys
else:
account = client.head_account(storage_url, auth_token)
key = account.get('x-account-meta-temp-url-key', '')
request.session['container'] = container
request.session['prefix'] = prefix
request.session["key"] = key
try:
meta = client.head_container(
storage_url, auth_token, container,
headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')})
except client.ClientException:
messages.add_message(request, messages.ERROR, _("Access denied."))
return redirect(containerview)
# Check CORS header - BASE_URL should be in there. Do not perform this
# check for users with no role. No role users will not be accessing
# containers in any way except for swiftbrowser. Hence their container has
# the proper headers. Norole users are unable to set the header anyways.
if meta.get(
'x-container-meta-access-control-allow-origin'
) != settings.BASE_URL and not request.session.get('norole', False):
# Add CORS headers so user can upload to this container.
headers = {
'X-Container-Meta-Access-Control-Expose-Headers':
'Access-Control-Allow-Origin',
'X-Container-Meta-Access-Control-Allow-Origin': settings.BASE_URL,
}
try:
client.put_container(
storage_url, auth_token, container, headers)
except client.ClientException:
messages.add_message(request, messages.ERROR, _(
"Access denied, unable to set CORS header."))
return redirect(containerview)
prefixes = prefix_list(prefix)
base_url = get_base_url(request)
account = storage_url.split('/')[-1]
read_acl = meta.get('x-container-read', '').split(',')
public = False
required_acl = ['.r:*', '.rlistings']
# The swifturl is the URL that the browser can send posts to upload files.
swift_url = storage_url + '/' + container + '/'
swift_slo_url = storage_url + '/' + container + '_segments/'
if prefix:
swift_url += prefix
swift_slo_url += prefix
# Posts from the browser to swift need a valid signature that's created
# from the tempurl key
signature = create_formpost_signature(swift_url, key)
slo_signature = create_formpost_signature(swift_slo_url, key)
if [x for x in read_acl if x in required_acl]:
public = True
return render_to_response(
"objectview.html",
{
'swift_url': swift_url,
'swift_slo_url': swift_slo_url,
'signature': signature,
'slo_signature': slo_signature,
'container': container,
'session': request.session,
'prefix': prefix,
'prefixes': prefixes,
'base_url': base_url,
'account': account,
'public': public,
'max_file_size': 5368709120,
'max_file_count': 1,
'expires': int(time.time() + 60 * 60 * 2),
},
context_instance=RequestContext(request)
)
def test_sync(self):
all_objects = []
# upload some containers
for policy in ENABLED_POLICIES:
container = 'container-%s-%s' % (policy.name, uuid.uuid4())
client.put_container(self.url,
self.token,
container,
headers={'X-Storage-Policy': policy.name})
obj = 'object-%s' % uuid.uuid4()
body = 'test-body'
client.put_object(self.url, self.token, container, obj, body)
all_objects.append((policy, container, obj))
Manager(['container-updater']).once()
headers = client.head_account(self.url, self.token)
self.assertEqual(int(headers['x-account-container-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-object-count']),
len(ENABLED_POLICIES))
self.assertEqual(int(headers['x-account-bytes-used']),
len(ENABLED_POLICIES) * len(body))
part, nodes = self.account_ring.get_nodes(self.account)
for node in nodes:
direct_delete_account(node, part, self.account)
# run the reaper
Manager(['account-reaper']).once()
for policy, container, obj in all_objects:
# verify that any container deletes were at same timestamp
cpart, cnodes = self.container_ring.get_nodes(
self.account, container)
delete_times = set()
for cnode in cnodes:
try:
direct_head_container(cnode, cpart, self.account,
container)
except ClientException as err:
self.assertEquals(err.http_status, 404)
delete_time = err.http_headers.get(
'X-Backend-DELETE-Timestamp')
# 'X-Backend-DELETE-Timestamp' confirms it was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
# Container replicas may not yet be deleted if we have a
# policy with object replicas < container replicas, so
# ignore successful HEAD. We'll check for all replicas to
# be deleted again after running the replicators.
pass
self.assertEqual(1, len(delete_times), delete_times)
# verify that all object deletes were at same timestamp
object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/')
part, nodes = object_ring.get_nodes(self.account, container, obj)
headers = {'X-Backend-Storage-Policy-Index': int(policy)}
delete_times = set()
for node in nodes:
try:
direct_get_object(node,
part,
self.account,
container,
obj,
headers=headers)
except ClientException as err:
self.assertEquals(err.http_status, 404)
delete_time = err.http_headers.get('X-Backend-Timestamp')
# 'X-Backend-Timestamp' confirms obj was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
self.fail('Found un-reaped /%s/%s/%s on %r in %s!' %
(self.account, container, obj, node, policy))
self.assertEqual(1, len(delete_times))
# run replicators and updaters
self.get_to_final_state()
for policy, container, obj in all_objects:
# verify that ALL container replicas are now deleted
cpart, cnodes = self.container_ring.get_nodes(
self.account, container)
delete_times = set()
for cnode in cnodes:
try:
direct_head_container(cnode, cpart, self.account,
container)
except ClientException as err:
self.assertEquals(err.http_status, 404)
delete_time = err.http_headers.get(
'X-Backend-DELETE-Timestamp')
# 'X-Backend-DELETE-Timestamp' confirms it was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
self.fail('Found un-reaped /%s/%s on %r' %
(self.account, container, cnode))
# sanity check that object state is still consistent...
object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/')
part, nodes = object_ring.get_nodes(self.account, container, obj)
headers = {'X-Backend-Storage-Policy-Index': int(policy)}
delete_times = set()
for node in nodes:
try:
direct_get_object(node,
part,
self.account,
container,
obj,
headers=headers)
except ClientException as err:
self.assertEquals(err.http_status, 404)
delete_time = err.http_headers.get('X-Backend-Timestamp')
# 'X-Backend-Timestamp' confirms obj was deleted
self.assertTrue(delete_time)
delete_times.add(delete_time)
else:
self.fail('Found un-reaped /%s/%s/%s on %r in %s!' %
(self.account, container, obj, node, policy))
self.assertEqual(1, len(delete_times))
def swift_cloud_project_status(request):
project_id = request.GET.get('project_id')
if not project_id:
raise Http404
try:
keystone = Keystone(request)
project = keystone.project_get(project_id)
except Exception as err:
log.exception(f"Keystone error: {err}")
return render(request, "vault/swift_cloud/project_status.html", {
"project": None,
"error": "Keystone Error"
})
sct_client = SCTClient(settings.SWIFT_CLOUD_TOOLS_URL,
settings.SWIFT_CLOUD_TOOLS_API_KEY)
status = ""
data = {}
try:
response = sct_client.transfer_get(project_id)
if response.ok:
data = response.json()
else:
status = _("Couldn't get migration data")
except Exception as err:
log.exception(f"Swift Cloud Tools Error: {err}")
if response.status_code == 404:
status = _("Migration not initialized yet")
if response.status_code < 300:
status = _("Waiting in migration queue")
if data.get("initial_date") and not data.get("final_date"):
status = _("Migrating...")
if data.get("final_date"):
status = _("Migration Completed")
http_conn, storage_url = get_conn_and_storage_url(request, project_id)
auth_token = request.session.get('auth_token')
head_account = {}
try:
head_account = client.head_account(storage_url,
auth_token,
http_conn=http_conn)
except Exception as err:
log.exception(f'Exception: {err}')
if head_account.get("x-account-meta-cloud-remove"):
status = _("Marked for removal")
return render(
request, "vault/swift_cloud/project_status.html", {
"project": project,
"status": status,
"metadata": json.dumps(head_account),
"migration_data": json.dumps(data)
})
def objectview(request, container, prefix=None):
""" Returns list of all objects in current container. """
storage_url = request.session.get("storage_url", "")
auth_token = request.session.get("auth_token", "")
# Users with no role use container keys
if request.session.get("norole"):
container_object = client.head_container(storage_url, auth_token, container)
key = container_object.get("x-container-meta-temp-url-key", "")
# Regular users use account keys
else:
account = client.head_account(storage_url, auth_token)
key = account.get("x-account-meta-temp-url-key", "")
request.session["container"] = container
request.session["prefix"] = prefix
request.session["key"] = key
redirect_url = get_base_url(request)
redirect_url += reverse("objectview", kwargs={"container": container})
try:
meta, objects = client.get_container(storage_url, auth_token, container, delimiter="/", prefix=prefix)
except client.ClientException:
messages.add_message(request, messages.ERROR, _("Access denied."))
return redirect(containerview)
# Check CORS header - BASE_URL should be in there. Do not perform this
# check for users with no role. No role users will not be accessing
# containers in any way except for swiftbrowser. Hence their container has
# the proper headers. Norole users are unable to set the header anyways.
if meta.get("x-container-meta-access-control-allow-origin") != settings.BASE_URL and not request.session.get(
"norole", False
):
# Add CORS headers so user can upload to this container.
headers = {
"X-Container-Meta-Access-Control-Expose-Headers": "Access-Control-Allow-Origin",
"X-Container-Meta-Access-Control-Allow-Origin": settings.BASE_URL,
}
try:
client.put_container(storage_url, auth_token, container, headers)
except client.ClientException:
messages.add_message(request, messages.ERROR, _("Access denied, unable to set CORS header."))
return redirect(containerview)
prefixes = prefix_list(prefix)
pseudofolders, objs = pseudofolder_object_list(objects, prefix)
base_url = get_base_url(request)
account = storage_url.split("/")[-1]
read_acl = meta.get("x-container-read", "").split(",")
public = False
required_acl = [".r:*", ".rlistings"]
swift_url = storage_url + "/" + container + "/"
swift_slo_url = storage_url + "/" + container + "_segments/"
if prefix:
swift_url += prefix
swift_slo_url += prefix
redirect_url += prefix
signature = create_formpost_signature(swift_url, key)
slo_signature = create_formpost_signature(swift_slo_url, key)
if [x for x in read_acl if x in required_acl]:
public = True
return render_to_response(
"objectview.html",
{
"swift_url": swift_url,
"swift_slo_url": swift_slo_url,
"signature": signature,
"slo_signature": slo_signature,
"redirect_url": redirect_url,
"container": container,
"objects": objs,
"folders": pseudofolders,
"session": request.session,
"prefix": prefix,
"prefixes": prefixes,
"base_url": base_url,
"account": account,
"public": public,
"max_file_size": 5368709120,
"max_file_count": 1,
"expires": int(time.time() + 60 * 60 * 2),
},
context_instance=RequestContext(request),
)
from swiftclient.client import get_account, head_account, post_account
from swiftclient.exceptions import ClientException
auth_url = 'http://10.111.1.123:5000/v2.0'
account = 'test'
user = '******'
key = 'testing'
try:
acuser = "******" % (account, user)
(storage_url, token) = get_auth(auth_url, acuser, key, auth_version='2.0')
conn = http_connection(storage_url)
req_headers = {'X-Account-Meta-Test1': 'aaabbbccc',
'X-Account-Meta-Test2': '0123456789'}
resp_headers = {}
post_account(storage_url, token, req_headers,
response_dict=resp_headers, http_conn=conn)
print "[Response headers of post_account()]"
pp = pprint.PrettyPrinter(indent=2)
pp.pprint(resp_headers)
print ""
headers = head_account(storage_url, token, http_conn=conn)
print "[Response headers of head_account()]"
pp.pprint(headers)
except ClientException, e:
print e
