def wrapper(*args, **kw): storage_url = args[0].session.get('storage_url', '') auth_token = args[0].session.get('auth_token', '') username = args[0].session.get('username', '') password = args[0].session.get('password', '') try: client.head_account(storage_url, auth_token) return fn(*args, **kw) except: #Attempt to get a new auth token try: auth_version = settings.SWIFT_AUTH_VERSION or 1 (storage_url, auth_token) = client.get_auth( settings.SWIFT_AUTH_URL, username, password, auth_version=auth_version) args[0].session['auth_token'] = auth_token args[0].session['storage_url'] = storage_url return fn(*args, **kw) except: messages.error(args[0], _("Session expired.")) return {'errors': 'true'}
def wrapper(*args, **kw): storage_url = args[0].session.get('storage_url', '') auth_token = args[0].session.get('auth_token', '') username = args[0].session.get('username', '') password = args[0].session.get('password', '') # If the following variables are available, attempt to get an # auth token if (storage_url and auth_token and username and password): try: client.head_account(storage_url, auth_token) return fn(*args, **kw) except: #Attempt to get a new auth token try: auth_version = settings.SWIFT_AUTH_VERSION or 1 (storage_url, auth_token) = client.get_auth( settings.SWIFT_AUTH_URL, username, password, auth_version=auth_version) args[0].session['auth_token'] = auth_token args[0].session['storage_url'] = storage_url return fn(*args, **kw) except: # Failure to get an auth token, tell the user the session # has expired. messages.error(args[0], _("Session expired.")) return redirect(swiftbrowser.views.login)
def test_server_error(self): body = "c" * 65 c.http_connection = self.fake_http_connection(500, body=body) self.assertRaises(c.ClientException, c.head_account, "http://www.tests.com", "asdf") try: c.head_account("http://www.tests.com", "asdf") except c.ClientException as e: new_body = "[first 60 chars of response] " + body[0:60] self.assertEqual(e.__str__()[-89:], new_body)
def test_server_error(self): body = 'c' * 65 c.http_connection = self.fake_http_connection(500, body=body) self.assertRaises(c.ClientException, c.head_account, 'http://www.tests.com', 'asdf') try: c.head_account('http://www.tests.com', 'asdf') except c.ClientException as e: new_body = "[first 60 chars of response] " + body[0:60] self.assertEquals(e.__str__()[-89:], new_body)
def wrapper(*args, **kw): storage_url = args[0].session.get('storage_url', '') auth_token = args[0].session.get('auth_token', '') username = args[0].session.get('username', '') password = args[0].session.get('password', '') # If the following variables are available, attempt to get an # auth token if (storage_url and auth_token and username and password): # If the user has no role, head the container to valid the token if args[0].session.get('norole'): storage_url = args[0].session.get('default_storage_url', '') auth_token = args[0].session.get('default_auth_token', '') #Attempt to get a new auth token try: client.head_container( storage_url, auth_token, args[0].session.get('user')) return fn(*args, **kw) except: # Failure to get an auth token, tell the user the session # has expiredself. messages.error(args[0], _("Session expired.")) # A regular user's token is validated by heading the account. else: try: client.head_account(storage_url, auth_token) return fn(*args, **kw) except: #Attempt to get a new auth token try: auth_version = settings.SWIFT_AUTH_VERSION or 1 (storage_url, auth_token) = client.get_auth( settings.SWIFT_AUTH_URL, username, password, auth_version=auth_version) args[0].session['auth_token'] = auth_token args[0].session['storage_url'] = storage_url return fn(*args, **kw) except: # Failure to get an auth token, tell the user the # session has expired. messages.error(args[0], _("Session expired.")) return redirect(swiftbrowser.views.login)
def get_temp_key(storage_url, auth_token): """ Tries to get meta-temp-url key from account. If not set, generate tempurl and save it to acocunt. This requires at least account owner rights. """ logging.debug(' in get_temp_key: ' ) try: account = client.head_account(storage_url, auth_token) except client.ClientException: return None logging.debug(' account in get_temp_key: %s ' % account) key = account.get('x-account-meta-temp-url-key') logging.debug(' key in get_temp_key: %s ' % key) if not key: chars = string.ascii_lowercase + string.digits key = ''.join(random.choice(chars) for x in range(32)) headers = {'x-account-meta-temp-url-key': key} try: client.post_account(storage_url, auth_token, headers) except client.ClientException: return None return key
def setUp(self): super(TestAccountReaper, self).setUp() self.all_objects = [] # upload some containers body = 'test-body' for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() client.put_object(self.url, self.token, container, obj, body) self.all_objects.append((policy, container, obj)) policy.load_ring('/etc/swift') Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-object-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-bytes-used']), len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account)
def test_ok(self): c.http_connection = self.fake_http_connection(200) value = c.head_account('http://www.tests.com', 'asdf') # TODO: Hmm. This doesn't really test too much as it uses a fake that # always returns the same dict. I guess it "exercises" the code, so # I'll leave it for now. self.assertEquals(type(value), dict)
def test_ok(self): c.http_connection = self.fake_http_connection(200) value = c.head_account('http://www.tests.com', 'asdf') # TODO: Hmm. This doesn't really test too much as it uses a fake that # always returns the same dict. I guess it "exercises" the code, so # I'll leave it for now. self.assertEqual(type(value), dict)
def accountview(request, project): storage_url = get_storage_endpoint(request, 'adminURL') if not storage_url: return auth_token = request.session.get('auth_token') http_conn = client.http_connection(storage_url, insecure=settings.SWIFT_INSECURE) head_acc = {} try: head_acc = client.head_account(storage_url, auth_token, http_conn=http_conn) except Exception as err: log.exception(f'Exception: {err}') context = { 'cloud': head_acc.get('x-account-meta-cloud'), 'total_containers': head_acc.get('x-account-container-count'), 'total_objects': head_acc.get('x-account-object-count'), 'total_bytes': head_acc.get('x-account-bytes-used'), 'account_domain_id': head_acc.get('x-account-project-domain-id'), } return render(request, 'storage/accountview.html', update_default_context(request, context))
def iter_accounts(ksclient): endpoint = ksclient.service_catalog.url_for( service_type='object-store', endpoint_type='adminURL') base_url = '%s/v1/%s' % (endpoint, cfg.CONF.reseller_prefix) for t in ksclient.tenants.list(): yield (t.id, swift.head_account('%s%s' % (base_url, t.id), ksclient.auth_token))
def get_swift_used(sc, tenant_id): tenant_url, token = get_swift_tenant_connection(sc, tenant_id) try: swift_account = swiftclient.head_account(url=tenant_url, token=token) except sc_exception: print 'Project %s has no swift quota' % tenant_id return return swift_account.get(SWIFT_USED_KEY, -1)
def get_default_temp_time(storage_url, auth_token): """Return in seconds the header Default-Temp-Time for the given tenant. If an exception is caught, return 0.""" try: cont = client.head_account(storage_url, auth_token) return cont.get('x-account-meta-default-temp-time', '') except: return 0
def iter_accounts(ksclient): try: endpoint = ksclient.service_catalog.url_for(service_type="object-store", endpoint_type="adminURL") except exceptions.EndpointNotFound: LOG.debug(_("Swift endpoint not found")) return for t in ksclient.tenants.list(): yield (t.id, swift.head_account(SwiftPollster._neaten_url(endpoint, t.id), ksclient.auth_token))
def login(request): """ Tries to login user and sets session data """ request.session.flush() #Process the form if there is a POST request. if (request.POST): form = LoginForm(request.POST) if form.is_valid(): username = form.cleaned_data['username'] password = form.cleaned_data['password'] try: auth_version = settings.SWIFT_AUTH_VERSION or 1 (storage_url, auth_token) = client.get_auth( settings.SWIFT_AUTH_URL, username, password, auth_version=auth_version) request.session['auth_token'] = auth_token request.session['storage_url'] = storage_url request.session['username'] = username request.session['password'] = password tenant_name, user = split_tenant_user_names(username) request.session['user'] = user request.session['tenant_name'] = tenant_name # Upon successful retrieval of a token, if we're unable to # head the account, then the user is not an admin or # swiftoperator and has access to only a container. try: client.head_account(storage_url, auth_token) except: request.session['norole'] = True return redirect(containerview) # Specify why the login failed. except client.ClientException, e: messages.error(request, _("Login failed: {0}".format( e))) # Generic login failure message. except Exception, e: print(e) messages.error(request, _("Login failed."))
def test_sync(self): all_objects = [] # upload some containers for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() body = 'test-body' client.put_object(self.url, self.token, container, obj, body) all_objects.append((policy, container, obj)) Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-object-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-bytes-used']), len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account) Manager(['account-reaper']).once() get_to_final_state() for policy, container, obj in all_objects: cpart, cnodes = self.container_ring.get_nodes( self.account, container) for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail('Found un-reaped /%s/%s on %r' % (self.account, container, node)) object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) for node in nodes: try: direct_get_object(node, part, self.account, container, obj) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy))
def iter_accounts(): ks = ksclient.Client(username=cfg.CONF.os_username, password=cfg.CONF.os_password, tenant_name=cfg.CONF.os_tenant_name, auth_url=cfg.CONF.os_auth_url) endpoint = ks.service_catalog.url_for(service_type='object-store', endpoint_type='adminURL') base_url = '%s/v1/%s' % (endpoint, cfg.CONF.reseller_prefix) for t in ks.tenants.list(): yield (t.id, swift.head_account('%s%s' % (base_url, t.id), ks.auth_token))
def get_tempurl_key(): (storage_url, auth_token) = client.get_auth(settings.SWIFT_AUTH_URL, settings.SWIFT_USER, settings.SWIFT_PASSWORD) meta = client.head_account(storage_url, auth_token) key = meta.get("x-account-meta-temp-url-key") if not key: key = random_key() headers = {"x-account-meta-temp-url-key": key} client.post_account(storage_url, auth_token, headers) return storage_url, key
def _get_account_info(self, ksclient, cache): try: endpoint = ksclient.service_catalog.url_for( service_type='object-store', endpoint_type='adminURL') except exceptions.EndpointNotFound: LOG.debug(_("Swift endpoint not found")) raise StopIteration() for t in cache['tenants']: yield (t.id, swift.head_account(self._neaten_url(endpoint, t.id), ksclient.auth_token))
def _get_account_info(self, ksclient, cache): try: endpoint = ksclient.service_catalog.url_for( service_type='object-store', endpoint_type=cfg.CONF.service_credentials.os_endpoint_type) except exceptions.EndpointNotFound: LOG.debug(_("Swift endpoint not found")) raise StopIteration() for t in cache[self.CACHE_KEY_TENANT]: yield (t.id, swift.head_account(self._neaten_url(endpoint, t.id), ksclient.auth_token))
def _get_account_info(self, ksclient, cache): try: endpoint = ksclient.service_catalog.url_for( service_type='object-store', endpoint_type=cfg.CONF.service_credentials.os_endpoint_type) except exceptions.EndpointNotFound: LOG.debug(_("Swift endpoint not found")) raise StopIteration() for t in cache['tenants']: yield (t.id, swift.head_account(self._neaten_url(endpoint, t.id), ksclient.auth_token))
def iter_accounts(ksclient): try: endpoint = ksclient.service_catalog.url_for( service_type='object-store', endpoint_type='adminURL') except exceptions.EndpointNotFound: LOG.debug(_("Swift endpoint not found")) return base_url = '%s/v1/%s' % (endpoint, cfg.CONF.reseller_prefix) for t in ksclient.tenants.list(): yield (t.id, swift.head_account('%s%s' % (base_url, t.id), ksclient.auth_token))
def iter_accounts(ksclient): try: endpoint = ksclient.service_catalog.url_for( service_type='object-store', endpoint_type='adminURL') except exceptions.EndpointNotFound: LOG.debug(_("Swift endpoint not found")) return for t in ksclient.tenants.list(): yield (t.id, swift.head_account( SwiftPollster._neaten_url(endpoint, t.id), ksclient.auth_token))
def get_swift_usage(tenant_url, token): usage = {} usage['gb_allocated'] = usage['gb_used'] = 0 try: swift_account = swiftclient.head_account(url=tenant_url, token=token) if SWIFT_QUOTA_KEY in swift_account: usage['gb_allocated'] = b_to_gb(swift_account[SWIFT_QUOTA_KEY]) usage['gb_used'] = b_to_gb(swift_account['x-account-bytes-used']) except ClientException: pass return usage
def get_fine_grained_temp_key(storage_url, auth_token, container_name=None): """ Tries to get meta-temp-url key from account or container. If not set, generate tempurl and save it. """ logging.debug(' in get_fine_grained_temp_key: container_name:%s, \ storage_url:%s ' % (container_name, storage_url) ) try: if container_name: container = client.head_container(storage_url, auth_token, container_name) key = container.get('x-container-meta-temp-url-key') logging.debug(' key in get_fine_grained_temp_key container: %s ' % key) else: account = client.head_account(storage_url, auth_token) key = account.get('x-account-meta-temp-url-key') logging.debug(' key in get_fine_grained_temp_key account: %s ' % key) except client.ClientException: return None # logging.debug(' account or container in get_temp_key: %s ' # % account or container) if not key: chars = string.ascii_lowercase + string.digits key = ''.join(random.choice(chars) for x in range(32)) if container_name: headers = {'x-container-meta-temp-url-key': key} try: client.post_container(storage_url, auth_token, container_name, headers) logging.debug(' post_container') except client.ClientException: return None raise ValueError('cannot get key, have no account rights to \ get account key!') else: headers = {'x-account-meta-temp-url-key': key} try: client.post_account(storage_url, auth_token, headers) logging.debug(' post_account') except client.ClientException: return None return key
def get_account(self, deep=True): if not self.http_conn: self.connect() account_info = swift.head_account(url=self.swift_url, token=self.token, http_conn=self.http_conn) account_head, containers = swift.get_account(url=self.swift_url, token=self.token, http_conn=self.http_conn) if self.debug: print(account_info) print(account_head) for container in containers: print(container) print if deep: self.get_containers(containers)
def setUp(self): super(TestAccountReaper, self).setUp() self.all_objects = [] int_client = self.make_internal_client() # upload some containers body = b'test-body' for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() client.put_object(self.url, self.token, container, obj, body) self.all_objects.append((policy, container, obj)) # Also create some reserved names container = get_reserved_name('reserved', policy.name, str(uuid.uuid4())) int_client.create_container( self.account, container, headers={'X-Storage-Policy': policy.name}) obj = get_reserved_name('object', str(uuid.uuid4())) int_client.upload_object(BytesIO(body), self.account, container, obj) self.all_objects.append((policy, container, obj)) policy.load_ring('/etc/swift') Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(self.all_objects)) self.assertEqual(int(headers['x-account-object-count']), len(self.all_objects)) self.assertEqual(int(headers['x-account-bytes-used']), len(self.all_objects) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account)
def test_sync(self): all_objects = [] # upload some containers for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() body = 'test-body' client.put_object(self.url, self.token, container, obj, body) all_objects.append((policy, container, obj)) Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-object-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-bytes-used']), len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account) # run the reaper Manager(['account-reaper']).once() for policy, container, obj in all_objects: # verify that any container deletes were at same timestamp cpart, cnodes = self.container_ring.get_nodes( self.account, container) delete_times = set() for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEqual(err.http_status, 404) delete_time = err.http_headers.get( 'X-Backend-DELETE-Timestamp') # 'X-Backend-DELETE-Timestamp' confirms it was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: # Container replicas may not yet be deleted if we have a # policy with object replicas < container replicas, so # ignore successful HEAD. We'll check for all replicas to # be deleted again after running the replicators. pass self.assertEqual(1, len(delete_times), delete_times) # verify that all object deletes were at same timestamp object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) headers = {'X-Backend-Storage-Policy-Index': int(policy)} delete_times = set() for node in nodes: try: direct_get_object(node, part, self.account, container, obj, headers=headers) except ClientException as err: self.assertEqual(err.http_status, 404) delete_time = err.http_headers.get('X-Backend-Timestamp') # 'X-Backend-Timestamp' confirms obj was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy)) self.assertEqual(1, len(delete_times)) # run replicators and updaters self.get_to_final_state() for policy, container, obj in all_objects: # verify that ALL container replicas are now deleted cpart, cnodes = self.container_ring.get_nodes( self.account, container) delete_times = set() for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEqual(err.http_status, 404) delete_time = err.http_headers.get( 'X-Backend-DELETE-Timestamp') # 'X-Backend-DELETE-Timestamp' confirms it was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s on %r' % (self.account, container, cnode)) # sanity check that object state is still consistent... object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) headers = {'X-Backend-Storage-Policy-Index': int(policy)} delete_times = set() for node in nodes: try: direct_get_object(node, part, self.account, container, obj, headers=headers) except ClientException as err: self.assertEqual(err.http_status, 404) delete_time = err.http_headers.get('X-Backend-Timestamp') # 'X-Backend-Timestamp' confirms obj was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy)) self.assertEqual(1, len(delete_times))
def get_swift_quota(sc, tenant): tenant_url, token = get_swift_tenant_connection(sc, tenant.id) swift_account = swift_client.head_account(url=tenant_url, token=token) quota = convert_quota_to_gb(int(swift_account.get(SWIFT_QUOTA_KEY, -1))) return quota
def objectview(request, container, prefix=None): """ Returns list of all objects in current container. """ storage_url = request.session.get('storage_url', '') auth_token = request.session.get('auth_token', '') # Users with no role use container keys if request.session.get('norole'): key = request.session.get("keys")[container] # Regular users use account keys else: account = client.head_account(storage_url, auth_token) key = account.get('x-account-meta-temp-url-key', '') request.session['container'] = container request.session['prefix'] = prefix request.session["key"] = key try: meta = client.head_container( storage_url, auth_token, container, headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')}) except client.ClientException: messages.add_message(request, messages.ERROR, _("Access denied.")) return redirect(containerview) # Check CORS header - BASE_URL should be in there. Do not perform this # check for users with no role. No role users will not be accessing # containers in any way except for swiftbrowser. Hence their container has # the proper headers. Norole users are unable to set the header anyways. if meta.get( 'x-container-meta-access-control-allow-origin' ) != settings.BASE_URL and not request.session.get('norole', False): # Add CORS headers so user can upload to this container. headers = { 'X-Container-Meta-Access-Control-Expose-Headers': 'Access-Control-Allow-Origin', 'X-Container-Meta-Access-Control-Allow-Origin': settings.BASE_URL, } try: client.put_container( storage_url, auth_token, container, headers) except client.ClientException: messages.add_message(request, messages.ERROR, _( "Access denied, unable to set CORS header.")) return redirect(containerview) prefixes = prefix_list(prefix) base_url = get_base_url(request) account = storage_url.split('/')[-1] read_acl = meta.get('x-container-read', '').split(',') public = False required_acl = ['.r:*', '.rlistings'] # The swifturl is the URL that the browser can send posts to upload files. swift_url = storage_url + '/' + container + '/' swift_slo_url = storage_url + '/' + container + '_segments/' if prefix: swift_url += prefix swift_slo_url += prefix # Posts from the browser to swift need a valid signature that's created # from the tempurl key signature = create_formpost_signature(swift_url, key) slo_signature = create_formpost_signature(swift_slo_url, key) if [x for x in read_acl if x in required_acl]: public = True return render_to_response( "objectview.html", { 'swift_url': swift_url, 'swift_slo_url': swift_slo_url, 'signature': signature, 'slo_signature': slo_signature, 'container': container, 'session': request.session, 'prefix': prefix, 'prefixes': prefixes, 'base_url': base_url, 'account': account, 'public': public, 'max_file_size': 5368709120, 'max_file_count': 1, 'expires': int(time.time() + 60 * 60 * 2), }, context_instance=RequestContext(request) )
def test_sync(self): all_objects = [] # upload some containers for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() body = 'test-body' client.put_object(self.url, self.token, container, obj, body) all_objects.append((policy, container, obj)) Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-object-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-bytes-used']), len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account) # run the reaper Manager(['account-reaper']).once() for policy, container, obj in all_objects: # verify that any container deletes were at same timestamp cpart, cnodes = self.container_ring.get_nodes( self.account, container) delete_times = set() for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEquals(err.http_status, 404) delete_time = err.http_headers.get( 'X-Backend-DELETE-Timestamp') # 'X-Backend-DELETE-Timestamp' confirms it was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: # Container replicas may not yet be deleted if we have a # policy with object replicas < container replicas, so # ignore successful HEAD. We'll check for all replicas to # be deleted again after running the replicators. pass self.assertEqual(1, len(delete_times), delete_times) # verify that all object deletes were at same timestamp object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) headers = {'X-Backend-Storage-Policy-Index': int(policy)} delete_times = set() for node in nodes: try: direct_get_object(node, part, self.account, container, obj, headers=headers) except ClientException as err: self.assertEquals(err.http_status, 404) delete_time = err.http_headers.get('X-Backend-Timestamp') # 'X-Backend-Timestamp' confirms obj was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy)) self.assertEqual(1, len(delete_times)) # run replicators and updaters self.get_to_final_state() for policy, container, obj in all_objects: # verify that ALL container replicas are now deleted cpart, cnodes = self.container_ring.get_nodes( self.account, container) delete_times = set() for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEquals(err.http_status, 404) delete_time = err.http_headers.get( 'X-Backend-DELETE-Timestamp') # 'X-Backend-DELETE-Timestamp' confirms it was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s on %r' % (self.account, container, cnode)) # sanity check that object state is still consistent... object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) headers = {'X-Backend-Storage-Policy-Index': int(policy)} delete_times = set() for node in nodes: try: direct_get_object(node, part, self.account, container, obj, headers=headers) except ClientException as err: self.assertEquals(err.http_status, 404) delete_time = err.http_headers.get('X-Backend-Timestamp') # 'X-Backend-Timestamp' confirms obj was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy)) self.assertEqual(1, len(delete_times))
def swift_cloud_project_status(request): project_id = request.GET.get('project_id') if not project_id: raise Http404 try: keystone = Keystone(request) project = keystone.project_get(project_id) except Exception as err: log.exception(f"Keystone error: {err}") return render(request, "vault/swift_cloud/project_status.html", { "project": None, "error": "Keystone Error" }) sct_client = SCTClient(settings.SWIFT_CLOUD_TOOLS_URL, settings.SWIFT_CLOUD_TOOLS_API_KEY) status = "" data = {} try: response = sct_client.transfer_get(project_id) if response.ok: data = response.json() else: status = _("Couldn't get migration data") except Exception as err: log.exception(f"Swift Cloud Tools Error: {err}") if response.status_code == 404: status = _("Migration not initialized yet") if response.status_code < 300: status = _("Waiting in migration queue") if data.get("initial_date") and not data.get("final_date"): status = _("Migrating...") if data.get("final_date"): status = _("Migration Completed") http_conn, storage_url = get_conn_and_storage_url(request, project_id) auth_token = request.session.get('auth_token') head_account = {} try: head_account = client.head_account(storage_url, auth_token, http_conn=http_conn) except Exception as err: log.exception(f'Exception: {err}') if head_account.get("x-account-meta-cloud-remove"): status = _("Marked for removal") return render( request, "vault/swift_cloud/project_status.html", { "project": project, "status": status, "metadata": json.dumps(head_account), "migration_data": json.dumps(data) })
def objectview(request, container, prefix=None): """ Returns list of all objects in current container. """ storage_url = request.session.get("storage_url", "") auth_token = request.session.get("auth_token", "") # Users with no role use container keys if request.session.get("norole"): container_object = client.head_container(storage_url, auth_token, container) key = container_object.get("x-container-meta-temp-url-key", "") # Regular users use account keys else: account = client.head_account(storage_url, auth_token) key = account.get("x-account-meta-temp-url-key", "") request.session["container"] = container request.session["prefix"] = prefix request.session["key"] = key redirect_url = get_base_url(request) redirect_url += reverse("objectview", kwargs={"container": container}) try: meta, objects = client.get_container(storage_url, auth_token, container, delimiter="/", prefix=prefix) except client.ClientException: messages.add_message(request, messages.ERROR, _("Access denied.")) return redirect(containerview) # Check CORS header - BASE_URL should be in there. Do not perform this # check for users with no role. No role users will not be accessing # containers in any way except for swiftbrowser. Hence their container has # the proper headers. Norole users are unable to set the header anyways. if meta.get("x-container-meta-access-control-allow-origin") != settings.BASE_URL and not request.session.get( "norole", False ): # Add CORS headers so user can upload to this container. headers = { "X-Container-Meta-Access-Control-Expose-Headers": "Access-Control-Allow-Origin", "X-Container-Meta-Access-Control-Allow-Origin": settings.BASE_URL, } try: client.put_container(storage_url, auth_token, container, headers) except client.ClientException: messages.add_message(request, messages.ERROR, _("Access denied, unable to set CORS header.")) return redirect(containerview) prefixes = prefix_list(prefix) pseudofolders, objs = pseudofolder_object_list(objects, prefix) base_url = get_base_url(request) account = storage_url.split("/")[-1] read_acl = meta.get("x-container-read", "").split(",") public = False required_acl = [".r:*", ".rlistings"] swift_url = storage_url + "/" + container + "/" swift_slo_url = storage_url + "/" + container + "_segments/" if prefix: swift_url += prefix swift_slo_url += prefix redirect_url += prefix signature = create_formpost_signature(swift_url, key) slo_signature = create_formpost_signature(swift_slo_url, key) if [x for x in read_acl if x in required_acl]: public = True return render_to_response( "objectview.html", { "swift_url": swift_url, "swift_slo_url": swift_slo_url, "signature": signature, "slo_signature": slo_signature, "redirect_url": redirect_url, "container": container, "objects": objs, "folders": pseudofolders, "session": request.session, "prefix": prefix, "prefixes": prefixes, "base_url": base_url, "account": account, "public": public, "max_file_size": 5368709120, "max_file_count": 1, "expires": int(time.time() + 60 * 60 * 2), }, context_instance=RequestContext(request), )
from swiftclient.client import get_account, head_account, post_account from swiftclient.exceptions import ClientException auth_url = 'http://10.111.1.123:5000/v2.0' account = 'test' user = '******' key = 'testing' try: acuser = "******" % (account, user) (storage_url, token) = get_auth(auth_url, acuser, key, auth_version='2.0') conn = http_connection(storage_url) req_headers = {'X-Account-Meta-Test1': 'aaabbbccc', 'X-Account-Meta-Test2': '0123456789'} resp_headers = {} post_account(storage_url, token, req_headers, response_dict=resp_headers, http_conn=conn) print "[Response headers of post_account()]" pp = pprint.PrettyPrinter(indent=2) pp.pprint(resp_headers) print "" headers = head_account(storage_url, token, http_conn=conn) print "[Response headers of head_account()]" pp.pprint(headers) except ClientException, e: print e