def containerview(request): """ Returns a list of all containers in current account. """ if not request.session.get('project_id'): messages.add_message(request, messages.ERROR, _('Select a project')) return HttpResponseRedirect(reverse('dashboard')) storage_url = get_endpoint(request, 'adminURL') auth_token = get_token_id(request) http_conn = client.http_connection(storage_url, insecure=settings.SWIFT_INSECURE) page = request.GET.get('page', 1) try: account_stat, containers = client.get_account(storage_url, auth_token, http_conn=http_conn) except client.ClientException as err: log.exception('{}{}'.format(_('Exception:').encode('UTF-8'), err)) messages.add_message(request, messages.ERROR, _('Unable to list containers')) account_stat = {} containers = [] containers = _hide_containers_with_prefixes(containers) account_stat = replace_hyphens(account_stat) context = utils.update_default_context(request, { 'account_stat': account_stat, 'containers': utils.generic_pagination(containers, page), }) return render_to_response('containerview.html', context, context_instance=RequestContext(request))
def get_temp_key(storage_url, auth_token, http_conn): """ Tries to get meta-temp-url key from account. If not set, generate tempurl and save it to account. This requires at least account owner rights. """ try: account = client.get_account(storage_url, auth_token, http_conn=http_conn) except client.ClientException: return None key = account[0].get('x-account-meta-temp-url-key') if not key: chars = string.ascii_lowercase + string.digits key = ''.join(random.choice(chars) for x in range(32)) headers = {'x-account-meta-temp-url-key': key} try: client.post_account(storage_url, token=auth_token, headers=headers, http_conn=http_conn) except client.ClientException: return None return key
def containerview(request): """ Returns a list of all containers in current account. """ storage_url = request.session.get('storage_url', '') auth_token = request.session.get('auth_token', '') try: account_stat, containers = client.get_account(storage_url, auth_token) except client.ClientException as exc: if exc.http_status == 403: account_stat = {} containers = [] base_url = get_base_url(request) msg = 'Container listing failed. You can manually choose a known ' msg += 'container by appending the name to the URL, for example: ' msg += '<a href="%s/objects/containername">' % base_url msg += '%s/objects/containername</a>' % base_url messages.add_message(request, messages.ERROR, msg) else: return redirect(login) account_stat = replace_hyphens(account_stat) return render( request, 'containerview.html', { 'account_stat': account_stat, 'containers': containers, 'session': request.session })
def get_temp_key(storage_url, auth_token, container): """ Tries to get meta-temp-url key from account. If not set, generate tempurl and save it to acocunt. This requires at least account owner rights. """ try: account = client.get_account(storage_url, auth_token) except client.ClientException: #Try to get the container temp url key instead try: container = client.get_container( storage_url, auth_token, container) return container[0].get('x-container-meta-temp-url-key') except client.ClientException: return None return None key = account[0].get('x-account-meta-temp-url-key') if not key: chars = string.ascii_lowercase + string.digits key = ''.join(random.choice(chars) for x in range(32)) headers = {'x-account-meta-temp-url-key': key} try: client.post_account(storage_url, auth_token, headers) except client.ClientException: return None return key
def test_one_node_fails(self): # Create container1 # Kill container1 servers excepting two of the primaries # Delete container1 # Restart other container1 primary server # Create container1/object1 (allowed because at least server thinks the # container exists) # Get to a final state # Assert all container1 servers indicate container1 is alive and # well with object1 # Assert account level also indicates container1 is alive and # well with object1 container1 = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnodes[0]['port'], self.port2server, self.pids) client.delete_container(self.url, self.token, container1) start_server(cnodes[0]['port'], self.port2server, self.pids) client.put_object(self.url, self.token, container1, 'object1', '123') get_to_final_state() for cnode in cnodes: self.assertEquals([ o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container1)[1] ], ['object1']) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '3')
def test_two_nodes_fail(self): # Create container1 # Kill container1 servers excepting one of the primaries # Delete container1 directly to the one primary still up # Restart other container1 servers # Get to a final state # Assert all container1 servers indicate container1 is gone (happens # because the one node that knew about the delete replicated to the # others.) # Assert account level also indicates container1 is gone container1 = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) cnp_port = kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnodes[0]['port'], self.port2server, self.pids) kill_server(cnodes[1]['port'], self.port2server, self.pids) direct_client.direct_delete_container(cnodes[2], cpart, self.account, container1) start_server(cnodes[0]['port'], self.port2server, self.pids) start_server(cnodes[1]['port'], self.port2server, self.pids) start_server(cnp_port, self.port2server, self.pids) get_to_final_state() for cnode in cnodes: exc = None try: direct_client.direct_get_container(cnode, cpart, self.account, container1) except client.ClientException as err: exc = err self.assertEquals(exc.http_status, 404) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '0') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0')
def containerview(request): """ Returns a list of all containers in current account. """ storage_url = request.session.get('storage_url', '') auth_token = request.session.get('auth_token', '') if not storage_url or not auth_token: return redirect(login) try: account_stat, containers = client.get_account(storage_url, auth_token) except client.ClientException as exc: traceback.print_exc() if exc.http_status == 403: account_stat = {} containers = [] base_url = get_base_url(request) msg = 'Container listing failed. You can manually choose a known ' msg += 'container by appending the name to the URL, for example: ' msg += '<a href="%s/objects/containername">' % base_url msg += '%s/objects/containername</a>' % base_url messages.add_message(request, messages.ERROR, msg) else: return redirect(login) account_stat = replace_hyphens(account_stat) return render_to_response('containerview.html', { 'account_stat': account_stat, 'containers': containers, 'session': request.session, }, context_instance=RequestContext(request))
def test_one_node_fails(self): # Create container1 # Kill container1 servers excepting two of the primaries # Delete container1 # Restart other container1 primary server # Create container1/object1 (allowed because at least server thinks the # container exists) # Get to a final state # Assert all container1 servers indicate container1 is alive and # well with object1 # Assert account level also indicates container1 is alive and # well with object1 container1 = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnodes[0]['port'], self.port2server, self.pids) client.delete_container(self.url, self.token, container1) start_server(cnodes[0]['port'], self.port2server, self.pids) client.put_object(self.url, self.token, container1, 'object1', '123') get_to_final_state() for cnode in cnodes: self.assertEquals( [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container1)[1]], ['object1']) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '3')
def test_first_two_nodes_fail(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) object1 = 'object1' client.put_object(self.url, self.token, container, object1, 'test') self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 in [o['name'] for o in client.get_container(self.url, self.token, container)[1]]) cpart, cnodes = self.container_ring.get_nodes(self.account, container) for x in xrange(2): kill(self.pids[self.port2server[cnodes[x]['port']]], SIGTERM) client.delete_object(self.url, self.token, container, object1) self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) self.assert_(object1 not in [o['name'] for o in client.get_container(self.url, self.token, container)[1]]) for x in xrange(2): self.pids[self.port2server[cnodes[x]['port']]] = \ Popen(['swift-container-server', '/etc/swift/container-server/%d.conf' % ((cnodes[x]['port'] - 6001) / 10)]).pid sleep(2) self.assert_(container in [c['name'] for c in client.get_account(self.url, self.token)[1]]) # This okay because the first node hasn't got the update that the # object was deleted yet. self.assert_(object1 in [o['name'] for o in direct_client.direct_get_container(cnodes[0], cpart, self.account, container)[1]]) # This fails because all three nodes have to indicate deletion before # we tell the user it worked. Since the first node 409s (it hasn't got # the update that the object was deleted yet), the whole must 503 # (until every is synced up, then the delete would work). exc = None try: client.delete_container(self.url, self.token, container) except client.ClientException, err: exc = err
def list_container(self): """ List all available containers. :return: list of container names. """ container_info = sc.get_account(self.storage_location, self.ks.auth_token) container_list = [container_info[1][count]['name'] for count in range(len(container_info[1]))] return container_list
def get_containers(storage_url, auth_token): containers = [] try: account_stat, containers = client.get_account(storage_url, auth_token) except ClientException as exc: logger.exception("Could not get containers") if exc.http_status == 403: logger.warn("Container listing failed") except: logger.exception("Could not get containers") return containers
def containers_list(request, project_id): if request.method == 'GET': token = get_token_connection(request) url = settings.SWIFT_URL + "/AUTH_" + project_id _, containers = swift_client.get_account(url, token) for c_id in reversed(range(len(containers))): if containers[c_id]['name'] in ('.dependency', '.storlet'): del containers[c_id] return JSONResponse(containers, status=status.HTTP_200_OK) return JSONResponse('Method ' + str(request.method) + ' not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def delete_account_cont(self, account_url, token): cnx = sclient.http_connection(account_url) al = sclient.get_account(None, token, http_conn=cnx, full_listing=True) for container in [c['name'] for c in al[1]]: ci = sclient.get_container(None, token, container, http_conn=cnx, full_listing=True) on = [od['name'] for od in ci[1]] for obj in on: sclient.delete_object('', token, container, obj, http_conn=cnx) sclient.delete_container('', token, container, http_conn=cnx)
def list_container(self): """ List all available containers. :return: list of container names. """ container_info = sc.get_account(self.storage_location, self.ks.auth_token) container_list = [ container_info[1][count]['name'] for count in range(len(container_info[1])) ] return container_list
def _assert_account_level(self, container_name, hdr_cont_count, hdr_obj_count, hdr_bytes, cont_count, cont_bytes): headers, containers = client.get_account(self.url, self.token) self.assertEqual(hdr_cont_count, headers['x-account-container-count']) self.assertEqual(hdr_obj_count, headers['x-account-object-count']) self.assertEqual(hdr_bytes, headers['x-account-bytes-used']) self.assertEqual(len(containers), 1) container = containers[0] self.assertEqual(container_name, container['name']) self.assertEqual(cont_count, container['count']) self.assertEqual(cont_bytes, container['bytes'])
def get_account_containers(storage_url, auth_token): """ List all containers in an account""" container_list = [] http_conn = client.http_connection(storage_url, insecure=settings.SWIFT_INSECURE) _, containers = client.get_account(storage_url, auth_token, http_conn=http_conn) for container in containers: container_list.append(container['name']) return container_list
def create_container_to_account(container_name, project_name, domain): ''' 校验account和manager,然后向指定的account下创建传入名称的container :param container_name: :param account: :param manager: :return: ''' status = '' message = '' data = {} resp = {"status": status, "data": data, "message": message} try: ainfo = db.session.query(SfoAccountManager).filter( and_(SfoAccountManager.account_stat == '1', SfoAccountManager.project_name == project_name, SfoAccountManager.domain == domain)).first() admin_user = SfoCofigureMethod.query_value_from_con_key('KEYSTONE_USERNAME') admin_password = SfoCofigureMethod.query_value_from_con_key('KEYSTONE_PASSWORD') if ainfo: stg_url, stg_token = stg_authenticate(account=project_name, user=admin_user.config_value, passwd=admin_password.config_value, domain=domain, admin=True) account_stat, container_list = swclient.get_account(stg_url, stg_token) if container_name: if len(container_list) > 0: has_ctn_list = map(lambda x: x['name'], container_list) if container_name in has_ctn_list: raise ValueError('Container %s exists ' % container_name ) requrl = stg_url + '/' + container_name crec = requests.put(requrl, headers={"X-Auth-Token": stg_token}) if crec.status_code <= 202: status = 200 message = 'OK' data['url'] = requrl else: status = 503 message = 'Container create failed' else: status = 404 message = "Not Found Record" except Exception as ex: status = 501 message = str(ex) finally: resp.update({"status": status, "data": data, "message": message}) return resp, status
def get_account(self, deep=True): if not self.http_conn: self.connect() account_info = swift.head_account(url=self.swift_url, token=self.token, http_conn=self.http_conn) account_head, containers = swift.get_account(url=self.swift_url, token=self.token, http_conn=self.http_conn) if self.debug: print(account_info) print(account_head) for container in containers: print(container) print if deep: self.get_containers(containers)
def test_two_nodes_fail(self): # Create container1 container1 = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) # Kill container1 servers excepting one of the primaries cnp_ipport = kill_nonprimary_server(cnodes, self.ipport2server, self.pids) kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server, self.pids) kill_server((cnodes[1]['ip'], cnodes[1]['port']), self.ipport2server, self.pids) # Delete container1 directly to the one primary still up direct_client.direct_delete_container(cnodes[2], cpart, self.account, container1) # Restart other container1 servers start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server, self.pids) start_server((cnodes[1]['ip'], cnodes[1]['port']), self.ipport2server, self.pids) start_server(cnp_ipport, self.ipport2server, self.pids) # Get to a final state self.get_to_final_state() # Assert all container1 servers indicate container1 is gone (happens # because the one node that knew about the delete replicated to the # others.) for cnode in cnodes: try: direct_client.direct_get_container(cnode, cpart, self.account, container1) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Assert account level also indicates container1 is gone headers, containers = client.get_account(self.url, self.token) self.assertEqual(headers['x-account-container-count'], '0') self.assertEqual(headers['x-account-object-count'], '0') self.assertEqual(headers['x-account-bytes-used'], '0')
def containerview(request): """ Returns a list of all containers in current account. """ storage_url = request.session.get('storage_url', '') auth_token = request.session.get('auth_token', '') try: account_stat, containers = client.get_account(storage_url, auth_token) except client.ClientException: return redirect(login) account_stat = replace_hyphens(account_stat) return render_to_response('containerview.html', { 'account_stat': account_stat, 'containers': containers, 'session': request.session, }, context_instance=RequestContext(request))
def handle_uploaded_file(request): f = request.FILES['file'] # with open('E:/sample.txt', 'wb+') as destination: # for chunk in f.chunks(): # destination.write(chunk) token = request.session.get('token') tenant_id = request.session.get('tenant_id') print 'token:%s***tenant:%s' % (token, tenant_id) url = 'http://192.168.0.55:8888/v1/AUTH_' + str(tenant_id) container = 'Publish' containers = c.get_account(url, token)[1] print containers if get_container(containers, container): pass else: print 'put container %s' % (container) c.put_container(url, token, container) c.put_object(url, token, container, name=f.name, contents=f)
def handle_uploaded_file(request): f = request.FILES["file"] # with open('E:/sample.txt', 'wb+') as destination: # for chunk in f.chunks(): # destination.write(chunk) token = request.session.get("token") tenant_id = request.session.get("tenant_id") print "token:%s***tenant:%s" % (token, tenant_id) url = "http://192.168.0.55:8888/v1/AUTH_" + str(tenant_id) container = "Publish" containers = c.get_account(url, token)[1] print containers if get_container(containers, container): pass else: print "put container %s" % (container) c.put_container(url, token, container) c.put_object(url, token, container, name=f.name, contents=f)
def containers(request): if 'auth_token' not in request.session.keys(): logger.info('No auth_token, attempting to authenticate') try: auth_version = settings.SWIFT_AUTH_VERSION or 1 conn = client.Connection(authurl=settings.SWIFT_AUTH_URL, user=settings.SWIFT_AUTH_USER, key=settings.SWIFT_AUTH_KEY, auth_version=auth_version, insecure=settings.SWIFT_SSL_INSECURE) (storage_url, auth_token) = conn.get_auth() logger.info('auth_token: %s' % auth_token) logger.info('storage_url: %s' % storage_url) request.session['storage_url'] = storage_url request.session['auth_token'] = auth_token except client.ClientException: messages.add_message(request, messages.ERROR, "Login failed.") auth_token = request.session['auth_token'] storage_url = request.session['storage_url'] try: http_conn = (urlparse(storage_url), client.HTTPConnection(storage_url, insecure=settings.SWIFT_SSL_INSECURE)) account_stat, containers = client.get_account(storage_url, auth_token, http_conn=http_conn) except client.ClientException as exc: if exc.http_status == 403: account_stat = {} containers = [] msg = 'Container listing failed.' messages.add_message(request, messages.ERROR, msg) else: request.session.flush() return redirect(settings.LOGIN_URL) account_stat = replace_hyphens(account_stat) return render(request,'containers.html', { 'account_stat': account_stat, 'containers': containers, 'session': request.session, })
def download_cloudstorage(request): ''' download data to cloud storage ''' token = request.session.get('token') tenant_id = request.session.get('tenant_id') print 'token:%s***tenant:%s' % (token, tenant_id) #url='http://192.168.0.55:8888/v1/AUTH_93e0ff98ddfb4bb28d936c049a89714b' url = 'http://192.168.0.55:8888/v1/AUTH_' + str(tenant_id) container = 'Subscriptions' containers = c.get_account(url, token)[1] print containers if get_container(containers, container): pass else: print 'put container %s' % (container) c.put_container(url, token, container) #TODO c.put_object(url, token, container, name='test222', contents='test') return HttpResponse('ok')
def download_cloudstorage(request): """ download data to cloud storage """ token = request.session.get("token") tenant_id = request.session.get("tenant_id") print "token:%s***tenant:%s" % (token, tenant_id) # url='http://192.168.0.55:8888/v1/AUTH_93e0ff98ddfb4bb28d936c049a89714b' url = "http://192.168.0.55:8888/v1/AUTH_" + str(tenant_id) container = "Subscriptions" containers = c.get_account(url, token)[1] print containers if get_container(containers, container): pass else: print "put container %s" % (container) c.put_container(url, token, container) # TODO c.put_object(url, token, container, name="test222", contents="test") return HttpResponse("ok")
def containerview(request): """ Returns a list of all containers in current account. """ if not request.session.get('project_id'): messages.add_message(request, messages.ERROR, _('Select a project')) return HttpResponseRedirect(reverse('dashboard')) storage_url = get_endpoint(request, 'adminURL') auth_token = get_token_id(request) http_conn = client.http_connection(storage_url, insecure=settings.SWIFT_INSECURE) page = request.GET.get('page', 1) try: account_stat, containers = client.get_account(storage_url, auth_token, http_conn=http_conn) except client.ClientException as err: log.exception('{}{}'.format(_('Exception:').encode('UTF-8'), err)) messages.add_message(request, messages.ERROR, _('Unable to list containers')) account_stat = {} containers = [] containers = _hide_containers_with_prefixes(containers) account_stat = replace_hyphens(account_stat) context = utils.update_default_context( request, { 'account_stat': account_stat, 'containers': utils.generic_pagination(containers, page), }) return render_to_response('containerview.html', context, context_instance=RequestContext(request))
def containerview(request): """ Returns a list of all containers in current account. """ storage_url = request.session.get('storage_url', settings.STORAGE_URL) auth_token = request.session.get('auth_token', 'demo') request.session['cloud'] = getcloud(request) request.session['showimagemap'] = settings.SHOW_IMAGEMAP try: account_stat, containers = client.get_account(storage_url, auth_token) except client.ClientException as exc: account_stat = {} containers = [] if exc.http_status == 403: base_url = get_base_url(request) msg = 'Container listing failed. You can manually choose a known ' msg += 'container by appending the name to the URL, for example: ' msg += '<a href="%s/objects/containername">' % base_url msg += '%s/objects/containername</a>' % base_url messages.add_message(request, messages.ERROR, msg) elif exc.http_status == 302: msg = 'Bucket listing failed with 302. \n ' msg += exc.msg messages.add_message(request, messages.ERROR, msg) else: msg = 'Bucket listing failed with %s. ' % exc.http_status msg += 'A 503 error could mean volume is not set.\n ' msg += exc.msg messages.add_message(request, messages.ERROR, msg) return redirect(login) account_stat = replace_hyphens(account_stat) return render_to_response('containerview.html', { 'account_stat': account_stat, 'containers': containers, 'session': request.session, }, context_instance=RequestContext(request))
def test_one_node_fails(self): # Create container1 container1 = "container-%s" % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) # Kill container1 servers excepting two of the primaries kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnodes[0]["port"], self.port2server, self.pids) # Delete container1 client.delete_container(self.url, self.token, container1) # Restart other container1 primary server start_server(cnodes[0]["port"], self.port2server, self.pids) # Create container1/object1 (allowed because at least server thinks the # container exists) client.put_object(self.url, self.token, container1, "object1", "123") # Get to a final state self.get_to_final_state() # Assert all container1 servers indicate container1 is alive and # well with object1 for cnode in cnodes: self.assertEquals( [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container1)[1]], ["object1"], ) # Assert account level also indicates container1 is alive and # well with object1 headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers["x-account-container-count"], "1") self.assertEquals(headers["x-account-object-count"], "1") self.assertEquals(headers["x-account-bytes-used"], "3")
def containerview(request): """ Returns a list of all containers in current account. """ # Users with no role will not be able to list containers. if request.session.get("norole"): # Redirect them to the container that is their username. return redirect(swiftbrowser.views.objects.objectview, request.session.get("user")) storage_url = request.session.get("storage_url", "") auth_token = request.session.get("auth_token", "") try: account_stat, containers = client.get_account(storage_url, auth_token) except client.ClientException: return redirect(login) account_stat = replace_hyphens(account_stat) account = storage_url.split("/")[-1] return render_to_response( "containerview.html", {"account": account, "account_stat": account_stat, "containers": containers, "session": request.session}, context_instance=RequestContext(request), )
def containerview(request): """ Returns a list of all containers in current account. """ # Redirect to limited_users_containerview if no role if ('norole' in request.session): return redirect(limited_users_containerview) storage_url = request.session.get('storage_url', '') auth_token = request.session.get('auth_token', '') try: account_stat, containers = client.get_account(storage_url, auth_token) except client.ClientException: return redirect(swiftbrowser.views.main.login) account_stat = replace_hyphens(account_stat) account = storage_url.split('/')[-1] return render_to_response('containerview.html', { 'account': account, 'account_stat': account_stat, 'containers': containers, 'session': request.session, }, context_instance=RequestContext(request))
def test_param_prefix(self): c.http_connection = self.fake_http_connection( 204, query_string="format=json&prefix=asdf/") c.get_account('http://www.test.com', 'asdf', prefix='asdf/')
def test_param_end_marker(self): c.http_connection = self.fake_http_connection( 204, query_string="format=json&end_marker=end_marker") c.get_account('http://www.test.com', 'asdf', end_marker='end_marker')
def test_main(self): # Create container1 and container2 container1 = 'container1' client.put_container(self.url, self.token, container1) container2 = 'container2' client.put_container(self.url, self.token, container2) # Assert account level sees them headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) self.assertTrue(found1) self.assertTrue(found2) # Create container2/object1 client.put_object(self.url, self.token, container2, 'object1', '1234') # Assert account level doesn't see it yet headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) self.assertTrue(found1) self.assertTrue(found2) # Get to final state self.get_to_final_state() # Assert account level now sees the container2/object1 headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assertTrue(found1) self.assertTrue(found2) apart, anodes = self.account_ring.get_nodes(self.account) kill_nonprimary_server(anodes, self.ipport2server, self.pids) kill_server((anodes[0]['ip'], anodes[0]['port']), self.ipport2server, self.pids) # Kill account servers excepting two of the primaries # Delete container1 client.delete_container(self.url, self.token, container1) # Put container2/object2 client.put_object(self.url, self.token, container2, 'object2', '12345') # Assert account level knows container1 is gone but doesn't know about # container2/object2 yet headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assertTrue(not found1) self.assertTrue(found2) # Run container updaters Manager(['container-updater']).once() # Assert account level now knows about container2/object2 headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 2) self.assertEquals(container['bytes'], 9) self.assertTrue(not found1) self.assertTrue(found2) # Restart other primary account server start_server((anodes[0]['ip'], anodes[0]['port']), self.ipport2server, self.pids) # Assert that server doesn't know about container1's deletion or the # new container2/object2 yet headers, containers = \ direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assertTrue(found1) self.assertTrue(found2) # Get to final state self.get_to_final_state() # Assert that server is now up to date headers, containers = \ direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 2) self.assertEquals(container['bytes'], 9) self.assertTrue(not found1) self.assertTrue(found2)
def account_info(): headers, containers = client.get_account(url=storageURL, token=authtoken) return headers, containers
def test_main(self): # Create container1 and container2 container1 = 'container1' client.put_container(self.url, self.token, container1) container2 = 'container2' client.put_container(self.url, self.token, container2) # Assert account level sees them headers, containers = client.get_account(self.url, self.token) self.assertEqual(headers['x-account-container-count'], '2') self.assertEqual(headers['x-account-object-count'], '0') self.assertEqual(headers['x-account-bytes-used'], '0') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEqual(container['count'], 0) self.assertEqual(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEqual(container['count'], 0) self.assertEqual(container['bytes'], 0) self.assertTrue(found1) self.assertTrue(found2) # Create container2/object1 client.put_object(self.url, self.token, container2, 'object1', '1234') # Assert account level doesn't see it yet headers, containers = client.get_account(self.url, self.token) self.assertEqual(headers['x-account-container-count'], '2') self.assertEqual(headers['x-account-object-count'], '0') self.assertEqual(headers['x-account-bytes-used'], '0') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEqual(container['count'], 0) self.assertEqual(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEqual(container['count'], 0) self.assertEqual(container['bytes'], 0) self.assertTrue(found1) self.assertTrue(found2) # Get to final state self.get_to_final_state() # Assert account level now sees the container2/object1 headers, containers = client.get_account(self.url, self.token) self.assertEqual(headers['x-account-container-count'], '2') self.assertEqual(headers['x-account-object-count'], '1') self.assertEqual(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEqual(container['count'], 0) self.assertEqual(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEqual(container['count'], 1) self.assertEqual(container['bytes'], 4) self.assertTrue(found1) self.assertTrue(found2) apart, anodes = self.account_ring.get_nodes(self.account) kill_nonprimary_server(anodes, self.ipport2server) kill_server((anodes[0]['ip'], anodes[0]['port']), self.ipport2server) # Kill account servers excepting two of the primaries # Delete container1 client.delete_container(self.url, self.token, container1) # Put container2/object2 client.put_object(self.url, self.token, container2, 'object2', '12345') # Assert account level knows container1 is gone but doesn't know about # container2/object2 yet headers, containers = client.get_account(self.url, self.token) self.assertEqual(headers['x-account-container-count'], '1') self.assertEqual(headers['x-account-object-count'], '1') self.assertEqual(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEqual(container['count'], 1) self.assertEqual(container['bytes'], 4) self.assertFalse(found1) self.assertTrue(found2) # Run container updaters Manager(['container-updater']).once() # Assert account level now knows about container2/object2 headers, containers = client.get_account(self.url, self.token) self.assertEqual(headers['x-account-container-count'], '1') self.assertEqual(headers['x-account-object-count'], '2') self.assertEqual(headers['x-account-bytes-used'], '9') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEqual(container['count'], 2) self.assertEqual(container['bytes'], 9) self.assertFalse(found1) self.assertTrue(found2) # Restart other primary account server start_server((anodes[0]['ip'], anodes[0]['port']), self.ipport2server) # Assert that server doesn't know about container1's deletion or the # new container2/object2 yet headers, containers = \ direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEqual(headers['x-account-container-count'], '2') self.assertEqual(headers['x-account-object-count'], '1') self.assertEqual(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEqual(container['count'], 1) self.assertEqual(container['bytes'], 4) self.assertTrue(found1) self.assertTrue(found2) # Get to final state self.get_to_final_state() # Assert that server is now up to date headers, containers = \ direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEqual(headers['x-account-container-count'], '1') self.assertEqual(headers['x-account-object-count'], '2') self.assertEqual(headers['x-account-bytes-used'], '9') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEqual(container['count'], 2) self.assertEqual(container['bytes'], 9) self.assertEqual(container['bytes'], 9) self.assertFalse(found1) self.assertTrue(found2)
def list_containers(self): containers = swift_client.get_account(self.endpoint, self.token)[1] return (cont['name'] for cont in containers)
def get_account_detail(self, account_id, token, s_type): cnx = self.get_cnx(account_id, s_type) return sclient.get_account(None, token, http_conn=cnx, full_listing=True)
def get_account_info(self): return swift_client.get_account(self.storage_url, self.token)
def test_param_limit(self): c.http_connection = self.fake_http_connection( 204, query_string="format=json&limit=10") c.get_account('http://www.test.com', 'asdf', limit=10)
def test_main(self): # Create container1 and container2 # Assert account level sees them # Create container2/object1 # Assert account level doesn't see it yet # Get to final state # Assert account level now sees the container2/object1 # Kill account servers excepting two of the primaries # Delete container1 # Assert account level knows container1 is gone but doesn't know about # container2/object2 yet # Put container2/object2 # Run container updaters # Assert account level now knows about container2/object2 # Restart other primary account server # Assert that server doesn't know about container1's deletion or the # new container2/object2 yet # Get to final state # Assert that server is now up to date container1 = 'container1' client.put_container(self.url, self.token, container1) container2 = 'container2' client.put_container(self.url, self.token, container2) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) self.assert_(found1) self.assert_(found2) client.put_object(self.url, self.token, container2, 'object1', '1234') headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) self.assert_(found1) self.assert_(found2) get_to_final_state() headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(found1) self.assert_(found2) apart, anodes = self.account_ring.get_nodes(self.account) kill_nonprimary_server(anodes, self.port2server, self.pids) kill_server(anodes[0]['port'], self.port2server, self.pids) client.delete_container(self.url, self.token, container1) client.put_object(self.url, self.token, container2, 'object2', '12345') headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(not found1) self.assert_(found2) processes = [] for node in xrange(1, 5): processes.append( Popen([ 'swift-container-updater', '/etc/swift/container-server/%d.conf' % node, 'once' ])) for process in processes: process.wait() headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 2) self.assertEquals(container['bytes'], 9) self.assert_(not found1) self.assert_(found2) start_server(anodes[0]['port'], self.port2server, self.pids) headers, containers = \ direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(found1) self.assert_(found2) get_to_final_state() headers, containers = \ direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 2) self.assertEquals(container['bytes'], 9) self.assert_(not found1) self.assert_(found2)
def test_no_content(self): c.http_connection = self.fake_http_connection(204) value = c.get_account('http://www.test.com', 'asdf')[1] self.assertEquals(value, [])
def test_main(self): container1 = 'container1' client.put_container(self.url, self.token, container1) container2 = 'container2' client.put_container(self.url, self.token, container2) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for c in containers: if c['name'] == container1: found1 = True self.assertEquals(c['count'], 0) self.assertEquals(c['bytes'], 0) elif c['name'] == container2: found2 = True self.assertEquals(c['count'], 0) self.assertEquals(c['bytes'], 0) self.assert_(found1) self.assert_(found2) client.put_object(self.url, self.token, container2, 'object1', '1234') headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for c in containers: if c['name'] == container1: found1 = True self.assertEquals(c['count'], 0) self.assertEquals(c['bytes'], 0) elif c['name'] == container2: found2 = True self.assertEquals(c['count'], 0) self.assertEquals(c['bytes'], 0) self.assert_(found1) self.assert_(found2) get_to_final_state() headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for c in containers: if c['name'] == container1: found1 = True self.assertEquals(c['count'], 0) self.assertEquals(c['bytes'], 0) elif c['name'] == container2: found2 = True self.assertEquals(c['count'], 1) self.assertEquals(c['bytes'], 4) self.assert_(found1) self.assert_(found2) apart, anodes = self.account_ring.get_nodes(self.account) kill(self.pids[self.port2server[anodes[0]['port']]], SIGTERM) client.delete_container(self.url, self.token, container1) client.put_object(self.url, self.token, container2, 'object2', '12345') headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for c in containers: if c['name'] == container1: found1 = True elif c['name'] == container2: found2 = True self.assertEquals(c['count'], 1) self.assertEquals(c['bytes'], 4) self.assert_(not found1) self.assert_(found2) ps = [] for n in xrange(1, 5): ps.append(Popen(['swift-container-updater', '/etc/swift/container-server/%d.conf' % n, 'once'])) for p in ps: p.wait() headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for c in containers: if c['name'] == container1: found1 = True elif c['name'] == container2: found2 = True self.assertEquals(c['count'], 2) self.assertEquals(c['bytes'], 9) self.assert_(not found1) self.assert_(found2) self.pids[self.port2server[anodes[0]['port']]] = \ Popen(['swift-account-server', '/etc/swift/account-server/%d.conf' % ((anodes[0]['port'] - 6002) / 10)]).pid sleep(2) # This is the earlier counts and bytes because the first node doesn't # have the newest udpates yet. headers, containers = \ direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for c in containers: if c['name'] == container1: found1 = True elif c['name'] == container2: found2 = True # This is the earlier count and bytes because the first node # doesn't have the newest udpates yet. self.assertEquals(c['count'], 1) self.assertEquals(c['bytes'], 4) # This okay because the first node hasn't got the update that # container1 was deleted yet. self.assert_(found1) self.assert_(found2) get_to_final_state() headers, containers = \ direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for c in containers: if c['name'] == container1: found1 = True elif c['name'] == container2: found2 = True self.assertEquals(c['count'], 2) self.assertEquals(c['bytes'], 9) self.assert_(not found1) self.assert_(found2)
class TestContainerFailures(TestCase): def setUp(self): (self.pids, self.port2server, self.account_ring, self.container_ring, self.object_ring, self.url, self.token, self.account) = reset_environment() def tearDown(self): kill_servers(self.port2server, self.pids) def test_one_node_fails(self): # Create container1 # Kill container1 servers excepting two of the primaries # Delete container1 # Restart other container1 primary server # Create container1/object1 (allowed because at least server thinks the # container exists) # Get to a final state # Assert all container1 servers indicate container1 is alive and # well with object1 # Assert account level also indicates container1 is alive and # well with object1 container1 = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnodes[0]['port'], self.port2server, self.pids) client.delete_container(self.url, self.token, container1) start_server(cnodes[0]['port'], self.port2server, self.pids) client.put_object(self.url, self.token, container1, 'object1', '123') get_to_final_state() for cnode in cnodes: self.assertEquals( [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container1)[1]], ['object1']) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '3') def test_two_nodes_fail(self): # Create container1 # Kill container1 servers excepting one of the primaries # Delete container1 directly to the one primary still up # Restart other container1 servers # Get to a final state # Assert all container1 servers indicate container1 is gone (happens # because the one node that knew about the delete replicated to the # others.) # Assert account level also indicates container1 is gone container1 = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) cnp_port = kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnodes[0]['port'], self.port2server, self.pids) kill_server(cnodes[1]['port'], self.port2server, self.pids) direct_client.direct_delete_container(cnodes[2], cpart, self.account, container1) start_server(cnodes[0]['port'], self.port2server, self.pids) start_server(cnodes[1]['port'], self.port2server, self.pids) start_server(cnp_port, self.port2server, self.pids) get_to_final_state() for cnode in cnodes: exc = None try: direct_client.direct_get_container(cnode, cpart, self.account, container1) except client.ClientException, err: exc = err self.assertEquals(exc.http_status, 404) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '0') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0')
def account_info(): headers, containers = client.get_account(url = storageURL, token = authtoken ) return headers, containers
def test_no_content(self): c.http_connection = self.fake_http_connection(204) value = c.get_account('http://www.test.com', 'asdf')[1] self.assertEqual(value, [])
def getMemberNames(self): _, containers = client.get_account(self.storage_url, self.auth_token, http_conn=self.http_connection) return [container['name'].encode("utf8") for container in containers]
def list_containers(self): containers = swift_client.get_account(self.endpoint, self.token, http_conn=self.http_conn)[1] return (cont['name'] for cont in containers)
def getMemberNames(self): _, containers = client.get_account( self.storage_url, self.auth_token, http_conn=self.http_connection) return [container['name'].encode("utf8") for container in containers]