def reset_environment():
call(['resetswift'])
pids = {}
try:
port2server = {}
for s, p in (('account', 6002), ('container', 6001), ('object', 6000)):
for n in xrange(1, 5):
pids['%s%d' % (s, n)] = \
Popen(['swift-%s-server' % s,
'/etc/swift/%s-server/%d.conf' % (s, n)]).pid
port2server[p + (n * 10)] = '%s%d' % (s, n)
pids['proxy'] = Popen(
['swift-proxy-server', '/etc/swift/proxy-server.conf']).pid
account_ring = Ring('/etc/swift/account.ring.gz')
container_ring = Ring('/etc/swift/container.ring.gz')
object_ring = Ring('/etc/swift/object.ring.gz')
attempt = 0
while True:
attempt += 1
try:
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'test:tester', 'testing')
account = url.split('/')[-1]
break
except Exception, err:
if attempt > 9:
print err
print 'Giving up after %s retries.' % attempt
raise err
print err
print 'Retrying in 2 seconds...'
sleep(2)
except BaseException, err:
kill_pids(pids)
raise err
def stg_authenticate(account, user, passwd, domain, admin=False):
'''
认证函数，如果认证成功，返回存储路径和token值
:param account: swift account账号, 在这就是系统编码
:param user: 系统编码对应的用户
:param passwd: 系统编码用户的密码
:param system_env: 系统编码的系统环境
:param admin （bool）: 管理员标志位
:return:
'''
try:
auth_account = user
auth_passwd = passwd
auth_version = config.swift_auth_version or 1
(storage_url,
auth_token) = swclient.get_auth(config.swift_auth_url,
auth_account,
auth_passwd,
auth_version=auth_version,
os_options={
'project_name':
account,
'project_domain_name':
domain,
'user_domain_name':
domain if not admin else "Default"
})
return (storage_url, auth_token)
except Exception as ex:
raise RuntimeError('swift authenticate fail ,the reason is %s' % ex)
def setUp(self):
resetswift()
try:
self.ipport2server = {}
self.configs = defaultdict(dict)
self.account_ring = get_ring(
'account',
self.acct_cont_required_replicas,
self.acct_cont_required_devices,
ipport2server=self.ipport2server,
config_paths=self.configs)
self.container_ring = get_ring(
'container',
self.acct_cont_required_replicas,
self.acct_cont_required_devices,
ipport2server=self.ipport2server,
config_paths=self.configs)
self.policy = get_policy(**self.policy_requirements)
self.object_ring = get_ring(
self.policy.ring_name,
self.obj_required_replicas,
self.obj_required_devices,
server='object',
ipport2server=self.ipport2server,
config_paths=self.configs)
self.servers_per_port = any(
int(readconf(c, section_name='object-replicator').get(
'servers_per_port', '0'))
for c in self.configs['object-replicator'].values())
Manager(['main']).start(wait=False)
for ipport in self.ipport2server:
check_server(ipport, self.ipport2server)
proxy_ipport = ('127.0.0.1', 8080)
self.ipport2server[proxy_ipport] = 'proxy'
self.url, self.token, self.account = check_server(
proxy_ipport, self.ipport2server)
self.account_1 = {
'url': self.url, 'token': self.token, 'account': self.account}
url2, token2 = get_auth(
'http://%s:%d/auth/v1.0' % proxy_ipport,
'test2:tester2', 'testing2')
self.account_2 = {
'url': url2, 'token': token2, 'account': url2.split('/')[-1]}
head_account(url2, token2) # sanity check
self.replicators = Manager(
['account-replicator', 'container-replicator',
'object-replicator'])
self.updaters = Manager(['container-updater', 'object-updater'])
except BaseException:
try:
raise
finally:
try:
Manager(['all']).kill()
except Exception:
pass
def authenticate(self, clone_conn=None):
if clone_conn:
self.conn_class = clone_conn.conn_class
self.storage_host = clone_conn.storage_host
self.storage_url = clone_conn.storage_url
self.storage_port = clone_conn.storage_port
self.storage_token = clone_conn.storage_token
return
if self.auth_version == "1":
auth_path = '%sv1.0' % (self.auth_prefix)
if self.account:
auth_user = '******' % (self.account, self.username)
else:
auth_user = self.username
else:
auth_user = self.username
auth_path = self.auth_prefix
auth_scheme = 'https://' if self.auth_ssl else 'http://'
auth_netloc = "%s:%d" % (self.auth_host, self.auth_port)
auth_url = auth_scheme + auth_netloc + auth_path
(storage_url, storage_token) = get_auth(
auth_url, auth_user, self.password, snet=False,
tenant_name=self.account, auth_version=self.auth_version,
os_options={})
if not (storage_url and storage_token):
raise AuthenticationFailed()
x = storage_url.split('/')
if x[0] == 'http:':
self.conn_class = httplib.HTTPConnection
self.storage_port = 80
elif x[0] == 'https:':
self.conn_class = httplib.HTTPSConnection
self.storage_port = 443
else:
raise ValueError('unexpected protocol %s' % (x[0]))
self.storage_host = x[2].split(':')[0]
if ':' in x[2]:
self.storage_port = int(x[2].split(':')[1])
# Make sure storage_url is a string and not unicode, since
# keystoneclient (called by swiftclient) returns them in
# unicode and this would cause troubles when doing
# no_safe_quote query.
self.storage_url = str('/%s/%s' % (x[3], x[4]))
self.account_name = str(x[4])
self.auth_user = auth_user
# With v2 keystone, storage_token is unicode.
# We want it to be string otherwise this would cause
# troubles when doing query with already encoded
# non ascii characters in its headers.
self.storage_token = str(storage_token)
self.user_acl = '%s:%s' % (self.account, self.username)
self.http_connect()
return self.storage_url, self.storage_token
def main():
options, commands = parser.parse_args()
commands.remove('split-brain')
if not commands:
parser.print_help()
return 'ERROR: must specify at least one command'
for cmd_args in commands:
cmd = cmd_args.split(':', 1)[0]
if cmd not in BrainSplitter.__commands__:
parser.print_help()
return 'ERROR: unknown command %s' % cmd
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'test:tester', 'testing')
brain = BrainSplitter(url, token, options.container, options.object)
for cmd_args in commands:
parts = cmd_args.split(':', 1)
command = parts[0]
if len(parts) > 1:
args = utils.list_from_csv(parts[1])
else:
args = ()
try:
brain.run(command, *args)
except ClientException as e:
print '**WARNING**: %s raised %s' % (command, e)
print 'STATUS'.join(['*' * 25] * 2)
brain.servers.status()
sys.exit()
def main():
options, commands = parser.parse_args()
if not commands:
parser.print_help()
return 'ERROR: must specify at least one command'
for cmd_args in commands:
cmd = cmd_args.split(':', 1)[0]
if cmd not in BrainSplitter.__commands__:
parser.print_help()
return 'ERROR: unknown command %s' % cmd
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0', 'test:tester',
'testing')
brain = BrainSplitter(url, token, options.container, options.object,
options.server_type)
for cmd_args in commands:
parts = cmd_args.split(':', 1)
command = parts[0]
if len(parts) > 1:
args = utils.list_from_csv(parts[1])
else:
args = ()
try:
brain.run(command, *args)
except ClientException as e:
print '**WARNING**: %s raised %s' % (command, e)
print 'STATUS'.join(['*' * 25] * 2)
brain.servers.status()
sys.exit()
def reset_environment():
call(['resetswift'])
pids = {}
try:
port2server = {}
for s, p in (('account', 6002), ('container', 6001), ('object', 6000)):
for n in xrange(1, 5):
pids['%s%d' % (s, n)] = \
Popen(['swift-%s-server' % s,
'/etc/swift/%s-server/%d.conf' % (s, n)]).pid
port2server[p + (n * 10)] = '%s%d' % (s, n)
pids['proxy'] = Popen(['swift-proxy-server',
'/etc/swift/proxy-server.conf']).pid
account_ring = Ring('/etc/swift/account.ring.gz')
container_ring = Ring('/etc/swift/container.ring.gz')
object_ring = Ring('/etc/swift/object.ring.gz')
attempt = 0
while True:
attempt += 1
try:
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'test:tester', 'testing')
account = url.split('/')[-1]
break
except Exception, err:
if attempt > 9:
print err
print 'Giving up after %s retries.' % attempt
raise err
print err
print 'Retrying in 2 seconds...'
sleep(2)
except BaseException, err:
kill_pids(pids)
raise err
def page_login(self, req):
""" create login page """
if req.method == 'POST':
try:
username = req.params_alt().get('username')
password = req.params_alt().get('password')
(storage_url, token) = get_auth(self.auth_url,
username, password,
auth_version=self.auth_version)
if self.token_bank.get(token, None):
self.token_bank[token].update({'url': storage_url,
'last': int(time())})
else:
self.token_bank[token] = {'url': storage_url,
'last': int(time())}
resp = HTTPFound(location=self.add_prefix(storage_url) + \
'?limit=%s' % self.items_per_page)
resp.set_cookie('_token', token, path=self.page_path,
max_age=self.cookie_max_age,
secure=self.secure)
self.memcache_update(token)
return resp
except Exception, err:
lang = self.get_lang(req)
resp = Response(charset='utf8')
resp.app_iter = self.tmpl({'ptype': 'login',
'top': self.page_path,
'title': self.title, 'lang': lang,
'message': 'Login Failed'})
return resp
def authenticate(self, clone_conn=None):
if clone_conn:
self.conn_class = clone_conn.conn_class
self.storage_netloc = clone_conn.storage_netloc
self.storage_url = clone_conn.storage_url
self.storage_token = clone_conn.storage_token
return
if self.auth_version == "1":
auth_path = '%sv1.0' % (self.auth_prefix)
if self.account:
auth_user = '******' % (self.account, self.username)
else:
auth_user = self.username
else:
auth_user = self.username
auth_path = self.auth_prefix
auth_scheme = 'https://' if self.auth_ssl else 'http://'
auth_netloc = "%s:%d" % (self.auth_host, self.auth_port)
auth_url = auth_scheme + auth_netloc + auth_path
authargs = dict(snet=False,
tenant_name=self.account,
auth_version=self.auth_version,
os_options={},
insecure=self.insecure)
(storage_url, storage_token) = get_auth(auth_url, auth_user,
self.password, **authargs)
if not (storage_url and storage_token):
raise AuthenticationFailed()
url = urllib.parse.urlparse(storage_url)
if url.scheme == 'http':
self.conn_class = http_client.HTTPConnection
elif url.scheme == 'https':
self.conn_class = http_client.HTTPSConnection
else:
raise ValueError('unexpected protocol %s' % (url.scheme))
self.storage_netloc = url.netloc
# Make sure storage_url is a string and not unicode, since
# keystoneclient (called by swiftclient) returns them in
# unicode and this would cause troubles when doing
# no_safe_quote query.
x = url.path.split('/')
self.storage_url = str('/%s/%s' % (x[1], x[2]))
self.account_name = str(x[2])
self.auth_user = auth_user
# With v2 keystone, storage_token is unicode.
# We want it to be string otherwise this would cause
# troubles when doing query with already encoded
# non ascii characters in its headers.
self.storage_token = str(storage_token)
self.user_acl = '%s:%s' % (self.account, self.username)
self.http_connect()
return self.storage_url, self.storage_token
def authenticate(self, clone_conn=None):
if clone_conn:
self.conn_class = clone_conn.conn_class
self.storage_host = clone_conn.storage_host
self.storage_url = clone_conn.storage_url
self.storage_port = clone_conn.storage_port
self.storage_token = clone_conn.storage_token
return
if self.auth_version == "1":
auth_path = '%sv1.0' % (self.auth_prefix)
if self.account:
auth_user = '******' % (self.account, self.username)
else:
auth_user = self.username
else:
auth_user = self.username
auth_path = self.auth_prefix
auth_scheme = 'https://' if self.auth_ssl else 'http://'
auth_netloc = "%s:%d" % (self.auth_host, self.auth_port)
auth_url = auth_scheme + auth_netloc + auth_path
(storage_url, storage_token) = get_auth(auth_url,
auth_user,
self.password,
snet=False,
tenant_name=self.account,
auth_version=self.auth_version,
os_options={})
if not (storage_url and storage_token):
raise AuthenticationFailed()
x = storage_url.split('/')
if x[0] == 'http:':
self.conn_class = httplib.HTTPConnection
self.storage_port = 80
elif x[0] == 'https:':
self.conn_class = httplib.HTTPSConnection
self.storage_port = 443
else:
raise ValueError('unexpected protocol %s' % (x[0]))
self.storage_host = x[2].split(':')[0]
if ':' in x[2]:
self.storage_port = int(x[2].split(':')[1])
# Make sure storage_url is a string and not unicode, since
# keystoneclient (called by swiftclient) returns them in
# unicode and this would cause troubles when doing
# no_safe_quote query.
self.storage_url = str('/%s/%s' % (x[3], x[4]))
self.account_name = str(x[4])
self.auth_user = auth_user
self.storage_token = storage_token
self.user_acl = '%s:%s' % (self.account, self.username)
self.http_connect()
return self.storage_url, self.storage_token
def authenticate(self, clone_conn=None):
if clone_conn:
self.conn_class = clone_conn.conn_class
self.storage_host = clone_conn.storage_host
self.storage_url = clone_conn.storage_url
self.storage_port = clone_conn.storage_port
self.storage_token = clone_conn.storage_token
return
if self.auth_version == "1":
auth_path = "%sv1.0" % (self.auth_prefix)
if self.account:
auth_user = "******" % (self.account, self.username)
else:
auth_user = self.username
else:
auth_user = self.username
auth_path = self.auth_prefix
auth_scheme = "https://" if self.auth_ssl else "http://"
auth_netloc = "%s:%d" % (self.auth_host, self.auth_port)
auth_url = auth_scheme + auth_netloc + auth_path
(storage_url, storage_token) = get_auth(
auth_url,
auth_user,
self.password,
snet=False,
tenant_name=self.account,
auth_version=self.auth_version,
os_options={},
)
if not (storage_url and storage_token):
raise AuthenticationFailed()
x = storage_url.split("/")
if x[0] == "http:":
self.conn_class = httplib.HTTPConnection
self.storage_port = 80
elif x[0] == "https:":
self.conn_class = httplib.HTTPSConnection
self.storage_port = 443
else:
raise ValueError("unexpected protocol %s" % (x[0]))
self.storage_host = x[2].split(":")[0]
if ":" in x[2]:
self.storage_port = int(x[2].split(":")[1])
# Make sure storage_url is a string and not unicode, since
# keystoneclient (called by swiftclient) returns them in
# unicode and this would cause troubles when doing
# no_safe_quote query.
self.storage_url = str("/%s/%s" % (x[3], x[4]))
self.storage_token = storage_token
self.http_connect()
return self.storage_url, self.storage_token
def __init__(self):
# get URL and token to use for requests
self.auth_info = SWIFT.get_auth('http://192.168.52.2:8080/auth/v1.0/','test:tester','testing')
self.swift_url = self.auth_info[0]
self.auth_token = self.auth_info[1]
self.user = getpass.getuser()
self.container_name = 'testdir'
def get_url_token(user_index, os_options):
authargs = dict(snet=False,
tenant_name=swift_test_tenant[user_index],
auth_version=swift_test_auth_version,
os_options=os_options,
insecure=insecure)
return get_auth(swift_test_auth, swift_test_user[user_index],
swift_test_key[user_index], **authargs)
def __init__(self, logger, conf, names):
self.logger = logger
self.aborted = False
self.user = conf.user
self.key = conf.key
self.auth_url = conf.auth
self.use_proxy = config_true_value(conf.use_proxy)
if not self.use_proxy and direct_client is None:
self.logger.critical("You need to have swift installed if you are "
"not using the proxy")
sys.exit(1)
self.auth_version = conf.auth_version
self.logger.info("Auth version: %s" % self.auth_version)
if self.use_proxy:
if using_http_proxy(self.auth_url):
logger.warn("Auth is going through HTTP proxy server. This "
"could affect test result")
url, token = client.get_auth(self.auth_url,
self.user,
self.key,
auth_version=self.auth_version)
self.token = token
self.account = url.split('/')[-1]
if conf.url == '':
self.url = url
else:
self.url = conf.url
else:
self.token = 'SlapChop!'
self.account = conf.account
self.url = conf.url
self.ip, self.port = self.url.split('/')[2].split(':')
if using_http_proxy(self.url):
logger.warn("Communication with Swift server is going through "
"HTTP proxy server. This could affect test result")
self.object_size = int(conf.object_size)
self.object_sources = conf.object_sources
self.lower_object_size = int(conf.lower_object_size)
self.upper_object_size = int(conf.upper_object_size)
self.files = []
if self.object_sources:
self.object_sources = self.object_sources.split()
self.files = [file(f, 'rb').read() for f in self.object_sources]
self.put_concurrency = int(conf.put_concurrency)
self.get_concurrency = int(conf.get_concurrency)
self.del_concurrency = int(conf.del_concurrency)
self.total_objects = int(conf.num_objects)
self.total_gets = int(conf.num_gets)
self.timeout = int(conf.timeout)
self.devices = conf.devices.split()
self.names = names
self.conn_pool = ConnectionPool(
self.url,
max(self.put_concurrency, self.get_concurrency,
self.del_concurrency))
def __init__(self):
self.last_headers_name = None
self.last_headers_value = None
# Get authentication token
self.storage_url, self.token = swiftclient.get_auth(
self.api_auth_url,
self.api_username,
self.api_key,
auth_version=self.auth_version,
os_options=dict(
list({
"tenant_id": self.tenant_id,
"tenant_name": self.tenant_name
}.items()) + list(self.os_extra_options.items())),
)
self.http_conn = swiftclient.http_connection(self.storage_url)
# Check container
try:
swiftclient.head_container(self.storage_url,
self.token,
self.container_name,
http_conn=self.http_conn)
except swiftclient.ClientException:
headers = {}
if self.auto_create_container:
if self.auto_create_container_public:
headers['X-Container-Read'] = '.r:*'
swiftclient.put_container(self.storage_url,
self.token,
self.container_name,
http_conn=self.http_conn,
headers=headers)
else:
raise ImproperlyConfigured("Container %s does not exist." %
self.container_name)
if self.auto_base_url:
# Derive a base URL based on the authentication information from
# the server, optionally overriding the protocol, host/port and
# potentially adding a path fragment before the auth information.
self.base_url = self.storage_url + '/'
if self.override_base_url is not None:
# override the protocol and host, append any path fragments
split_derived = urllib.parse.urlsplit(self.base_url)
split_override = urllib.parse.urlsplit(self.override_base_url)
split_result = [''] * 5
split_result[0:2] = split_override[0:2]
split_result[2] = (split_override[2] +
split_derived[2]).replace('//', '/')
self.base_url = urllib.parse.urlunsplit(split_result)
self.base_url = urllib.parse.urljoin(self.base_url,
self.container_name)
self.base_url += '/'
else:
self.base_url = self.override_base_url
def get_token(self):
if time() - self._token_creation_time >= self.auth_token_duration:
new_token = swiftclient.get_auth(self.api_auth_url,
self.api_username,
self.api_key,
auth_version=self.auth_version,
os_options=self.os_options)[1]
self.token = new_token
return self._token
def get_url_token(user_index, os_options):
authargs = dict(
snet=False,
tenant_name=swift_test_tenant[user_index],
auth_version=swift_test_auth_version,
os_options=os_options,
insecure=insecure,
)
return get_auth(swift_test_auth, swift_test_user[user_index], swift_test_key[user_index], **authargs)
def retry(func, *args, **kwargs):
"""
You can use the kwargs to override:
'retries' (default: 5)
'use_account' (default: 1) - which user's token to pass
'url_account' (default: matches 'use_account') - which user's storage URL
'resource' (default: url[url_account] - URL to connect to; retry()
will interpolate the variable :storage_url: if present
"""
global url, token, parsed, conn
retries = kwargs.get('retries', 5)
attempts, backoff = 0, 1
# use account #1 by default; turn user's 1-indexed account into 0-indexed
use_account = kwargs.pop('use_account', 1) - 1
# access our own account by default
url_account = kwargs.pop('url_account', use_account + 1) - 1
os_options = {
'user_domain_name': swift_test_domain[use_account],
'project_domain_name': swift_test_domain[use_account]
}
while attempts <= retries:
attempts += 1
try:
if not url[use_account] or not token[use_account]:
url[use_account], token[use_account] = \
get_auth(swift_test_auth, swift_test_user[use_account],
swift_test_key[use_account],
snet=False,
tenant_name=swift_test_tenant[use_account],
auth_version=swift_test_auth_version,
os_options=os_options)
parsed[use_account] = conn[use_account] = None
if not parsed[use_account] or not conn[use_account]:
parsed[use_account], conn[use_account] = \
connection(url[use_account])
# default resource is the account url[url_account]
resource = kwargs.pop('resource', '%(storage_url)s')
template_vars = {'storage_url': url[url_account]}
parsed_result = urlparse(resource % template_vars)
return func(url[url_account], token[use_account], parsed_result,
conn[url_account], *args, **kwargs)
except (socket.error, HTTPException):
if attempts > retries:
raise
parsed[use_account] = conn[use_account] = None
except AuthError:
url[use_account] = token[use_account] = None
continue
except InternalServerError:
pass
if attempts <= retries:
sleep(backoff)
backoff *= 2
raise Exception('No result after %s retries.' % retries)
def __init__(self, **settings):
# check if some of the settings provided as class attributes
# should be overwritten
for name, value in settings.items():
if hasattr(self, name):
setattr(self, name, value)
self.last_headers_name = None
self.last_headers_value = None
# Get authentication token
self.storage_url, self.token = swiftclient.get_auth(
self.api_auth_url,
self.api_username,
self.api_key,
auth_version=self.auth_version,
os_options={"tenant_name": self.tenant_name},
)
self.http_conn = swiftclient.http_connection(self.storage_url)
# Check container
try:
swiftclient.head_container(self.storage_url,
self.token,
self.container_name,
http_conn=self.http_conn)
except swiftclient.ClientException:
if self.auto_create_container:
swiftclient.put_container(self.storage_url,
self.token,
self.container_name,
http_conn=self.http_conn)
else:
raise ImproperlyConfigured("Container %s does not exist." %
self.container_name)
if self.auto_base_url:
# Derive a base URL based on the authentication information from
# the server, optionally overriding the protocol, host/port and
# potentially adding a path fragment before the auth information.
self.base_url = self.storage_url + '/'
if self.override_base_url is not None:
# override the protocol and host, append any path fragments
split_derived = urlparse.urlsplit(self.base_url)
split_override = urlparse.urlsplit(self.override_base_url)
split_result = [''] * 5
split_result[0:2] = split_override[0:2]
split_result[2] = (split_override[2] +
split_derived[2]).replace('//', '/')
self.base_url = urlparse.urlunsplit(split_result)
self.base_url = urlparse.urljoin(self.base_url,
self.container_name)
self.base_url += '/'
else:
self.base_url = self.override_base_url
def retry(func, *args, **kwargs):
"""
You can use the kwargs to override:
'retries' (default: 5)
'use_account' (default: 1) - which user's token to pass
'url_account' (default: matches 'use_account') - which user's storage URL
'resource' (default: url[url_account] - URL to connect to; retry()
will interpolate the variable :storage_url: if present
"""
global url, token, parsed, conn
retries = kwargs.get('retries', 5)
attempts, backoff = 0, 1
# use account #1 by default; turn user's 1-indexed account into 0-indexed
use_account = kwargs.pop('use_account', 1) - 1
# access our own account by default
url_account = kwargs.pop('url_account', use_account + 1) - 1
os_options = {'user_domain_name': swift_test_domain[use_account],
'project_domain_name': swift_test_domain[use_account]}
while attempts <= retries:
attempts += 1
try:
if not url[use_account] or not token[use_account]:
url[use_account], token[use_account] = \
get_auth(swift_test_auth, swift_test_user[use_account],
swift_test_key[use_account],
snet=False,
tenant_name=swift_test_tenant[use_account],
auth_version=swift_test_auth_version,
os_options=os_options)
parsed[use_account] = conn[use_account] = None
if not parsed[use_account] or not conn[use_account]:
parsed[use_account], conn[use_account] = \
connection(url[use_account])
# default resource is the account url[url_account]
resource = kwargs.pop('resource', '%(storage_url)s')
template_vars = {'storage_url': url[url_account]}
parsed_result = urlparse(resource % template_vars)
return func(url[url_account], token[use_account],
parsed_result, conn[url_account],
*args, **kwargs)
except (socket.error, HTTPException):
if attempts > retries:
raise
parsed[use_account] = conn[use_account] = None
except AuthError:
url[use_account] = token[use_account] = None
continue
except InternalServerError:
pass
if attempts <= retries:
sleep(backoff)
backoff *= 2
raise Exception('No result after %s retries.' % retries)
def __init__(self, logger, conf, names):
self.logger = logger
self.aborted = False
self.user = conf.user
self.key = conf.key
self.auth_url = conf.auth
self.use_proxy = config_true_value(conf.use_proxy)
if not self.use_proxy and direct_client is None:
self.logger.critical("You need to have swift installed if you are "
"not using the proxy")
sys.exit(1)
self.auth_version = conf.auth_version
self.logger.info("Auth version: %s" % self.auth_version)
if self.use_proxy:
if using_http_proxy(self.auth_url):
logger.warn("Auth is going through HTTP proxy server. This "
"could affect test result")
url, token = client.get_auth(self.auth_url, self.user, self.key,
auth_version=self.auth_version)
self.token = token
self.account = url.split('/')[-1]
if conf.url == '':
self.url = url
else:
self.url = conf.url
else:
self.token = 'SlapChop!'
self.account = conf.account
self.url = conf.url
self.ip, self.port = self.url.split('/')[2].split(':')
if using_http_proxy(self.url):
logger.warn("Communication with Swift server is going through "
"HTTP proxy server. This could affect test result")
self.object_size = int(conf.object_size)
self.object_sources = conf.object_sources
self.lower_object_size = int(conf.lower_object_size)
self.upper_object_size = int(conf.upper_object_size)
self.files = []
if self.object_sources:
self.object_sources = self.object_sources.split()
self.files = [file(f, 'rb').read() for f in self.object_sources]
self.put_concurrency = int(conf.put_concurrency)
self.get_concurrency = int(conf.get_concurrency)
self.del_concurrency = int(conf.del_concurrency)
self.total_objects = int(conf.num_objects)
self.total_gets = int(conf.num_gets)
self.timeout = int(conf.timeout)
self.devices = conf.devices.split()
self.names = names
self.conn_pool = ConnectionPool(self.url,
max(self.put_concurrency,
self.get_concurrency,
self.del_concurrency))
def get_token(self):
if time() - self._token_creation_time >= self.auth_token_duration:
new_token = swiftclient.get_auth(
self.api_auth_url,
self.api_username,
self.api_key,
auth_version=self.auth_version,
os_options={"tenant_name": self.tenant_name})[1]
self.token = new_token
return self._token
def get_storage_token(self):
""" Get a storage token for the swift cluster """
account = self.get_account_str()
try:
url, token = swiftclient.get_auth(self.authurl, account, self.key)
except swiftclient.client.ClientException as e:
raise SageFSException('HTTP Error: %s - %s'
% (e.http_status, e.http_reason))
self.storeurl = url
self.storetoken = token
def __init__(self):
self.last_headers_name = None
self.last_headers_value = None
# Get authentication token
self.storage_url, self.token = swiftclient.get_auth(
self.api_auth_url,
self.api_username,
self.api_key,
auth_version=self.auth_version,
os_options=dict({"tenant_name": self.tenant_name}.items() +
self.os_extra_options.items()),
cacert=self.custom_ca,
insecure=self.ssl_insecure
)
self.http_conn = swiftclient.http_connection(self.storage_url)
# Check container
try:
swiftclient.head_container(self.storage_url, self.token,
self.container_name,
http_conn=self.http_conn)
except swiftclient.ClientException:
headers = {}
if self.auto_create_container:
if self.auto_create_container_public:
headers['X-Container-Read'] = '.r:*'
swiftclient.put_container(self.storage_url, self.token,
self.container_name,
http_conn=self.http_conn,
headers=headers)
else:
raise ImproperlyConfigured(
"Container %s does not exist." % self.container_name)
if self.auto_base_url:
# Derive a base URL based on the authentication information from
# the server, optionally overriding the protocol, host/port and
# potentially adding a path fragment before the auth information.
self.base_url = self.storage_url + '/'
if self.override_base_url is not None:
# override the protocol and host, append any path fragments
split_derived = urlparse.urlsplit(self.base_url)
split_override = urlparse.urlsplit(self.override_base_url)
split_result = [''] * 5
split_result[0:2] = split_override[0:2]
split_result[2] = (split_override[2] +
split_derived[2]).replace('//', '/')
self.base_url = urlparse.urlunsplit(split_result)
self.base_url = urlparse.urljoin(self.base_url,
self.container_name)
self.base_url += '/'
else:
self.base_url = self.override_base_url
def __init__(self, **settings):
# check if some of the settings provided as class attributes
# should be overwritten
for name, value in settings.items():
if hasattr(self, name):
setattr(self, name, value)
self.last_headers_name = None
self.last_headers_value = None
# Get authentication token
self.storage_url, self.token = swiftclient.get_auth(
self.api_auth_url,
self.api_username,
self.api_key,
auth_version=self.auth_version,
os_options={"tenant_name": self.tenant_name},
)
self.http_conn = swiftclient.http_connection(self.storage_url)
# Check container
try:
swiftclient.head_container(self.storage_url, self.token,
self.container_name,
http_conn=self.http_conn)
except swiftclient.ClientException:
if self.auto_create_container:
swiftclient.put_container(self.storage_url, self.token,
self.container_name,
http_conn=self.http_conn)
else:
raise ImproperlyConfigured(
"Container %s does not exist." % self.container_name)
if self.auto_base_url:
# Derive a base URL based on the authentication information from
# the server, optionally overriding the protocol, host/port and
# potentially adding a path fragment before the auth information.
self.base_url = self.storage_url + '/'
if self.override_base_url is not None:
# override the protocol and host, append any path fragments
split_derived = urlparse.urlsplit(self.base_url)
split_override = urlparse.urlsplit(self.override_base_url)
split_result = [''] * 5
split_result[0:2] = split_override[0:2]
split_result[2] = (split_override[2] +
split_derived[2]).replace('//', '/')
self.base_url = urlparse.urlunsplit(split_result)
self.base_url = urlparse.urljoin(self.base_url,
self.container_name)
self.base_url += '/'
else:
self.base_url = self.override_base_url
def authenticate(self, clone_conn=None):
if clone_conn:
self.conn_class = clone_conn.conn_class
self.storage_host = clone_conn.storage_host
self.storage_url = clone_conn.storage_url
self.storage_port = clone_conn.storage_port
self.storage_token = clone_conn.storage_token
return
if self.auth_version == "1":
auth_path = '%sv1.0' % (self.auth_prefix)
if self.account:
auth_user = '******' % (self.account, self.username)
else:
auth_user = self.username
else:
auth_user = self.username
auth_path = self.auth_prefix
auth_scheme = 'https://' if self.auth_ssl else 'http://'
auth_netloc = "%s:%d" % (self.auth_host, self.auth_port)
auth_url = auth_scheme + auth_netloc + auth_path
(storage_url, storage_token) = get_auth(auth_url,
auth_user,
self.password,
snet=False,
tenant_name=self.account,
auth_version=self.auth_version,
os_options={})
if not (storage_url and storage_token):
raise AuthenticationFailed()
x = storage_url.split('/')
if x[0] == 'http:':
self.conn_class = httplib.HTTPConnection
self.storage_port = 80
elif x[0] == 'https:':
self.conn_class = httplib.HTTPSConnection
self.storage_port = 443
else:
raise ValueError('unexpected protocol %s' % (x[0]))
self.storage_host = x[2].split(':')[0]
if ':' in x[2]:
self.storage_port = int(x[2].split(':')[1])
self.storage_url = '/%s/%s' % (x[3], x[4])
self.storage_token = storage_token
self.http_connect()
return self.storage_url, self.storage_token
def get_auth_token():
"""
Get Authenticate token and Storage URL
Returns:
(token, storage_url)
"""
auth_url = "http://%s:8080/auth/v1.0" % PROXY_IP
url, token = client.get_auth(auth_url,
':'.join((ACCOUNT, USER)),
KEY)
return (token, url)
def authenticate(self, clone_conn=None):
if clone_conn:
self.conn_class = clone_conn.conn_class
self.storage_host = clone_conn.storage_host
self.storage_url = clone_conn.storage_url
self.storage_port = clone_conn.storage_port
self.storage_token = clone_conn.storage_token
return
if self.auth_version == "1":
auth_path = '%sv1.0' % (self.auth_prefix)
if self.account:
auth_user = '******' % (self.account, self.username)
else:
auth_user = self.username
else:
auth_user = self.username
auth_path = self.auth_prefix
auth_scheme = 'https://' if self.auth_ssl else 'http://'
auth_netloc = "%s:%d" % (self.auth_host, self.auth_port)
auth_url = auth_scheme + auth_netloc + auth_path
(storage_url, storage_token) = get_auth(auth_url,
auth_user, self.password,
snet=False,
tenant_name=self.account,
auth_version=self.auth_version,
os_options={})
if not (storage_url and storage_token):
raise AuthenticationFailed()
x = storage_url.split('/')
if x[0] == 'http:':
self.conn_class = httplib.HTTPConnection
self.storage_port = 80
elif x[0] == 'https:':
self.conn_class = httplib.HTTPSConnection
self.storage_port = 443
else:
raise ValueError('unexpected protocol %s' % (x[0]))
self.storage_host = x[2].split(':')[0]
if ':' in x[2]:
self.storage_port = int(x[2].split(':')[1])
self.storage_url = '/%s/%s' % (x[3], x[4])
self.storage_token = storage_token
self.http_connect()
return self.storage_url, self.storage_token
def __init__(self, logger, conf, names):
self.logger = logger
self.aborted = False
self.user = conf.user
self.key = conf.key
self.auth_url = conf.auth
self.use_proxy = conf.use_proxy.lower() in TRUE_VALUES
self.auth_version = conf.auth_version
self.logger.info("Auth version: %s" % self.auth_version)
if self.use_proxy:
url, token = client.get_auth(self.auth_url,
self.user,
self.key,
auth_version=self.auth_version)
self.token = token
self.account = url.split('/')[-1]
if conf.url == '':
self.url = url
else:
self.url = conf.url
else:
self.token = 'SlapChop!'
self.account = conf.account
self.url = conf.url
self.ip, self.port = self.url.split('/')[2].split(':')
self.containers = [
'%s_%d' % (conf.container_name, i)
for i in xrange(int(conf.num_containers))
]
self.object_size = int(conf.object_size)
self.object_sources = conf.object_sources
self.lower_object_size = int(conf.lower_object_size)
self.upper_object_size = int(conf.upper_object_size)
self.files = []
if self.object_sources:
self.object_sources = self.object_sources.split()
self.files = [file(f, 'rb').read() for f in self.object_sources]
self.put_concurrency = int(conf.put_concurrency)
self.get_concurrency = int(conf.get_concurrency)
self.del_concurrency = int(conf.del_concurrency)
self.total_objects = int(conf.num_objects)
self.total_gets = int(conf.num_gets)
self.timeout = int(conf.timeout)
self.devices = conf.devices.split()
self.names = names
self.conn_pool = ConnectionPool(
self.url,
max(self.put_concurrency, self.get_concurrency,
self.del_concurrency))
def _get_auth(self, user_name):
info = self.users.get(user_name)
if info is None:
return None, None
os_options = {'user_domain_name': info['domain'],
'project_domain_name': info['domain']}
authargs = dict(snet=False, tenant_name=info['account'],
auth_version=self.auth_version, os_options=os_options,
insecure=self.insecure)
storage_url, token = get_auth(
self.auth_url, user_name, info['password'], **authargs)
return storage_url, token
def _get_auth(self, user_name):
info = self.users.get(user_name)
if info is None:
return None, None
os_options = {"user_domain_name": info["domain"], "project_domain_name": info["domain"]}
authargs = dict(
snet=False,
tenant_name=info["account"],
auth_version=self.auth_version,
os_options=os_options,
insecure=self.insecure,
)
storage_url, token = get_auth(self.auth_url, user_name, info["password"], **authargs)
return storage_url, token
def authenticate(self):
if self.auth_version == "1":
auth_path = '%sv1.0' % (self.auth_prefix)
if self.account:
auth_user = '******' % (self.account, self.username)
else:
auth_user = self.username
else:
auth_user = self.username
auth_path = self.auth_prefix
auth_scheme = 'https://' if self.auth_ssl else 'http://'
auth_netloc = "%s:%d" % (self.auth_host, self.auth_port)
auth_url = auth_scheme + auth_netloc + auth_path
if self.insecure:
try:
import requests
from requests.packages.urllib3.exceptions import \
InsecureRequestWarning
except ImportError:
pass
else:
requests.packages.urllib3.disable_warnings(
InsecureRequestWarning)
authargs = dict(snet=False,
tenant_name=self.account,
auth_version=self.auth_version,
os_options={},
insecure=self.insecure)
(storage_url, storage_token) = get_auth(auth_url, auth_user,
self.password, **authargs)
if not (storage_url and storage_token):
raise AuthenticationFailed()
self.storage_url = storage_url
self.auth_user = auth_user
# With v2 keystone, storage_token is unicode.
# We want it to be string otherwise this would cause
# troubles when doing query with already encoded
# non ascii characters in its headers.
self.storage_token = str(storage_token)
self.user_acl = '%s:%s' % (self.account, self.username)
self.http_connect()
return self.storage_path, self.storage_token
def __init__(self, logger, conf, names):
self.logger = logger
self.aborted = False
self.user = conf.user
self.key = conf.key
self.auth_url = conf.auth
self.use_proxy = conf.use_proxy.lower() in TRUE_VALUES
self.auth_version = conf.auth_version
self.logger.info("Auth version: %s" % self.auth_version)
if self.use_proxy:
url, token = client.get_auth(self.auth_url, self.user, self.key,
auth_version=self.auth_version)
self.token = token
self.account = url.split('/')[-1]
if conf.url == '':
self.url = url
else:
self.url = conf.url
else:
self.token = 'SlapChop!'
self.account = conf.account
self.url = conf.url
self.ip, self.port = self.url.split('/')[2].split(':')
self.containers = ['%s_%d' % (conf.container_name, i)
for i in xrange(int(conf.num_containers))]
self.object_size = int(conf.object_size)
self.object_sources = conf.object_sources
self.lower_object_size = int(conf.lower_object_size)
self.upper_object_size = int(conf.upper_object_size)
self.files = []
if self.object_sources:
self.object_sources = self.object_sources.split()
self.files = [file(f, 'rb').read() for f in self.object_sources]
self.put_concurrency = int(conf.put_concurrency)
self.get_concurrency = int(conf.get_concurrency)
self.del_concurrency = int(conf.del_concurrency)
self.total_objects = int(conf.num_objects)
self.total_gets = int(conf.num_gets)
self.timeout = int(conf.timeout)
self.devices = conf.devices.split()
self.names = names
self.conn_pool = ConnectionPool(self.url,
max(self.put_concurrency, self.get_concurrency,
self.del_concurrency))
def check_server(ipport, ipport2server):
server = ipport2server[ipport]
if server[:-1] in ('account', 'container', 'object'):
if int(server[-1]) > 4:
return None
path = '/connect/1/2'
if server[:-1] == 'container':
path += '/3'
elif server[:-1] == 'object':
path += '/3/4'
try_until = time() + CHECK_SERVER_TIMEOUT
while True:
try:
conn = HTTPConnection(*ipport)
conn.request('GET', path)
resp = conn.getresponse()
# 404 because it's a nonsense path (and mount_check is false)
# 507 in case the test target is a VM using mount_check
if resp.status not in (404, 507):
raise Exception(
'Unexpected status %s' % resp.status)
break
except Exception as err:
if time() > try_until:
print(err)
print('Giving up on %s:%s after %s seconds.' % (
server, ipport, CHECK_SERVER_TIMEOUT))
raise err
sleep(0.1)
else:
try_until = time() + CHECK_SERVER_TIMEOUT
while True:
try:
url, token = get_auth('http://%s:%d/auth/v1.0' % ipport,
'test:tester', 'testing')
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
except Exception as err:
if time() > try_until:
print(err)
print('Giving up on proxy:8080 after 30 seconds.')
raise err
sleep(0.1)
return None
def authenticate(self):
if self.auth_version == "1" and self.account:
auth_user = '******' % (self.account, self.username)
else:
auth_user = self.username
if self.insecure:
try:
import requests
from requests.packages.urllib3.exceptions import \
InsecureRequestWarning
except ImportError:
pass
else:
requests.packages.urllib3.disable_warnings(
InsecureRequestWarning)
if self.domain:
os_opts = {
'project_domain_name': self.domain,
'user_domain_name': self.domain
}
else:
os_opts = {}
authargs = dict(snet=False,
tenant_name=self.account,
auth_version=self.auth_version,
os_options=os_opts,
insecure=self.insecure)
(storage_url, storage_token) = get_auth(self.auth_url, auth_user,
self.password, **authargs)
if not (storage_url and storage_token):
raise AuthenticationFailed()
self.storage_url = storage_url
self.auth_user = auth_user
# With v2 keystone, storage_token is unicode.
# We want it to be string otherwise this would cause
# troubles when doing query with already encoded
# non ascii characters in its headers.
self.storage_token = str(storage_token)
self.user_acl = '%s:%s' % (self.account, self.username)
self.http_connect()
return self.storage_path, self.storage_token
def check_server(port, port2server, pids, timeout=CHECK_SERVER_TIMEOUT):
server = port2server[port]
if server[:-1] in ('account', 'container', 'object'):
if int(server[-1]) > 4:
return None
path = '/connect/1/2'
if server[:-1] == 'container':
path += '/3'
elif server[:-1] == 'object':
path += '/3/4'
try_until = time() + timeout
while True:
try:
conn = HTTPConnection('127.0.0.1', port)
conn.request('GET', path)
resp = conn.getresponse()
# 404 because it's a nonsense path (and mount_check is false)
# 507 in case the test target is a VM using mount_check
if resp.status not in (404, 507):
raise Exception(
'Unexpected status %s' % resp.status)
break
except Exception as err:
if time() > try_until:
print err
print 'Giving up on %s:%s after %s seconds.' % (
server, port, timeout)
raise err
sleep(0.1)
else:
try_until = time() + timeout
while True:
try:
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'test:tester', 'testing')
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
except Exception as err:
if time() > try_until:
print err
print 'Giving up on proxy:8080 after 30 seconds.'
raise err
sleep(0.1)
return None
def retry(func, *args, **kwargs):
"""
You can use the kwargs to override the 'retries' (default: 5) and
'use_account' (default: 1).
"""
global url, token, parsed, conn
retries = kwargs.get('retries', 5)
use_account = 1
if 'use_account' in kwargs:
use_account = kwargs['use_account']
del kwargs['use_account']
use_account -= 1
attempts = 0
backoff = 1
while attempts <= retries:
attempts += 1
try:
if not url[use_account] or not token[use_account]:
url[use_account], token[use_account] = \
get_auth(swift_test_auth, swift_test_user[use_account],
swift_test_key[use_account],
snet=False,
tenant_name=swift_test_tenant[use_account],
auth_version=swift_test_auth_version,
os_options={})
parsed[use_account] = conn[use_account] = None
if not parsed[use_account] or not conn[use_account]:
parsed[use_account], conn[use_account] = \
http_connection(url[use_account])
return func(url[use_account], token[use_account],
parsed[use_account], conn[use_account], *args,
**kwargs)
except (socket.error, HTTPException):
if attempts > retries:
raise
parsed[use_account] = conn[use_account] = None
except AuthError:
url[use_account] = token[use_account] = None
continue
except InternalServerError:
pass
if attempts <= retries:
sleep(backoff)
backoff *= 2
raise Exception('No result after %s retries.' % retries)
def retry(func, *args, **kwargs):
"""
You can use the kwargs to override the 'retries' (default: 5) and
'use_account' (default: 1).
"""
global url, token, parsed, conn
retries = kwargs.get('retries', 5)
use_account = 1
if 'use_account' in kwargs:
use_account = kwargs['use_account']
del kwargs['use_account']
use_account -= 1
attempts = 0
backoff = 1
while attempts <= retries:
attempts += 1
try:
if not url[use_account] or not token[use_account]:
url[use_account], token[use_account] = \
get_auth(swift_test_auth, swift_test_user[use_account],
swift_test_key[use_account],
snet=False,
tenant_name=swift_test_tenant[use_account],
auth_version=swift_test_auth_version,
os_options={})
parsed[use_account] = conn[use_account] = None
if not parsed[use_account] or not conn[use_account]:
parsed[use_account], conn[use_account] = \
http_connection(url[use_account])
return func(url[use_account], token[use_account],
parsed[use_account], conn[use_account],
*args, **kwargs)
except (socket.error, HTTPException):
if attempts > retries:
raise
parsed[use_account] = conn[use_account] = None
except AuthError:
url[use_account] = token[use_account] = None
continue
except InternalServerError:
pass
if attempts <= retries:
sleep(backoff)
backoff *= 2
raise Exception('No result after %s retries.' % retries)
def authenticate(self):
if self.auth_version == "1":
auth_path = '%sv1.0' % (self.auth_prefix)
if self.account:
auth_user = '******' % (self.account, self.username)
else:
auth_user = self.username
else:
auth_user = self.username
auth_path = self.auth_prefix
auth_scheme = 'https://' if self.auth_ssl else 'http://'
auth_netloc = "%s:%d" % (self.auth_host, self.auth_port)
auth_url = auth_scheme + auth_netloc + auth_path
if self.insecure:
try:
import requests
from requests.packages.urllib3.exceptions import \
InsecureRequestWarning
except ImportError:
pass
else:
requests.packages.urllib3.disable_warnings(
InsecureRequestWarning)
authargs = dict(snet=False, tenant_name=self.account,
auth_version=self.auth_version, os_options={},
insecure=self.insecure)
(storage_url, storage_token) = get_auth(
auth_url, auth_user, self.password, **authargs)
if not (storage_url and storage_token):
raise AuthenticationFailed()
self.storage_url = storage_url
self.auth_user = auth_user
# With v2 keystone, storage_token is unicode.
# We want it to be string otherwise this would cause
# troubles when doing query with already encoded
# non ascii characters in its headers.
self.storage_token = str(storage_token)
self.user_acl = '%s:%s' % (self.account, self.username)
self.http_connect()
return self.storage_path, self.storage_token
def check_server(ipport, ipport2server, pids, timeout=CHECK_SERVER_TIMEOUT):
server = ipport2server[ipport]
if server[:-1] in ("account", "container", "object"):
if int(server[-1]) > 4:
return None
path = "/connect/1/2"
if server[:-1] == "container":
path += "/3"
elif server[:-1] == "object":
path += "/3/4"
try_until = time() + timeout
while True:
try:
conn = HTTPConnection(*ipport)
conn.request("GET", path)
resp = conn.getresponse()
# 404 because it's a nonsense path (and mount_check is false)
# 507 in case the test target is a VM using mount_check
if resp.status not in (404, 507):
raise Exception("Unexpected status %s" % resp.status)
break
except Exception as err:
if time() > try_until:
print(err)
print("Giving up on %s:%s after %s seconds." % (server, ipport, timeout))
raise err
sleep(0.1)
else:
try_until = time() + timeout
while True:
try:
url, token = get_auth("http://%s:%d/auth/v1.0" % ipport, "test:tester", "testing")
account = url.split("/")[-1]
head_account(url, token)
return url, token, account
except Exception as err:
if time() > try_until:
print(err)
print("Giving up on proxy:8080 after 30 seconds.")
raise err
sleep(0.1)
return None
def get_token(self):
if time() - self._token_creation_time >= self.auth_token_duration:
os_options = {
'tenant_id': self.tenant_id,
'tenant_name': self.tenant_name,
'user_domain_id': self.user_domain_id,
'user_domain_name': self.user_domain_name,
'project_domain_id': self.project_domain_id,
'project_domain_name': self.project_domain_name
}
os_options.update(self.os_extra_options)
new_token = swiftclient.get_auth(
self.api_auth_url,
self.api_username,
self.api_key,
auth_version=self.auth_version,
os_options=os_options,
)[1]
self.token = new_token
return self._token
def __init__(self, logger, conf, names):
self.logger = logger
self.aborted = False
self.user = conf.user
self.key = conf.key
self.auth_url = conf.auth
self.use_proxy = config_true_value(conf.use_proxy)
self.auth_version = conf.auth_version
self.logger.info("Auth version: %s" % self.auth_version)
if self.use_proxy:
url, token = client.get_auth(self.auth_url, self.user, self.key, auth_version=self.auth_version)
self.token = token
self.account = url.split("/")[-1]
if conf.url == "":
self.url = url
else:
self.url = conf.url
else:
self.token = "SlapChop!"
self.account = conf.account
self.url = conf.url
self.ip, self.port = self.url.split("/")[2].split(":")
self.object_size = int(conf.object_size)
self.object_sources = conf.object_sources
self.lower_object_size = int(conf.lower_object_size)
self.upper_object_size = int(conf.upper_object_size)
self.files = []
if self.object_sources:
self.object_sources = self.object_sources.split()
self.files = [file(f, "rb").read() for f in self.object_sources]
self.put_concurrency = int(conf.put_concurrency)
self.get_concurrency = int(conf.get_concurrency)
self.del_concurrency = int(conf.del_concurrency)
self.total_objects = int(conf.num_objects)
self.total_gets = int(conf.num_gets)
self.timeout = int(conf.timeout)
self.devices = conf.devices.split()
self.names = names
self.conn_pool = ConnectionPool(self.url, max(self.put_concurrency, self.get_concurrency, self.del_concurrency))
def page_login(self, req):
""" create login page """
if req.method == 'POST':
try:
username = req.params_alt().get('username')
password = req.params_alt().get('password')
(storage_url, token) = get_auth(self.auth_url,
username,
password,
auth_version=self.auth_version)
if self.token_bank.get(token, None):
self.token_bank[token].update({
'url': storage_url,
'last': int(time())
})
else:
self.token_bank[token] = {
'url': storage_url,
'last': int(time())
}
resp = HTTPFound(location=self.add_prefix(storage_url) + \
'?limit=%s' % self.items_per_page)
resp.set_cookie('_token',
token,
path=self.page_path,
max_age=self.cookie_max_age,
secure=self.secure)
self.memcache_update(token)
return resp
except Exception, err:
lang = self.get_lang(req)
resp = Response(charset='utf8')
resp.app_iter = self.tmpl({
'ptype': 'login',
'top': self.page_path,
'title': self.title,
'lang': lang,
'message': 'Login Failed'
})
return resp
def main():
options, commands = parser.parse_args()
if not commands:
parser.print_help()
return 'ERROR: must specify at least one command'
for cmd_args in commands:
cmd = cmd_args.split(':', 1)[0]
if cmd not in BrainSplitter.__commands__:
parser.print_help()
return 'ERROR: unknown command %s' % cmd
url, token = get_auth(PROXY_BASE_URL + '/auth/v1.0',
'test:tester', 'testing')
if options.server_type == 'object' and not options.policy_name:
options.policy_name = POLICIES.default.name
if options.policy_name:
options.server_type = 'object'
policy = POLICIES.get_by_name(options.policy_name)
if not policy:
return 'ERROR: unknown policy %r' % options.policy_name
else:
policy = None
brain = BrainSplitter(url, token, options.container, options.object,
options.server_type, policy=policy)
for cmd_args in commands:
parts = cmd_args.split(':', 1)
command = parts[0]
if len(parts) > 1:
args = utils.list_from_csv(parts[1])
else:
args = ()
try:
brain.run(command, *args)
except ClientException as e:
print('**WARNING**: %s raised %s' % (command, e))
print('STATUS'.join(['*' * 25] * 2))
brain.servers.status()
sys.exit()
def main():
options, commands = parser.parse_args()
if not commands:
parser.print_help()
return 'ERROR: must specify at least one command'
for cmd_args in commands:
cmd = cmd_args.split(':', 1)[0]
if cmd not in BrainSplitter.__commands__:
parser.print_help()
return 'ERROR: unknown command %s' % cmd
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'test:tester', 'testing')
if options.server_type == 'object' and not options.policy_name:
options.policy_name = POLICIES.default.name
if options.policy_name:
options.server_type = 'object'
policy = POLICIES.get_by_name(options.policy_name)
if not policy:
return 'ERROR: unknown policy %r' % options.policy_name
else:
policy = None
brain = BrainSplitter(url, token, options.container, options.object,
options.server_type, policy=policy)
for cmd_args in commands:
parts = cmd_args.split(':', 1)
command = parts[0]
if len(parts) > 1:
args = utils.list_from_csv(parts[1])
else:
args = ()
try:
brain.run(command, *args)
except ClientException as e:
print '**WARNING**: %s raised %s' % (command, e)
print 'STATUS'.join(['*' * 25] * 2)
brain.servers.status()
sys.exit()
def getAuth(self): return swiftclient.get_auth(self.authurl, self.user, self.key)
def _check_proxy(ipport, user, key):
url, token = get_auth('http://%s:%d/auth/v1.0' % ipport,
user, key)
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
def setUp(self):
super(TestAccountGetFakeResponsesMatch, self).setUp()
self.url, self.token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'admin:admin', 'admin')
def setUp(self):
super(TestAccountGetFakeResponsesMatch, self).setUp()
self.url, self.token = get_auth(PROXY_BASE_URL + '/auth/v1.0',
'admin:admin', 'admin')
def setUp(self):
(self.pids, self.port2server, self.account_ring, self.container_ring,
self.object_ring, self.policy, self.url, self.token,
self.account, self.configs) = reset_environment()
self.url, self.token = get_auth(
'http://127.0.0.1:8080/auth/v1.0', 'admin:admin', 'admin')
def __init__(self, **settings):
# check if some of the settings provided as class attributes
# should be overwritten
for name, value in settings.items():
if hasattr(self, name):
setattr(self, name, value)
validate_settings(self)
self.last_headers_name = None
self.last_headers_value = None
self.os_options = {
'tenant_id': self.tenant_id,
'tenant_name': self.tenant_name,
'user_domain_id': self.user_domain_id,
'user_domain_name': self.user_domain_name,
'project_domain_id': self.project_domain_id,
'project_domain_name': self.project_domain_name
}
self.os_options.update(self.os_extra_options)
# Get authentication token
self.storage_url, self.token = swiftclient.get_auth(
self.api_auth_url,
self.api_username,
self.api_key,
auth_version=self.auth_version,
os_options=self.os_options)
self.http_conn = swiftclient.http_connection(self.storage_url)
# Check container
try:
swiftclient.head_container(self.storage_url,
self.token,
self.container_name,
http_conn=self.http_conn)
except swiftclient.ClientException:
headers = {}
if self.auto_create_container:
if self.auto_create_container_public:
headers['X-Container-Read'] = '.r:*'
if self.auto_create_container_allow_orgin:
headers['X-Container-Meta-Access-Control-Allow-Origin'] = \
self.auto_create_container_allow_orgin
swiftclient.put_container(self.storage_url,
self.token,
self.container_name,
http_conn=self.http_conn,
headers=headers)
else:
raise ImproperlyConfigured(
"Container %s does not exist." % self.container_name)
if self.auto_base_url:
# Derive a base URL based on the authentication information from
# the server, optionally overriding the protocol, host/port and
# potentially adding a path fragment before the auth information.
self.base_url = self.storage_url + '/'
if self.override_base_url is not None:
# override the protocol and host, append any path fragments
split_derived = urlparse.urlsplit(self.base_url)
split_override = urlparse.urlsplit(self.override_base_url)
split_result = [''] * 5
split_result[0:2] = split_override[0:2]
split_result[2] = (split_override[2] + split_derived[2]
).replace('//', '/')
self.base_url = urlparse.urlunsplit(split_result)
self.base_url = urlparse.urljoin(self.base_url,
self.container_name)
self.base_url += '/'
else:
self.base_url = self.override_base_url
import swiftclient as SWIFT
import getpass
# get URL and token to use for requests
auth_info = SWIFT.get_auth('http://192.168.52.2:8080/auth/v1.0/','test:tester','testing')
swift_url = auth_info[0]
auth_token = auth_info[1]
user = getpass.getuser()
container_name = 'testdir'
obj_name = 'testobj'
contents = 'test content of testobj'
element_id = 23
# delete an object
#rv = SWIFT.delete_object(swift_url, auth_token, container_name, obj_name+str(13))
#print rv
# delete a container
#rv = SWIFT.delete_object(swift_url, auth_token, container='testdir')
#print rv
# create a container
"""
element_id += 1
container_meta = {'X-Container-Meta-ID' : element_id, 'X-Container-Meta-Creator' : user}
rv = SWIFT.put_container(swift_url, auth_token, container_name, headers=container_meta )
print 'put_container:', rv
"""
def _check_proxy(ipport, user, key):
url, token = get_auth('http://%s:%d/auth/v1.0' % ipport,
user, key)
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
if resp.status not in (404, 507):
raise Exception(
'Unexpected status %s' % resp.status)
break
except Exception, err:
if time() > try_until:
print err
print 'Giving up on %s:%s after %s seconds.' % (
server, port, timeout)
raise err
sleep(0.1)
else:
try_until = time() + timeout
while True:
try:
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'test:tester', 'testing')
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
except Exception, err:
if time() > try_until:
print err
print 'Giving up on proxy:8080 after 30 seconds.'
raise err
sleep(0.1)
return None
def kill_server(port, port2server, pids):
try:
kill(pids[port2server[port]], SIGTERM)
def generate_report(conf, policy_name=None):
global json_output
json_output = config_true_value(conf.get('dump_json', 'no'))
if policy_name is None:
policy = POLICIES.default
else:
policy = POLICIES.get_by_name(policy_name)
if policy is None:
exit('Unable to find policy: %s' % policy_name)
if not json_output:
print('Using storage policy: %s ' % policy.name)
swift_dir = conf.get('swift_dir', '/etc/swift')
retries = int(conf.get('retries', 5))
concurrency = int(conf.get('concurrency', 25))
endpoint_type = str(conf.get('endpoint_type', 'publicURL'))
region_name = str(conf.get('region_name', ''))
container_report = config_true_value(conf.get('container_report', 'yes'))
object_report = config_true_value(conf.get('object_report', 'yes'))
if not (object_report or container_report):
exit("Neither container or object report is set to run")
user_domain_name = str(conf.get('user_domain_name', ''))
project_domain_name = str(conf.get('project_domain_name', ''))
project_name = str(conf.get('project_name', ''))
insecure = config_true_value(conf.get('keystone_api_insecure', 'no'))
coropool = GreenPool(size=concurrency)
os_options = {'endpoint_type': endpoint_type}
if user_domain_name:
os_options['user_domain_name'] = user_domain_name
if project_domain_name:
os_options['project_domain_name'] = project_domain_name
if project_name:
os_options['project_name'] = project_name
if region_name:
os_options['region_name'] = region_name
url, token = get_auth(conf['auth_url'],
conf['auth_user'],
conf['auth_key'],
auth_version=conf.get('auth_version', '1.0'),
os_options=os_options,
insecure=insecure)
account = url.rsplit('/', 1)[1]
connpool = Pool(max_size=concurrency)
connpool.create = lambda: SimpleClient(
url=url, token=token, retries=retries)
container_ring = Ring(swift_dir, ring_name='container')
object_ring = Ring(swift_dir, ring_name=policy.ring_name)
output = {}
if container_report:
output['container'] = container_dispersion_report(
coropool, connpool, account, container_ring, retries,
conf.get('partitions'), policy)
if object_report:
output['object'] = object_dispersion_report(coropool, connpool,
account, object_ring,
retries,
conf.get('partitions'),
policy)
return output
def _check_proxy(user, key):
url, token = get_auth(PROXY_BASE_URL + '/auth/v1.0',
user, key)
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
def setUp(self):
super(TestAccountGetFakeResponsesMatch, self).setUp()
self.url, self.token = get_auth(
'http://127.0.0.1:8080/auth/v1.0', 'admin:admin', 'admin')
# 507 in case the test target is a VM using mount_check
if resp.status not in (404, 507):
raise Exception('Unexpected status %s' % resp.status)
break
except Exception, err:
if time() > try_until:
print err
print 'Giving up on %s:%s after %s seconds.' % (
server, port, timeout)
raise err
sleep(0.1)
else:
try_until = time() + timeout
while True:
try:
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'test:tester', 'testing')
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
except Exception, err:
if time() > try_until:
print err
print 'Giving up on proxy:8080 after 30 seconds.'
raise err
sleep(0.1)
return None
def kill_server(port, port2server, pids):
try:
kill(pids[port2server[port]], SIGTERM)