def setUp(self):
resetswift()
try:
self.ipport2server = {}
self.configs = defaultdict(dict)
self.account_ring = get_ring(
'account',
self.acct_cont_required_replicas,
self.acct_cont_required_devices,
ipport2server=self.ipport2server,
config_paths=self.configs)
self.container_ring = get_ring(
'container',
self.acct_cont_required_replicas,
self.acct_cont_required_devices,
ipport2server=self.ipport2server,
config_paths=self.configs)
self.policy = get_policy(**self.policy_requirements)
self.object_ring = get_ring(
self.policy.ring_name,
self.obj_required_replicas,
self.obj_required_devices,
server='object',
ipport2server=self.ipport2server,
config_paths=self.configs)
self.servers_per_port = any(
int(readconf(c, section_name='object-replicator').get(
'servers_per_port', '0'))
for c in self.configs['object-replicator'].values())
Manager(['main']).start(wait=False)
for ipport in self.ipport2server:
check_server(ipport, self.ipport2server)
proxy_ipport = ('127.0.0.1', 8080)
self.ipport2server[proxy_ipport] = 'proxy'
self.url, self.token, self.account = check_server(
proxy_ipport, self.ipport2server)
self.account_1 = {
'url': self.url, 'token': self.token, 'account': self.account}
url2, token2 = get_auth(
'http://%s:%d/auth/v1.0' % proxy_ipport,
'test2:tester2', 'testing2')
self.account_2 = {
'url': url2, 'token': token2, 'account': url2.split('/')[-1]}
head_account(url2, token2) # sanity check
self.replicators = Manager(
['account-replicator', 'container-replicator',
'object-replicator'])
self.updaters = Manager(['container-updater', 'object-updater'])
except BaseException:
try:
raise
finally:
try:
Manager(['all']).kill()
except Exception:
pass
def stage2_swift(auth_url, token, swift_url, dry_run=True):
print "===================================="
print "========= Swift Data ============="
print "===================================="
account_details = swiftclient.head_account(swift_url, token)
print "Data used : %s Bytes" % account_details['x-account-bytes-used']
print "Containers: %s" % account_details['x-account-container-count']
print "Objects : %s" % account_details['x-account-object-count']
def stage2_swift(auth_url, token, swift_url, dry_run=True):
print "===================================="
print "========= Swift Data ============="
print "===================================="
account_details = swiftclient.head_account(swift_url, token)
print "Data used : %s Bytes" % account_details['x-account-bytes-used']
print "Containers: %s" % account_details['x-account-container-count']
print "Objects : %s" % account_details['x-account-object-count']
def check_server(ipport, ipport2server):
server = ipport2server[ipport]
if server[:-1] in ('account', 'container', 'object'):
if int(server[-1]) > 4:
return None
path = '/connect/1/2'
if server[:-1] == 'container':
path += '/3'
elif server[:-1] == 'object':
path += '/3/4'
try_until = time() + CHECK_SERVER_TIMEOUT
while True:
try:
conn = HTTPConnection(*ipport)
conn.request('GET', path)
resp = conn.getresponse()
# 404 because it's a nonsense path (and mount_check is false)
# 507 in case the test target is a VM using mount_check
if resp.status not in (404, 507):
raise Exception(
'Unexpected status %s' % resp.status)
break
except Exception as err:
if time() > try_until:
print(err)
print('Giving up on %s:%s after %s seconds.' % (
server, ipport, CHECK_SERVER_TIMEOUT))
raise err
sleep(0.1)
else:
try_until = time() + CHECK_SERVER_TIMEOUT
while True:
try:
url, token = get_auth('http://%s:%d/auth/v1.0' % ipport,
'test:tester', 'testing')
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
except Exception as err:
if time() > try_until:
print(err)
print('Giving up on proxy:8080 after 30 seconds.')
raise err
sleep(0.1)
return None
def check_server(port, port2server, pids, timeout=CHECK_SERVER_TIMEOUT):
server = port2server[port]
if server[:-1] in ('account', 'container', 'object'):
if int(server[-1]) > 4:
return None
path = '/connect/1/2'
if server[:-1] == 'container':
path += '/3'
elif server[:-1] == 'object':
path += '/3/4'
try_until = time() + timeout
while True:
try:
conn = HTTPConnection('127.0.0.1', port)
conn.request('GET', path)
resp = conn.getresponse()
# 404 because it's a nonsense path (and mount_check is false)
# 507 in case the test target is a VM using mount_check
if resp.status not in (404, 507):
raise Exception(
'Unexpected status %s' % resp.status)
break
except Exception as err:
if time() > try_until:
print err
print 'Giving up on %s:%s after %s seconds.' % (
server, port, timeout)
raise err
sleep(0.1)
else:
try_until = time() + timeout
while True:
try:
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'test:tester', 'testing')
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
except Exception as err:
if time() > try_until:
print err
print 'Giving up on proxy:8080 after 30 seconds.'
raise err
sleep(0.1)
return None
def check_server(ipport, ipport2server, pids, timeout=CHECK_SERVER_TIMEOUT):
server = ipport2server[ipport]
if server[:-1] in ("account", "container", "object"):
if int(server[-1]) > 4:
return None
path = "/connect/1/2"
if server[:-1] == "container":
path += "/3"
elif server[:-1] == "object":
path += "/3/4"
try_until = time() + timeout
while True:
try:
conn = HTTPConnection(*ipport)
conn.request("GET", path)
resp = conn.getresponse()
# 404 because it's a nonsense path (and mount_check is false)
# 507 in case the test target is a VM using mount_check
if resp.status not in (404, 507):
raise Exception("Unexpected status %s" % resp.status)
break
except Exception as err:
if time() > try_until:
print(err)
print("Giving up on %s:%s after %s seconds." % (server, ipport, timeout))
raise err
sleep(0.1)
else:
try_until = time() + timeout
while True:
try:
url, token = get_auth("http://%s:%d/auth/v1.0" % ipport, "test:tester", "testing")
account = url.split("/")[-1]
head_account(url, token)
return url, token, account
except Exception as err:
if time() > try_until:
print(err)
print("Giving up on proxy:8080 after 30 seconds.")
raise err
sleep(0.1)
return None
def reset_environment():
call(['resetswift'])
pids = {}
try:
port2server = {}
for s, p in (('account', 6002), ('container', 6001), ('object', 6000)):
for n in xrange(1, 5):
pids['%s%d' % (s, n)] = \
Popen(['swift-%s-server' % s,
'/etc/swift/%s-server/%d.conf' % (s, n)]).pid
port2server[p + (n * 10)] = '%s%d' % (s, n)
pids['proxy'] = Popen(['swift-proxy-server',
'/etc/swift/proxy-server.conf']).pid
account_ring = Ring('/etc/swift/account.ring.gz')
container_ring = Ring('/etc/swift/container.ring.gz')
object_ring = Ring('/etc/swift/object.ring.gz')
attempt = 0
while True:
attempt += 1
try:
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'test:tester', 'testing')
account = url.split('/')[-1]
head_account(url, token)
break
except Exception, err:
if attempt > 9:
print err
print 'Giving up after %s retries.' % attempt
raise err
print err
print 'Retrying in 2 seconds...'
sleep(2)
except BaseException, err:
kill_pids(pids)
raise err
def _check_proxy(ipport, user, key):
url, token = get_auth('http://%s:%d/auth/v1.0' % ipport,
user, key)
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
break
except Exception, err:
if time() > try_until:
print err
print 'Giving up on %s:%s after %s seconds.' % (
server, port, timeout)
raise err
sleep(0.1)
else:
try_until = time() + timeout
while True:
try:
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'test:tester', 'testing')
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
except Exception, err:
if time() > try_until:
print err
print 'Giving up on proxy:8080 after 30 seconds.'
raise err
sleep(0.1)
return None
def kill_server(port, port2server, pids):
try:
kill(pids[port2server[port]], SIGTERM)
except Exception, err:
print err
break
except Exception, err:
if time() > try_until:
print err
print 'Giving up on %s:%s after %s seconds.' % (
server, port, timeout)
raise err
sleep(0.1)
else:
try_until = time() + timeout
while True:
try:
url, token = get_auth('http://127.0.0.1:8080/auth/v1.0',
'test:tester', 'testing')
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
except Exception, err:
if time() > try_until:
print err
print 'Giving up on proxy:8080 after 30 seconds.'
raise err
sleep(0.1)
return None
def kill_server(port, port2server, pids):
try:
kill(pids[port2server[port]], SIGTERM)
except Exception, err:
print err
def _check_proxy(user, key):
url, token = get_auth(PROXY_BASE_URL + '/auth/v1.0',
user, key)
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
def _check_proxy(ipport, user, key):
url, token = get_auth('http://%s:%d/auth/v1.0' % ipport,
user, key)
account = url.split('/')[-1]
head_account(url, token)
return url, token, account
