def setUp(self): resetswift() try: self.ipport2server = {} self.configs = defaultdict(dict) self.account_ring = get_ring( 'account', self.acct_cont_required_replicas, self.acct_cont_required_devices, ipport2server=self.ipport2server, config_paths=self.configs) self.container_ring = get_ring( 'container', self.acct_cont_required_replicas, self.acct_cont_required_devices, ipport2server=self.ipport2server, config_paths=self.configs) self.policy = get_policy(**self.policy_requirements) self.object_ring = get_ring( self.policy.ring_name, self.obj_required_replicas, self.obj_required_devices, server='object', ipport2server=self.ipport2server, config_paths=self.configs) self.servers_per_port = any( int(readconf(c, section_name='object-replicator').get( 'servers_per_port', '0')) for c in self.configs['object-replicator'].values()) Manager(['main']).start(wait=False) for ipport in self.ipport2server: check_server(ipport, self.ipport2server) proxy_ipport = ('127.0.0.1', 8080) self.ipport2server[proxy_ipport] = 'proxy' self.url, self.token, self.account = check_server( proxy_ipport, self.ipport2server) self.account_1 = { 'url': self.url, 'token': self.token, 'account': self.account} url2, token2 = get_auth( 'http://%s:%d/auth/v1.0' % proxy_ipport, 'test2:tester2', 'testing2') self.account_2 = { 'url': url2, 'token': token2, 'account': url2.split('/')[-1]} head_account(url2, token2) # sanity check self.replicators = Manager( ['account-replicator', 'container-replicator', 'object-replicator']) self.updaters = Manager(['container-updater', 'object-updater']) except BaseException: try: raise finally: try: Manager(['all']).kill() except Exception: pass
def stage2_swift(auth_url, token, swift_url, dry_run=True): print "====================================" print "========= Swift Data =============" print "====================================" account_details = swiftclient.head_account(swift_url, token) print "Data used : %s Bytes" % account_details['x-account-bytes-used'] print "Containers: %s" % account_details['x-account-container-count'] print "Objects : %s" % account_details['x-account-object-count']
def check_server(ipport, ipport2server): server = ipport2server[ipport] if server[:-1] in ('account', 'container', 'object'): if int(server[-1]) > 4: return None path = '/connect/1/2' if server[:-1] == 'container': path += '/3' elif server[:-1] == 'object': path += '/3/4' try_until = time() + CHECK_SERVER_TIMEOUT while True: try: conn = HTTPConnection(*ipport) conn.request('GET', path) resp = conn.getresponse() # 404 because it's a nonsense path (and mount_check is false) # 507 in case the test target is a VM using mount_check if resp.status not in (404, 507): raise Exception( 'Unexpected status %s' % resp.status) break except Exception as err: if time() > try_until: print(err) print('Giving up on %s:%s after %s seconds.' % ( server, ipport, CHECK_SERVER_TIMEOUT)) raise err sleep(0.1) else: try_until = time() + CHECK_SERVER_TIMEOUT while True: try: url, token = get_auth('http://%s:%d/auth/v1.0' % ipport, 'test:tester', 'testing') account = url.split('/')[-1] head_account(url, token) return url, token, account except Exception as err: if time() > try_until: print(err) print('Giving up on proxy:8080 after 30 seconds.') raise err sleep(0.1) return None
def check_server(port, port2server, pids, timeout=CHECK_SERVER_TIMEOUT): server = port2server[port] if server[:-1] in ('account', 'container', 'object'): if int(server[-1]) > 4: return None path = '/connect/1/2' if server[:-1] == 'container': path += '/3' elif server[:-1] == 'object': path += '/3/4' try_until = time() + timeout while True: try: conn = HTTPConnection('127.0.0.1', port) conn.request('GET', path) resp = conn.getresponse() # 404 because it's a nonsense path (and mount_check is false) # 507 in case the test target is a VM using mount_check if resp.status not in (404, 507): raise Exception( 'Unexpected status %s' % resp.status) break except Exception as err: if time() > try_until: print err print 'Giving up on %s:%s after %s seconds.' % ( server, port, timeout) raise err sleep(0.1) else: try_until = time() + timeout while True: try: url, token = get_auth('http://127.0.0.1:8080/auth/v1.0', 'test:tester', 'testing') account = url.split('/')[-1] head_account(url, token) return url, token, account except Exception as err: if time() > try_until: print err print 'Giving up on proxy:8080 after 30 seconds.' raise err sleep(0.1) return None
def check_server(ipport, ipport2server, pids, timeout=CHECK_SERVER_TIMEOUT): server = ipport2server[ipport] if server[:-1] in ("account", "container", "object"): if int(server[-1]) > 4: return None path = "/connect/1/2" if server[:-1] == "container": path += "/3" elif server[:-1] == "object": path += "/3/4" try_until = time() + timeout while True: try: conn = HTTPConnection(*ipport) conn.request("GET", path) resp = conn.getresponse() # 404 because it's a nonsense path (and mount_check is false) # 507 in case the test target is a VM using mount_check if resp.status not in (404, 507): raise Exception("Unexpected status %s" % resp.status) break except Exception as err: if time() > try_until: print(err) print("Giving up on %s:%s after %s seconds." % (server, ipport, timeout)) raise err sleep(0.1) else: try_until = time() + timeout while True: try: url, token = get_auth("http://%s:%d/auth/v1.0" % ipport, "test:tester", "testing") account = url.split("/")[-1] head_account(url, token) return url, token, account except Exception as err: if time() > try_until: print(err) print("Giving up on proxy:8080 after 30 seconds.") raise err sleep(0.1) return None
def reset_environment(): call(['resetswift']) pids = {} try: port2server = {} for s, p in (('account', 6002), ('container', 6001), ('object', 6000)): for n in xrange(1, 5): pids['%s%d' % (s, n)] = \ Popen(['swift-%s-server' % s, '/etc/swift/%s-server/%d.conf' % (s, n)]).pid port2server[p + (n * 10)] = '%s%d' % (s, n) pids['proxy'] = Popen(['swift-proxy-server', '/etc/swift/proxy-server.conf']).pid account_ring = Ring('/etc/swift/account.ring.gz') container_ring = Ring('/etc/swift/container.ring.gz') object_ring = Ring('/etc/swift/object.ring.gz') attempt = 0 while True: attempt += 1 try: url, token = get_auth('http://127.0.0.1:8080/auth/v1.0', 'test:tester', 'testing') account = url.split('/')[-1] head_account(url, token) break except Exception, err: if attempt > 9: print err print 'Giving up after %s retries.' % attempt raise err print err print 'Retrying in 2 seconds...' sleep(2) except BaseException, err: kill_pids(pids) raise err
def _check_proxy(ipport, user, key): url, token = get_auth('http://%s:%d/auth/v1.0' % ipport, user, key) account = url.split('/')[-1] head_account(url, token) return url, token, account
break except Exception, err: if time() > try_until: print err print 'Giving up on %s:%s after %s seconds.' % ( server, port, timeout) raise err sleep(0.1) else: try_until = time() + timeout while True: try: url, token = get_auth('http://127.0.0.1:8080/auth/v1.0', 'test:tester', 'testing') account = url.split('/')[-1] head_account(url, token) return url, token, account except Exception, err: if time() > try_until: print err print 'Giving up on proxy:8080 after 30 seconds.' raise err sleep(0.1) return None def kill_server(port, port2server, pids): try: kill(pids[port2server[port]], SIGTERM) except Exception, err: print err
def _check_proxy(user, key): url, token = get_auth(PROXY_BASE_URL + '/auth/v1.0', user, key) account = url.split('/')[-1] head_account(url, token) return url, token, account