def test_direct_delete_account_replication_net(self): part = '0' account = 'a' mock_path = 'swift.common.bufferedhttp.http_connect_raw' with mock.patch(mock_path) as fake_connect: fake_connect.return_value.getresponse.return_value.status = 200 direct_client.direct_delete_account( self.node, part, account, headers={'X-Backend-Use-Replication-Network': 't'}) args, kwargs = fake_connect.call_args ip = args[0] self.assertEqual(self.node['replication_ip'], ip) self.assertNotEqual(self.node['ip'], ip) port = args[1] self.assertEqual(self.node['replication_port'], port) self.assertNotEqual(self.node['port'], port) method = args[2] self.assertEqual('DELETE', method) path = args[3] self.assertEqual('/sda/0/a', path) headers = args[4] self.assertIn('X-Timestamp', headers) self.assertIn('User-Agent', headers)
def setUp(self): super(TestAccountReaper, self).setUp() self.all_objects = [] # upload some containers body = 'test-body' for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() client.put_object(self.url, self.token, container, obj, body) self.all_objects.append((policy, container, obj)) policy.load_ring('/etc/swift') Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-object-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-bytes-used']), len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account)
def setUp(self): super(TestAccountReaper, self).setUp() self.all_objects = [] # upload some containers body = 'test-body' for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() client.put_object(self.url, self.token, container, obj, body) self.all_objects.append((policy, container, obj)) policy.load_ring('/etc/swift') Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-object-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-bytes-used']), len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account)
def test_sync(self): all_objects = [] # upload some containers for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() body = 'test-body' client.put_object(self.url, self.token, container, obj, body) all_objects.append((policy, container, obj)) Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-object-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-bytes-used']), len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account) Manager(['account-reaper']).once() get_to_final_state() for policy, container, obj in all_objects: cpart, cnodes = self.container_ring.get_nodes( self.account, container) for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail('Found un-reaped /%s/%s on %r' % (self.account, container, node)) object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) for node in nodes: try: direct_get_object(node, part, self.account, container, obj) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy))
def test_sync(self): all_objects = [] # upload some containers for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() body = 'test-body' client.put_object(self.url, self.token, container, obj, body) all_objects.append((policy, container, obj)) Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-object-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-bytes-used']), len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account) Manager(['account-reaper']).once() get_to_final_state() for policy, container, obj in all_objects: cpart, cnodes = self.container_ring.get_nodes( self.account, container) for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail('Found un-reaped /%s/%s on %r' % (self.account, container, node)) object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) for node in nodes: try: direct_get_object(node, part, self.account, container, obj) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy))
def test_direct_delete_account_failure(self): part = '0' account = 'a' with mocked_http_conn(500) as conn: try: direct_client.direct_delete_account(self.node, part, account) except ClientException as err: pass self.assertEqual('DELETE', conn.method) self.assertEqual('/sda/0/a', conn.path) self.assertEqual(err.http_status, 500)
def test_direct_delete_account_failure(self): part = '0' account = 'a' with mocked_http_conn(500) as conn: try: direct_client.direct_delete_account(self.node, part, account) except ClientException as err: pass self.assertEqual('DELETE', conn.method) self.assertEqual('/sda/0/a', conn.path) self.assertEqual(err.http_status, 500)
def test_direct_delete_account_failure(self): node = {'ip': '1.2.3.4', 'port': '6200', 'device': 'sda'} part = '0' account = 'a' with mocked_http_conn(500) as conn: try: direct_client.direct_delete_account(node, part, account) except ClientException as err: pass self.assertEqual('DELETE', conn.method) self.assertEqual('/sda/0/a', conn.path) self.assertEqual(err.http_status, 500)
def test_direct_delete_account_failure(self): part = '0' account = 'a' with mocked_http_conn(500) as conn: with self.assertRaises(ClientException) as raised: direct_client.direct_delete_account(self.node, part, account) self.assertEqual(self.node['ip'], conn.host) self.assertEqual(self.node['port'], conn.port) self.assertEqual('DELETE', conn.method) self.assertEqual('/sda/0/a', conn.path) self.assertIn('X-Timestamp', conn.req_headers) self.assertIn('User-Agent', conn.req_headers) self.assertEqual(raised.exception.http_status, 500)
def test_direct_delete_account_failure(self): part = '0' account = 'a' with mocked_http_conn(500) as conn: with self.assertRaises(ClientException) as raised: direct_client.direct_delete_account(self.node, part, account) self.assertEqual(self.node['ip'], conn.host) self.assertEqual(self.node['port'], conn.port) self.assertEqual('DELETE', conn.method) self.assertEqual('/sda/0/a', conn.path) self.assertIn('X-Timestamp', conn.req_headers) self.assertIn('User-Agent', conn.req_headers) self.assertEqual(raised.exception.http_status, 500)
def test_direct_delete_account(self): part = '0' account = 'a' mock_path = 'swift.common.bufferedhttp.http_connect_raw' with mock.patch(mock_path) as fake_connect: fake_connect.return_value.getresponse.return_value.status = 200 direct_client.direct_delete_account(self.node, part, account) args, kwargs = fake_connect.call_args method = args[2] self.assertEqual('DELETE', method) path = args[3] self.assertEqual('/sda/0/a', path) headers = args[4] self.assertTrue('X-Timestamp' in headers)
def test_direct_delete_account(self): part = '0' account = 'a' mock_path = 'swift.common.bufferedhttp.http_connect_raw' with mock.patch(mock_path) as fake_connect: fake_connect.return_value.getresponse.return_value.status = 200 direct_client.direct_delete_account(self.node, part, account) args, kwargs = fake_connect.call_args method = args[2] self.assertEqual('DELETE', method) path = args[3] self.assertEqual('/sda/0/a', path) headers = args[4] self.assertTrue('X-Timestamp' in headers)
def setUp(self): super(TestAccountReaper, self).setUp() self.all_objects = [] int_client = self.make_internal_client() # upload some containers body = b'test-body' for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() client.put_object(self.url, self.token, container, obj, body) self.all_objects.append((policy, container, obj)) # Also create some reserved names container = get_reserved_name('reserved', policy.name, str(uuid.uuid4())) int_client.create_container( self.account, container, headers={'X-Storage-Policy': policy.name}) obj = get_reserved_name('object', str(uuid.uuid4())) int_client.upload_object(BytesIO(body), self.account, container, obj) self.all_objects.append((policy, container, obj)) policy.load_ring('/etc/swift') Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(self.all_objects)) self.assertEqual(int(headers['x-account-object-count']), len(self.all_objects)) self.assertEqual(int(headers['x-account-bytes-used']), len(self.all_objects) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account)
def test_sync(self): all_objects = [] # upload some containers for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() body = 'test-body' client.put_object(self.url, self.token, container, obj, body) all_objects.append((policy, container, obj)) Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-object-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-bytes-used']), len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account) # run the reaper Manager(['account-reaper']).once() for policy, container, obj in all_objects: # verify that any container deletes were at same timestamp cpart, cnodes = self.container_ring.get_nodes( self.account, container) delete_times = set() for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEquals(err.http_status, 404) delete_time = err.http_headers.get( 'X-Backend-DELETE-Timestamp') # 'X-Backend-DELETE-Timestamp' confirms it was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: # Container replicas may not yet be deleted if we have a # policy with object replicas < container replicas, so # ignore successful HEAD. We'll check for all replicas to # be deleted again after running the replicators. pass self.assertEqual(1, len(delete_times), delete_times) # verify that all object deletes were at same timestamp object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) headers = {'X-Backend-Storage-Policy-Index': int(policy)} delete_times = set() for node in nodes: try: direct_get_object(node, part, self.account, container, obj, headers=headers) except ClientException as err: self.assertEquals(err.http_status, 404) delete_time = err.http_headers.get('X-Backend-Timestamp') # 'X-Backend-Timestamp' confirms obj was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy)) self.assertEqual(1, len(delete_times)) # run replicators and updaters self.get_to_final_state() for policy, container, obj in all_objects: # verify that ALL container replicas are now deleted cpart, cnodes = self.container_ring.get_nodes( self.account, container) delete_times = set() for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEquals(err.http_status, 404) delete_time = err.http_headers.get( 'X-Backend-DELETE-Timestamp') # 'X-Backend-DELETE-Timestamp' confirms it was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s on %r' % (self.account, container, cnode)) # sanity check that object state is still consistent... object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) headers = {'X-Backend-Storage-Policy-Index': int(policy)} delete_times = set() for node in nodes: try: direct_get_object(node, part, self.account, container, obj, headers=headers) except ClientException as err: self.assertEquals(err.http_status, 404) delete_time = err.http_headers.get('X-Backend-Timestamp') # 'X-Backend-Timestamp' confirms obj was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy)) self.assertEqual(1, len(delete_times))
def test_sync(self): all_objects = [] # upload some containers for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() body = 'test-body' client.put_object(self.url, self.token, container, obj, body) all_objects.append((policy, container, obj)) Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-object-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-bytes-used']), len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account) # run the reaper Manager(['account-reaper']).once() for policy, container, obj in all_objects: # verify that any container deletes were at same timestamp cpart, cnodes = self.container_ring.get_nodes( self.account, container) delete_times = set() for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEqual(err.http_status, 404) delete_time = err.http_headers.get( 'X-Backend-DELETE-Timestamp') # 'X-Backend-DELETE-Timestamp' confirms it was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: # Container replicas may not yet be deleted if we have a # policy with object replicas < container replicas, so # ignore successful HEAD. We'll check for all replicas to # be deleted again after running the replicators. pass self.assertEqual(1, len(delete_times), delete_times) # verify that all object deletes were at same timestamp object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) headers = {'X-Backend-Storage-Policy-Index': int(policy)} delete_times = set() for node in nodes: try: direct_get_object(node, part, self.account, container, obj, headers=headers) except ClientException as err: self.assertEqual(err.http_status, 404) delete_time = err.http_headers.get('X-Backend-Timestamp') # 'X-Backend-Timestamp' confirms obj was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy)) self.assertEqual(1, len(delete_times)) # run replicators and updaters self.get_to_final_state() for policy, container, obj in all_objects: # verify that ALL container replicas are now deleted cpart, cnodes = self.container_ring.get_nodes( self.account, container) delete_times = set() for cnode in cnodes: try: direct_head_container(cnode, cpart, self.account, container) except ClientException as err: self.assertEqual(err.http_status, 404) delete_time = err.http_headers.get( 'X-Backend-DELETE-Timestamp') # 'X-Backend-DELETE-Timestamp' confirms it was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s on %r' % (self.account, container, cnode)) # sanity check that object state is still consistent... object_ring = POLICIES.get_object_ring(policy.idx, '/etc/swift/') part, nodes = object_ring.get_nodes(self.account, container, obj) headers = {'X-Backend-Storage-Policy-Index': int(policy)} delete_times = set() for node in nodes: try: direct_get_object(node, part, self.account, container, obj, headers=headers) except ClientException as err: self.assertEqual(err.http_status, 404) delete_time = err.http_headers.get('X-Backend-Timestamp') # 'X-Backend-Timestamp' confirms obj was deleted self.assertTrue(delete_time) delete_times.add(delete_time) else: self.fail('Found un-reaped /%s/%s/%s on %r in %s!' % (self.account, container, obj, node, policy)) self.assertEqual(1, len(delete_times))