def test_put_container_update_fails(self): # put object req_timestamp = Timestamp(time.time()) headers = { 'X-Backend-Storage-Policy-Index': str(int(self.policy)), 'x-timestamp': req_timestamp.internal, 'content-type': 'application/octet-stream', 'X-Container-Host': '10.0.0.1:6010', 'X-Container-Device': 'sda1', 'X-Container-Partition': '0', } body = 'test body' req = Request.blank(self._get_path(), method='PUT', headers=headers) req.body = body with mocked_http_conn(503) as fakeconn: resp = req.get_response(self.app) self.assertRaises(StopIteration, next, fakeconn.code_iter) self.assertEqual(resp.status_int, 201) # check async keys storage_policy = server.diskfile.get_async_dir(int(self.policy)) hashpath = hash_path('a', 'c', self.buildKey('o')) start_key = '%s.%s' % (storage_policy, hashpath) end_key = '%s.%s/' % (storage_policy, hashpath) keys = self.client.getKeyRange(start_key, end_key).wait() self.assertEqual(1, len(keys)) # the async_update async_key = keys[0] expected = start_key + '.%s' % req_timestamp.internal self.assertEqual(expected, async_key) resp = self.client.get(expected) async_data = server.msgpack.unpackb(resp.wait().value) expected = { 'account': 'a', 'container': 'c', 'obj': self.buildKey('o'), 'op': 'PUT', 'headers': { 'User-Agent': 'obj-server %d' % os.getpid(), 'X-Size': str(len(body)), 'X-Content-Type': 'application/octet-stream', 'X-Etag': hashlib.md5(body).hexdigest(), 'X-Timestamp': req_timestamp.internal, 'X-Trans-Id': '-', 'X-Backend-Storage-Policy-Index': str(int(self.policy)), 'Referer': req.as_referer(), }, } self.assertEqual(async_data, expected)
def test_put_container_update(self): container_updates = [] def capture_updates(ip, port, method, path, headers, *args, **kwargs): container_updates.append((ip, port, method, path, headers)) # put object req_timestamp = Timestamp(time.time()) headers = { 'X-Backend-Storage-Policy-Index': str(int(self.policy)), 'x-timestamp': req_timestamp.internal, 'content-type': 'application/octet-stream', 'X-Container-Host': '10.0.0.1:6010', 'X-Container-Device': 'sda1', 'X-Container-Partition': '0', } body = 'test body' req = Request.blank(self._get_path(), method='PUT', headers=headers) req.body = body with mocked_http_conn(201, give_connect=capture_updates) as fakeconn: resp = req.get_response(self.app) self.assertRaises(StopIteration, next, fakeconn.code_iter) self.assertEqual(resp.status_int, 201) self.assertEqual(len(container_updates), 1) for update in container_updates: ip, port, method, path, headers = update self.assertEqual(ip, '10.0.0.1') self.assertEqual(port, '6010') self.assertEqual(method, 'PUT') device, part, account, container, obj = split_path( path, minsegs=5, rest_with_last=True) self.assertEqual(device, 'sda1') self.assertEqual(part, '0') self.assertEqual(account, 'a') self.assertEqual(container, 'c') self.assertEqual(obj, self.buildKey('o')) expected = { 'User-Agent': 'obj-server %d' % os.getpid(), 'X-Size': str(len(body)), 'X-Content-Type': 'application/octet-stream', 'X-Etag': hashlib.md5(body).hexdigest(), 'X-Timestamp': req_timestamp.internal, 'X-Trans-Id': '-', 'X-Backend-Storage-Policy-Index': str(int(self.policy)), 'Referer': req.as_referer(), } self.assertEqual(headers, expected)
def test_put_object_async_update(self): # put object req_timestamp = Timestamp(time.time()) headers = { 'X-Backend-Storage-Policy-Index': str(int(self.policy)), 'x-timestamp': req_timestamp.internal, 'content-type': 'application/octet-stream', 'X-Container-Host': '10.0.0.1:6010', 'X-Container-Device': 'sda1', 'X-Container-Partition': '0', } body = 'test body' req = Request.blank(self._get_path(), method='PUT', headers=headers) req.body = body with mocked_http_conn(503) as fakeconn: resp = req.get_response(self.app) self.assertRaises(StopIteration, next, fakeconn.code_iter) self.assertEqual(resp.status_int, 201) # check async keys storage_policy = server.diskfile.get_async_dir(int(self.policy)) hashpath = hash_path('a', 'c', self.buildKey('o')) start_key = '%s.%s' % (storage_policy, hashpath) end_key = '%s.%s/' % (storage_policy, hashpath) keys = self.client.getKeyRange(start_key, end_key).wait() self.assertEqual(1, len(keys)) # the async_update # check updater updates container_updates = [] def capture_updates(ip, port, method, path, headers, *args, **kwargs): container_updates.append((ip, port, method, path, headers)) with mocked_http_conn(201, 201, 201, give_connect=capture_updates) as fakeconn: self.updater.run_once(devices=self.devices) self.assertRaises(StopIteration, next, fakeconn.code_iter) self.assertEqual(self.updater.stats, { 'success': 1, 'found_updates': 1}) self.assertEquals(self.container_ring.replicas, len(container_updates)) expected_headers = { 'user-agent': 'obj-updater %d' % os.getpid(), 'X-Size': str(len(body)), 'X-Content-Type': 'application/octet-stream', 'X-Etag': hashlib.md5(body).hexdigest(), 'X-Timestamp': req_timestamp.internal, 'X-Trans-Id': '-', 'X-Backend-Storage-Policy-Index': str(int(self.policy)), 'Referer': req.as_referer(), } expected_updates = [ ('10.0.0.0', 1000, 'PUT', 'sda', '1', 'a', 'c', self.buildKey('o'), expected_headers), ('10.0.0.1', 1001, 'PUT', 'sdb', '1', 'a', 'c', self.buildKey('o'), expected_headers), ('10.0.0.2', 1002, 'PUT', 'sdc', '1', 'a', 'c', self.buildKey('o'), expected_headers), ] self.assertEqual(len(container_updates), len(expected_updates)) for update, expected in zip(container_updates, expected_updates): ip, port, method, path, headers = update device, part, account, container, obj = split_path( path, minsegs=5, rest_with_last=True) update_parts = (ip, port, method, device, part, account, container, obj, headers) self.assertEqual(update_parts, expected) # check async keys empty storage_policy = server.diskfile.get_async_dir(int(self.policy)) hashpath = hash_path('a', 'c', self.buildKey('o')) start_key = '%s.%s' % (storage_policy, hashpath) end_key = '%s.%s/' % (storage_policy, hashpath) keys = self.client.getKeyRange(start_key, end_key).wait() self.assertEqual(0, len(keys)) # async update consumed
def test_put_object_async_update(self): # put object req_timestamp = Timestamp(time.time()) headers = { 'X-Backend-Storage-Policy-Index': str(int(self.policy)), 'x-timestamp': req_timestamp.internal, 'content-type': 'application/octet-stream', 'X-Container-Host': '10.0.0.1:6010', 'X-Container-Device': 'sda1', 'X-Container-Partition': '0', } body = 'test body' req = Request.blank(self._get_path(), method='PUT', headers=headers) req.body = body with mocked_http_conn(503) as fakeconn: resp = req.get_response(self.app) self.assertRaises(StopIteration, next, fakeconn.code_iter) self.assertEqual(resp.status_int, 201) # check async keys storage_policy = server.diskfile.get_async_dir(int(self.policy)) hashpath = hash_path('a', 'c', self.buildKey('o')) start_key = '%s.%s' % (storage_policy, hashpath) end_key = '%s.%s/' % (storage_policy, hashpath) keys = self.client.getKeyRange(start_key, end_key).wait() self.assertEqual(1, len(keys)) # the async_update # check updater updates container_updates = [] def capture_updates(ip, port, method, path, headers, *args, **kwargs): container_updates.append((ip, port, method, path, headers)) with mocked_http_conn(201, 201, 201, give_connect=capture_updates) as fakeconn: self.updater.run_once(devices=self.devices) self.assertRaises(StopIteration, next, fakeconn.code_iter) self.assertEqual(self.updater.stats, { 'success': 1, 'found_updates': 1 }) self.assertEquals(self.container_ring.replicas, len(container_updates)) expected_headers = { 'user-agent': 'obj-updater %d' % os.getpid(), 'X-Size': str(len(body)), 'X-Content-Type': 'application/octet-stream', 'X-Etag': hashlib.md5(body).hexdigest(), 'X-Timestamp': req_timestamp.internal, 'X-Trans-Id': '-', 'X-Backend-Storage-Policy-Index': str(int(self.policy)), 'Referer': req.as_referer(), } expected_updates = [ ('10.0.0.0', 1000, 'PUT', 'sda', '1', 'a', 'c', self.buildKey('o'), expected_headers), ('10.0.0.1', 1001, 'PUT', 'sdb', '1', 'a', 'c', self.buildKey('o'), expected_headers), ('10.0.0.2', 1002, 'PUT', 'sdc', '1', 'a', 'c', self.buildKey('o'), expected_headers), ] self.assertEqual(len(container_updates), len(expected_updates)) for update, expected in zip(container_updates, expected_updates): ip, port, method, path, headers = update device, part, account, container, obj = split_path( path, minsegs=5, rest_with_last=True) update_parts = (ip, port, method, device, part, account, container, obj, headers) self.assertEqual(update_parts, expected) # check async keys empty storage_policy = server.diskfile.get_async_dir(int(self.policy)) hashpath = hash_path('a', 'c', self.buildKey('o')) start_key = '%s.%s' % (storage_policy, hashpath) end_key = '%s.%s/' % (storage_policy, hashpath) keys = self.client.getKeyRange(start_key, end_key).wait() self.assertEqual(0, len(keys)) # async update consumed