def test_put_container_update_fails(self):
# put object
req_timestamp = Timestamp(time.time())
headers = {
'X-Backend-Storage-Policy-Index': str(int(self.policy)),
'x-timestamp': req_timestamp.internal,
'content-type': 'application/octet-stream',
'X-Container-Host': '10.0.0.1:6010',
'X-Container-Device': 'sda1',
'X-Container-Partition': '0',
}
body = 'test body'
req = Request.blank(self._get_path(), method='PUT', headers=headers)
req.body = body
with mocked_http_conn(503) as fakeconn:
resp = req.get_response(self.app)
self.assertRaises(StopIteration, next, fakeconn.code_iter)
self.assertEqual(resp.status_int, 201)
# check async keys
storage_policy = server.diskfile.get_async_dir(int(self.policy))
hashpath = hash_path('a', 'c', self.buildKey('o'))
start_key = '%s.%s' % (storage_policy, hashpath)
end_key = '%s.%s/' % (storage_policy, hashpath)
keys = self.client.getKeyRange(start_key, end_key).wait()
self.assertEqual(1, len(keys)) # the async_update
async_key = keys[0]
expected = start_key + '.%s' % req_timestamp.internal
self.assertEqual(expected, async_key)
resp = self.client.get(expected)
async_data = server.msgpack.unpackb(resp.wait().value)
expected = {
'account': 'a',
'container': 'c',
'obj': self.buildKey('o'),
'op': 'PUT',
'headers': {
'User-Agent': 'obj-server %d' % os.getpid(),
'X-Size': str(len(body)),
'X-Content-Type': 'application/octet-stream',
'X-Etag': hashlib.md5(body).hexdigest(),
'X-Timestamp': req_timestamp.internal,
'X-Trans-Id': '-',
'X-Backend-Storage-Policy-Index': str(int(self.policy)),
'Referer': req.as_referer(),
},
}
self.assertEqual(async_data, expected)
def test_put_container_update(self):
container_updates = []
def capture_updates(ip, port, method, path, headers, *args, **kwargs):
container_updates.append((ip, port, method, path, headers))
# put object
req_timestamp = Timestamp(time.time())
headers = {
'X-Backend-Storage-Policy-Index': str(int(self.policy)),
'x-timestamp': req_timestamp.internal,
'content-type': 'application/octet-stream',
'X-Container-Host': '10.0.0.1:6010',
'X-Container-Device': 'sda1',
'X-Container-Partition': '0',
}
body = 'test body'
req = Request.blank(self._get_path(), method='PUT', headers=headers)
req.body = body
with mocked_http_conn(201, give_connect=capture_updates) as fakeconn:
resp = req.get_response(self.app)
self.assertRaises(StopIteration, next, fakeconn.code_iter)
self.assertEqual(resp.status_int, 201)
self.assertEqual(len(container_updates), 1)
for update in container_updates:
ip, port, method, path, headers = update
self.assertEqual(ip, '10.0.0.1')
self.assertEqual(port, '6010')
self.assertEqual(method, 'PUT')
device, part, account, container, obj = split_path(
path, minsegs=5, rest_with_last=True)
self.assertEqual(device, 'sda1')
self.assertEqual(part, '0')
self.assertEqual(account, 'a')
self.assertEqual(container, 'c')
self.assertEqual(obj, self.buildKey('o'))
expected = {
'User-Agent': 'obj-server %d' % os.getpid(),
'X-Size': str(len(body)),
'X-Content-Type': 'application/octet-stream',
'X-Etag': hashlib.md5(body).hexdigest(),
'X-Timestamp': req_timestamp.internal,
'X-Trans-Id': '-',
'X-Backend-Storage-Policy-Index': str(int(self.policy)),
'Referer': req.as_referer(),
}
self.assertEqual(headers, expected)
def test_put_object_async_update(self):
# put object
req_timestamp = Timestamp(time.time())
headers = {
'X-Backend-Storage-Policy-Index': str(int(self.policy)),
'x-timestamp': req_timestamp.internal,
'content-type': 'application/octet-stream',
'X-Container-Host': '10.0.0.1:6010',
'X-Container-Device': 'sda1',
'X-Container-Partition': '0',
}
body = 'test body'
req = Request.blank(self._get_path(), method='PUT', headers=headers)
req.body = body
with mocked_http_conn(503) as fakeconn:
resp = req.get_response(self.app)
self.assertRaises(StopIteration, next, fakeconn.code_iter)
self.assertEqual(resp.status_int, 201)
# check async keys
storage_policy = server.diskfile.get_async_dir(int(self.policy))
hashpath = hash_path('a', 'c', self.buildKey('o'))
start_key = '%s.%s' % (storage_policy, hashpath)
end_key = '%s.%s/' % (storage_policy, hashpath)
keys = self.client.getKeyRange(start_key, end_key).wait()
self.assertEqual(1, len(keys)) # the async_update
# check updater updates
container_updates = []
def capture_updates(ip, port, method, path, headers, *args, **kwargs):
container_updates.append((ip, port, method, path, headers))
with mocked_http_conn(201, 201, 201,
give_connect=capture_updates) as fakeconn:
self.updater.run_once(devices=self.devices)
self.assertRaises(StopIteration, next, fakeconn.code_iter)
self.assertEqual(self.updater.stats, {
'success': 1, 'found_updates': 1})
self.assertEquals(self.container_ring.replicas,
len(container_updates))
expected_headers = {
'user-agent': 'obj-updater %d' % os.getpid(),
'X-Size': str(len(body)),
'X-Content-Type': 'application/octet-stream',
'X-Etag': hashlib.md5(body).hexdigest(),
'X-Timestamp': req_timestamp.internal,
'X-Trans-Id': '-',
'X-Backend-Storage-Policy-Index': str(int(self.policy)),
'Referer': req.as_referer(),
}
expected_updates = [
('10.0.0.0', 1000, 'PUT', 'sda', '1', 'a', 'c',
self.buildKey('o'), expected_headers),
('10.0.0.1', 1001, 'PUT', 'sdb', '1', 'a', 'c',
self.buildKey('o'), expected_headers),
('10.0.0.2', 1002, 'PUT', 'sdc', '1', 'a', 'c',
self.buildKey('o'), expected_headers),
]
self.assertEqual(len(container_updates), len(expected_updates))
for update, expected in zip(container_updates, expected_updates):
ip, port, method, path, headers = update
device, part, account, container, obj = split_path(
path, minsegs=5, rest_with_last=True)
update_parts = (ip, port, method, device, part, account,
container, obj, headers)
self.assertEqual(update_parts, expected)
# check async keys empty
storage_policy = server.diskfile.get_async_dir(int(self.policy))
hashpath = hash_path('a', 'c', self.buildKey('o'))
start_key = '%s.%s' % (storage_policy, hashpath)
end_key = '%s.%s/' % (storage_policy, hashpath)
keys = self.client.getKeyRange(start_key, end_key).wait()
self.assertEqual(0, len(keys)) # async update consumed
def test_put_object_async_update(self):
# put object
req_timestamp = Timestamp(time.time())
headers = {
'X-Backend-Storage-Policy-Index': str(int(self.policy)),
'x-timestamp': req_timestamp.internal,
'content-type': 'application/octet-stream',
'X-Container-Host': '10.0.0.1:6010',
'X-Container-Device': 'sda1',
'X-Container-Partition': '0',
}
body = 'test body'
req = Request.blank(self._get_path(), method='PUT', headers=headers)
req.body = body
with mocked_http_conn(503) as fakeconn:
resp = req.get_response(self.app)
self.assertRaises(StopIteration, next, fakeconn.code_iter)
self.assertEqual(resp.status_int, 201)
# check async keys
storage_policy = server.diskfile.get_async_dir(int(self.policy))
hashpath = hash_path('a', 'c', self.buildKey('o'))
start_key = '%s.%s' % (storage_policy, hashpath)
end_key = '%s.%s/' % (storage_policy, hashpath)
keys = self.client.getKeyRange(start_key, end_key).wait()
self.assertEqual(1, len(keys)) # the async_update
# check updater updates
container_updates = []
def capture_updates(ip, port, method, path, headers, *args, **kwargs):
container_updates.append((ip, port, method, path, headers))
with mocked_http_conn(201, 201, 201,
give_connect=capture_updates) as fakeconn:
self.updater.run_once(devices=self.devices)
self.assertRaises(StopIteration, next, fakeconn.code_iter)
self.assertEqual(self.updater.stats, {
'success': 1,
'found_updates': 1
})
self.assertEquals(self.container_ring.replicas, len(container_updates))
expected_headers = {
'user-agent': 'obj-updater %d' % os.getpid(),
'X-Size': str(len(body)),
'X-Content-Type': 'application/octet-stream',
'X-Etag': hashlib.md5(body).hexdigest(),
'X-Timestamp': req_timestamp.internal,
'X-Trans-Id': '-',
'X-Backend-Storage-Policy-Index': str(int(self.policy)),
'Referer': req.as_referer(),
}
expected_updates = [
('10.0.0.0', 1000, 'PUT', 'sda', '1', 'a', 'c', self.buildKey('o'),
expected_headers),
('10.0.0.1', 1001, 'PUT', 'sdb', '1', 'a', 'c', self.buildKey('o'),
expected_headers),
('10.0.0.2', 1002, 'PUT', 'sdc', '1', 'a', 'c', self.buildKey('o'),
expected_headers),
]
self.assertEqual(len(container_updates), len(expected_updates))
for update, expected in zip(container_updates, expected_updates):
ip, port, method, path, headers = update
device, part, account, container, obj = split_path(
path, minsegs=5, rest_with_last=True)
update_parts = (ip, port, method, device, part, account, container,
obj, headers)
self.assertEqual(update_parts, expected)
# check async keys empty
storage_policy = server.diskfile.get_async_dir(int(self.policy))
hashpath = hash_path('a', 'c', self.buildKey('o'))
start_key = '%s.%s' % (storage_policy, hashpath)
end_key = '%s.%s/' % (storage_policy, hashpath)
keys = self.client.getKeyRange(start_key, end_key).wait()
self.assertEqual(0, len(keys)) # async update consumed
