def setUp(self):
super(TestReconstructorRebuild, self).setUp()
# create EC container
headers = {'X-Storage-Policy': self.policy.name}
client.put_container(self.url,
self.token,
self.container_name,
headers=headers)
# PUT object and POST some metadata
self.proxy_put()
self.headers_post = {
self._make_name('x-object-meta-').decode('utf8'):
self._make_name('meta-bar-').decode('utf8')
}
client.post_object(self.url,
self.token,
self.container_name,
self.object_name,
headers=dict(self.headers_post))
self.opart, self.onodes = self.object_ring.get_nodes(
self.account, self.container_name, self.object_name)
# stash frag etags and metadata for later comparison
self.frag_headers, self.frag_etags = self._assert_all_nodes_have_frag()
for node_index, hdrs in self.frag_headers.items():
# sanity check
self.assertIn(
'X-Backend-Durable-Timestamp', hdrs,
'Missing durable timestamp in %r' % self.frag_headers)
def _set_headers(storage_url, auth_token, container, object_name, deleted):
try:
header = client.head_object(storage_url, auth_token, container,
object_name)
new_header = {
'X-Object-Meta-Deleted': deleted,
'X-Object-Meta-Format': header['x-object-meta-format'],
'X-Object-Meta-Resolution': header['x-object-meta-resolution'],
'X-Object-Meta-Name': header['x-object-meta-name'],
'X-Object-Meta-Type': header['x-object-meta-type']
}
copy_of = None
if header['x-object-meta-type'] == 'thumbnail':
new_header['X-Object-Meta-Original'] = header[
'x-object-meta-original']
copy_of = header['x-object-meta-original']
if header['x-object-meta-type'] == 'original':
new_header['X-Object-Meta-Thumb'] = header['x-object-meta-thumb']
copy_of = header['x-object-meta-thumb']
client.post_object(storage_url,
auth_token,
container,
object_name,
headers=new_header)
return copy_of
except swift_exception.ClientException as e:
raise e
except Exception as e:
raise e
def test_unicode_ok(self):
conn = c.http_connection(u'http://www.test.com/')
args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91')
text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91'
headers = {'X-Header1': text,
b'X-Header2': 'value',
'X-2': '1', 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:kl:qr',
'X-Object-Meta-Header-not-encoded': text,
b'X-Object-Meta-Header-encoded': 'value'}
resp = MockHttpResponse()
conn[1].getresponse = resp.fake_response
conn[1]._request = resp._fake_request
c.post_object(*args, headers=headers, http_conn=conn)
# Test for RFC-2616 encoded symbols
self.assertIn(('a-b', b".x:yz mn:kl:qr"), resp.buffer)
# Test unicode header
self.assertIn(('x-header1', text.encode('utf8')),
resp.buffer)
self.assertIn((b'x-object-meta-header-not-encoded',
text.encode('utf8')), resp.buffer)
self.assertIn((b'x-object-meta-header-encoded', b'value'),
resp.buffer)
self.assertIn((b'x-header2', b'value'), resp.buffer)
def setUp(self):
super(TestReconstructorRebuild, self).setUp()
self.container_name = 'container-%s' % uuid.uuid4()
self.object_name = 'object-%s' % uuid.uuid4()
# sanity
self.assertEqual(self.policy.policy_type, EC_POLICY)
self.reconstructor = Manager(["object-reconstructor"])
# create EC container
headers = {'X-Storage-Policy': self.policy.name}
client.put_container(self.url, self.token, self.container_name,
headers=headers)
# PUT object and POST some metadata
contents = Body()
headers = {'x-object-meta-foo': 'meta-foo'}
self.headers_post = {'x-object-meta-bar': 'meta-bar'}
self.etag = client.put_object(self.url, self.token,
self.container_name,
self.object_name,
contents=contents, headers=headers)
client.post_object(self.url, self.token, self.container_name,
self.object_name, headers=dict(self.headers_post))
self.opart, self.onodes = self.object_ring.get_nodes(
self.account, self.container_name, self.object_name)
# stash frag etags and metadata for later comparison
self.frag_headers, self.frag_etags = self._assert_all_nodes_have_frag()
for node_index, hdrs in self.frag_headers.items():
# sanity check
self.assertIn(
'X-Backend-Durable-Timestamp', hdrs,
'Missing durable timestamp in %r' % self.frag_headers)
def test_unicode_ok(self):
conn = c.http_connection(u'http://www.test.com/')
args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91')
text = u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91'
headers = {
'X-Header1': text,
b'X-Header2': 'value',
'X-2': '1',
'X-3': {
'a': 'b'
},
'a-b': '.x:yz mn:kl:qr',
'X-Object-Meta-Header-not-encoded': text,
b'X-Object-Meta-Header-encoded': 'value'
}
resp = MockHttpResponse()
conn[1].getresponse = resp.fake_response
conn[1]._request = resp._fake_request
c.post_object(*args, headers=headers, http_conn=conn)
# Test for RFC-2616 encoded symbols
self.assertIn(('a-b', b".x:yz mn:kl:qr"), resp.buffer)
# Test unicode header
self.assertIn(('x-header1', text.encode('utf8')), resp.buffer)
self.assertIn(
(b'x-object-meta-header-not-encoded', text.encode('utf8')),
resp.buffer)
self.assertIn((b'x-object-meta-header-encoded', b'value'), resp.buffer)
self.assertIn((b'x-header2', b'value'), resp.buffer)
def convert(vid_format, videos, local_file, container, storage_url, auth_token, this_id, id_original, c):
temp_file = videos[vid_format]['temp_file']
headers = _get_video_headers(storage_url, auth_token, container, this_id)
conv = c.convert(local_file, temp_file, videos[vid_format]['options'])
for ftimecode in conv:
timecode = int(ftimecode * 100)
if timecode == 100:
break
video_keys[id_original][vid_format + 'status'] = int(timecode)
headers['X-Object-Meta-Status'] = timecode
client.post_object(storage_url, auth_token, container, this_id, headers=headers)
print "Converting %s (%f) ...%s\n" % (vid_format, timecode, local_file)
if os.path.exists(temp_file):
size = os.path.getsize(temp_file)
print '\n\nSIZE: ', size, '\n\n'
headers = _get_video_headers(storage_url, auth_token, container, this_id)
headers['X-Object-Meta-Content-Type'] = 'video/' + vid_format
headers['X-Object-Meta-Status'] = 100
try:
with open(temp_file, 'r+b') as f:
client.put_object(storage_url, auth_token, container, this_id, f, headers=headers)
except swift_exception.ClientException as e:
storage_url, auth_token = _reauthorize(request)
with open(temp_file, 'r+b') as f:
client.put_object(storage_url, auth_token, container, this_id, f, headers=headers)
video_keys[id_original][vid_format + 'status'] = 100
if os.path.exists(temp_file):
os.remove(temp_file)
def test_server_error(self):
body = 'c' * 60
c.http_connection = self.fake_http_connection(500, body=body)
args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', {})
self.assertRaises(c.ClientException, c.post_object, *args)
try:
c.post_object(*args)
except c.ClientException as e:
self.assertEqual(e.http_response_content, body)
def test_server_error(self):
body = 'c' * 60
c.http_connection = self.fake_http_connection(500, body=body)
args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', {})
self.assertRaises(c.ClientException, c.post_object, *args)
try:
c.post_object(*args)
except c.ClientException as e:
self.assertEquals(e.http_response_content, body)
def commitMeta(self):
if self._commitList != {}:
post_object(
self._url,
self._token,
Configuration.container_name,
self._objId,
self._commitList)
self._commitList = {}
def test_sync_unexpired_object_metadata(self):
# verify that metadata can be sync'd to a frag that has missed a POST
# and consequently that frag appears to be expired, when in fact the
# POST removed the x-delete-at header
client.put_container(self.url, self.token, self.container_name,
headers={'x-storage-policy': self.policy.name})
opart, onodes = self.object_ring.get_nodes(
self.account, self.container_name, self.object_name)
delete_at = int(time.time() + 3)
contents = 'body-%s' % uuid.uuid4()
headers = {'x-delete-at': delete_at}
client.put_object(self.url, self.token, self.container_name,
self.object_name, headers=headers, contents=contents)
# fail a primary
post_fail_node = random.choice(onodes)
post_fail_path = self.device_dir('object', post_fail_node)
self.kill_drive(post_fail_path)
# post over w/o x-delete-at
client.post_object(self.url, self.token, self.container_name,
self.object_name, {'content-type': 'something-new'})
# revive failed primary
self.revive_drive(post_fail_path)
# wait for the delete_at to pass, and check that it thinks the object
# is expired
timeout = time.time() + 5
while time.time() < timeout:
try:
direct_client.direct_head_object(
post_fail_node, opart, self.account, self.container_name,
self.object_name, headers={
'X-Backend-Storage-Policy-Index': int(self.policy)})
except direct_client.ClientException as err:
if err.http_status != 404:
raise
break
else:
time.sleep(0.1)
else:
self.fail('Failed to get a 404 from node with expired object')
self.assertEqual(err.http_status, 404)
self.assertIn('X-Backend-Timestamp', err.http_headers)
# but from the proxy we've got the whole story
headers, body = client.get_object(self.url, self.token,
self.container_name,
self.object_name)
self.assertNotIn('X-Delete-At', headers)
self.reconstructor.once()
# ... and all the nodes have the final unexpired state
for node in onodes:
headers = direct_client.direct_head_object(
node, opart, self.account, self.container_name,
self.object_name, headers={
'X-Backend-Storage-Policy-Index': int(self.policy)})
self.assertNotIn('X-Delete-At', headers)
def set_object_expiry_time(request, container, objectname):
"""For the given object, set the x-delete-at. """
storage_url = request.session.get('storage_url', '')
auth_token = request.session.get('auth_token', '')
form = TimeForm(request.POST)
if form.is_valid():
days_to_expiry = float(form.cleaned_data['days'])
hours_to_expiry = float(form.cleaned_data['hours'])
# When these values are zero, remove expiration
if (days_to_expiry + hours_to_expiry == 0.0):
try:
client.post_object(
storage_url,
auth_token,
container,
objectname,
{})
messages.add_message(
request,
messages.INFO,
_("Object expiry time removed!"))
except Exception:
messages.error(request, "Error updating object expiry time.")
else:
seconds_to_expiry = int(time.time()) + int(
days_to_expiry * 24 * 3600
+ hours_to_expiry * 60 * 60)
try:
client.post_object(
storage_url,
auth_token,
container,
objectname,
{"x-delete-at": seconds_to_expiry})
messages.add_message(
request,
messages.INFO,
_("Object expiry time updated!"))
except Exception:
messages.error(request, "Error updating object expiry time.")
prefix = '/'.join(objectname.split('/')[:-1])
if prefix:
prefix += '/'
else:
messages.error(request, "Invalid form.")
return redirect(swiftbrowser.views.objectview, container, prefix)
def test_unicode_ok(self):
conn = c.http_connection(u'http://www.test.com/')
args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91')
headers = { #'X-Header1': u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
'X-2': 1, 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:kl:qr'}
resp = MockHttpResponse()
conn[1].getresponse = resp.fake_response
conn[1].send = resp.fake_send
c.post_object(*args, headers=headers, http_conn=conn)
# Test for RFC-2616 encoded symbols
self.assertTrue("a-b: .x:yz mn:kl:qr" in resp.buffer[0],
"[a-b: .x:yz mn:kl:qr] header is missing")
def test_unicode_ok(self):
conn = c.http_connection(u'http://www.test.com/')
args = (u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91')
headers = {'X-Header1': u'\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91',
'X-2': '1', 'X-3': {'a': 'b'}, 'a-b': '.x:yz mn:kl:qr'}
resp = MockHttpResponse()
conn[1].getresponse = resp.fake_response
conn[1].send = resp.fake_send
c.post_object(*args, headers=headers, http_conn=conn)
# Test for RFC-2616 encoded symbols
self.assertTrue("a-b: .x:yz mn:kl:qr" in resp.buffer[0],
"[a-b: .x:yz mn:kl:qr] header is missing")
def test_main(self):
# create EC container
headers = {'X-Storage-Policy': self.policy.name}
client.put_container(self.url,
self.token,
self.container_name,
headers=headers)
# PUT object
contents = Body()
headers = {'x-object-meta-foo': 'meta-foo'}
headers_post = {'x-object-meta-bar': 'meta-bar'}
etag = client.put_object(self.url,
self.token,
self.container_name,
self.object_name,
contents=contents,
headers=headers)
client.post_object(self.url,
self.token,
self.container_name,
self.object_name,
headers=headers_post)
del headers_post['X-Auth-Token'] # WTF, where did this come from?
# built up a list of node lists to kill data from,
# first try a single node
# then adjacent nodes and then nodes >1 node apart
opart, onodes = self.object_ring.get_nodes(self.account,
self.container_name,
self.object_name)
single_node = [random.choice(onodes)]
adj_nodes = [onodes[0], onodes[-1]]
far_nodes = [onodes[0], onodes[-2]]
test_list = [single_node, adj_nodes, far_nodes]
for node_list in test_list:
for onode in node_list:
try:
self._check_node(onode, opart, etag, headers_post)
except AssertionError as e:
self.fail(
str(e) + '\n... for node %r of scenario %r' %
(self._format_node(onode),
[self._format_node(n) for n in node_list]))
def test_server_error(self):
body = "c" * 60
c.http_connection = self.fake_http_connection(500, body=body)
args = ("http://www.test.com", "asdf", "asdf", "asdf", {})
self.assertRaises(c.ClientException, c.post_object, *args)
try:
value = c.post_object(*args)
except c.ClientException as e:
self.assertEquals(e.http_response_content, body)
def test_unicode_ok(self):
conn = c.http_connection(u"http://www.test.com/")
args = (
u"\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91",
"\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91",
u"\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91",
u"\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91",
)
text = u"\u5929\u7a7a\u4e2d\u7684\u4e4c\u4e91"
headers = {"X-Header1": text, "X-2": "1", "X-3": {"a": "b"}, "a-b": ".x:yz mn:kl:qr"}
resp = MockHttpResponse()
conn[1].getresponse = resp.fake_response
conn[1]._request = resp._fake_request
c.post_object(*args, headers=headers, http_conn=conn)
# Test for RFC-2616 encoded symbols
self.assertTrue((b"a-b", b"a-b: .x:yz mn:kl:qr"), resp.buffer)
# Test unicode header
self.assertIn((b"x-header1", text.encode("utf8")), resp.buffer)
def _set_headers(storage_url, auth_token, container, object_name, deleted, request):
try:
try:
header = client.head_object(storage_url, auth_token, container, object_name)
except swift_exception.ClientException as e:
storage_url, auth_token = _reauthorize(request)
header = client.head_object(storage_url, auth_token, container, object_name)
new_header = {'X-Object-Meta-Deleted': deleted,
'X-Object-Meta-Format': header['x-object-meta-format'],
'X-Object-Meta-Resolution': header['x-object-meta-resolution'],
'X-Object-Meta-Name': header['x-object-meta-name'],
'X-Object-Meta-Type': header['x-object-meta-type']
}
copy_of = None
if (header['x-object-meta-format'] in video_types) and (header['x-object-meta-type'] == 'original'):
new_header['X-Object-Meta-Mp4-Id'] = header['x-object-meta-mp4-id']
if 'x-object-meta-status' in header:
new_header['X-Object-Meta-Status'] = header['x-object-meta-status']
if header['x-object-meta-type'] == 'thumbnail':
new_header['X-Object-Meta-Original'] = header['x-object-meta-original']
copy_of = header['x-object-meta-original']
if header['x-object-meta-type'] == 'original':
new_header['X-Object-Meta-Thumb'] = header['x-object-meta-thumb']
if container == 'Video':
copy_of = [header['x-object-meta-thumb'], header['x-object-meta-mp4-id']]
else:
copy_of = header['x-object-meta-thumb']
if 'x-object-meta-preview-id' in header:
new_header['X-Object-Meta-Preview-Id'] = header['x-object-meta-preview-id']
copy_of = [header['x-object-meta-thumb'], header['x-object-meta-preview-id']]
try:
client.post_object(storage_url, auth_token, container, object_name, headers=new_header)
except swift_exception.ClientException as e:
storage_url, auth_token = _reauthorize(request)
client.post_object(storage_url, auth_token, container, object_name, headers=new_header)
return copy_of
except swift_exception.ClientException as e:
raise e
except Exception as e:
raise e
def _set_headers(storage_url, auth_token, container, object_name, deleted):
try:
header = client.head_object(storage_url, auth_token, container, object_name)
new_header = {'X-Object-Meta-Deleted': deleted,
'X-Object-Meta-Format': header['x-object-meta-format'],
'X-Object-Meta-Resolution': header['x-object-meta-resolution'],
'X-Object-Meta-Name': header['x-object-meta-name'],
'X-Object-Meta-Type': header['x-object-meta-type']
}
copy_of = None
if header['x-object-meta-type'] == 'thumbnail':
new_header['X-Object-Meta-Original'] = header['x-object-meta-original']
copy_of = header['x-object-meta-original']
if header['x-object-meta-type'] == 'original':
new_header['X-Object-Meta-Thumb'] = header['x-object-meta-thumb']
copy_of = header['x-object-meta-thumb']
client.post_object(storage_url, auth_token, container, object_name, headers=new_header)
return copy_of
except swift_exception.ClientException as e:
raise e
except Exception as e:
raise e
def test_main(self):
# create EC container
headers = {'X-Storage-Policy': self.policy.name}
client.put_container(self.url, self.token, self.container_name,
headers=headers)
# PUT object
contents = Body()
headers = {'x-object-meta-foo': 'meta-foo'}
headers_post = {'x-object-meta-bar': 'meta-bar'}
etag = client.put_object(self.url, self.token,
self.container_name,
self.object_name,
contents=contents, headers=headers)
client.post_object(self.url, self.token, self.container_name,
self.object_name, headers=headers_post)
del headers_post['X-Auth-Token'] # WTF, where did this come from?
# built up a list of node lists to kill data from,
# first try a single node
# then adjacent nodes and then nodes >1 node apart
opart, onodes = self.object_ring.get_nodes(
self.account, self.container_name, self.object_name)
single_node = [random.choice(onodes)]
adj_nodes = [onodes[0], onodes[-1]]
far_nodes = [onodes[0], onodes[-2]]
test_list = [single_node, adj_nodes, far_nodes]
for node_list in test_list:
for onode in node_list:
try:
self._check_node(onode, opart, etag, headers_post)
except AssertionError as e:
self.fail(
str(e) + '\n... for node %r of scenario %r' % (
self._format_node(onode),
[self._format_node(n) for n in node_list]))
def test_handoff_non_durable(self):
# verify that reconstructor reverts non-durable frags from handoff to
# primary (and also durable frag of same object on same handoff) and
# cleans up non-durable data files on handoffs after revert
headers = {'X-Storage-Policy': self.policy.name}
client.put_container(self.url,
self.token,
self.container_name,
headers=headers)
# get our node lists
opart, onodes = self.object_ring.get_nodes(self.account,
self.container_name,
self.object_name)
pdevs = [self.device_dir(onode) for onode in onodes]
hnodes = list(
itertools.islice(self.object_ring.get_more_nodes(opart), 2))
# kill a primary nodes so we can force data onto a handoff
self.kill_drive(pdevs[0])
# PUT object at t1
contents = Body(total=3.5 * 2**20)
headers = {'x-object-meta-foo': 'meta-foo'}
headers_post = {'x-object-meta-bar': 'meta-bar'}
client.put_object(self.url,
self.token,
self.container_name,
self.object_name,
contents=contents,
headers=headers)
client.post_object(self.url,
self.token,
self.container_name,
self.object_name,
headers=headers_post)
# (Some versions of?) swiftclient will mutate the headers dict on post
headers_post.pop('X-Auth-Token', None)
# this primary can't serve the data; we expect 507 here and not 404
# because we're using mount_check to kill nodes
self.assert_direct_get_fails(onodes[0], opart, 507)
# these primaries and first handoff do have the data
for onode in (onodes[1:]):
self.assert_direct_get_succeeds(onode, opart)
_hdrs, older_frag_etag = self.assert_direct_get_succeeds(
hnodes[0], opart)
self.assert_direct_get_fails(hnodes[1], opart, 404)
# make sure we can GET the object; there's 5 primaries and 1 handoff
headers, older_obj_etag = self.proxy_get()
self.assertEqual(contents.etag, older_obj_etag)
self.assertEqual('meta-bar', headers.get('x-object-meta-bar'))
# PUT object at t2; make all frags non-durable so that the previous
# durable frags at t1 remain on object server; use InternalClient so
# that x-backend-no-commit is passed through
internal_client = self.make_internal_client()
contents2 = Body(total=2.5 * 2**20) # different content
self.assertNotEqual(contents2.etag, older_obj_etag) # sanity check
headers = {
'x-backend-no-commit': 'True',
'x-object-meta-bar': 'meta-bar-new'
}
internal_client.upload_object(contents2, self.account,
self.container_name.decode('utf8'),
self.object_name.decode('utf8'), headers)
# GET should still return the older durable object
headers, obj_etag = self.proxy_get()
self.assertEqual(older_obj_etag, obj_etag)
self.assertEqual('meta-bar', headers.get('x-object-meta-bar'))
# on handoff we have older durable and newer non-durable
_hdrs, frag_etag = self.assert_direct_get_succeeds(hnodes[0], opart)
self.assertEqual(older_frag_etag, frag_etag)
_hdrs, newer_frag_etag = self.assert_direct_get_succeeds(
hnodes[0], opart, require_durable=False)
self.assertNotEqual(older_frag_etag, newer_frag_etag)
# now make all the newer frags durable only on the 5 primaries
self.assertEqual(5, self.make_durable(onodes[1:], opart))
# now GET will return the newer object
headers, newer_obj_etag = self.proxy_get()
self.assertEqual(contents2.etag, newer_obj_etag)
self.assertNotEqual(older_obj_etag, newer_obj_etag)
self.assertEqual('meta-bar-new', headers.get('x-object-meta-bar'))
# fix the 507'ing primary
self.revive_drive(pdevs[0])
# fire up reconstructor on handoff node only; commit_window is
# set to zero to ensure the nondurable handoff frag is purged
hnode_id = self.config_number(hnodes[0])
self.run_custom_daemon(ObjectReconstructor, 'object-reconstructor',
hnode_id, {'commit_window': '0'})
# primary now has only the newer non-durable frag
self.assert_direct_get_fails(onodes[0], opart, 404)
_hdrs, frag_etag = self.assert_direct_get_succeeds(
onodes[0], opart, require_durable=False)
self.assertEqual(newer_frag_etag, frag_etag)
# handoff has only the older durable
_hdrs, frag_etag = self.assert_direct_get_succeeds(hnodes[0], opart)
self.assertEqual(older_frag_etag, frag_etag)
headers, frag_etag = self.assert_direct_get_succeeds(
hnodes[0], opart, require_durable=False)
self.assertEqual(older_frag_etag, frag_etag)
self.assertEqual('meta-bar', headers.get('x-object-meta-bar'))
# fire up reconstructor on handoff node only, again
self.reconstructor.once(number=hnode_id)
# primary now has the newer non-durable frag and the older durable frag
headers, frag_etag = self.assert_direct_get_succeeds(onodes[0], opart)
self.assertEqual(older_frag_etag, frag_etag)
self.assertEqual('meta-bar', headers.get('x-object-meta-bar'))
headers, frag_etag = self.assert_direct_get_succeeds(
onodes[0], opart, require_durable=False)
self.assertEqual(newer_frag_etag, frag_etag)
self.assertEqual('meta-bar-new', headers.get('x-object-meta-bar'))
# handoff has nothing
self.assert_direct_get_fails(hnodes[0],
opart,
404,
require_durable=False)
# kill all but first two primaries
for pdev in pdevs[2:]:
self.kill_drive(pdev)
# fire up reconstructor on the remaining primary[1]; without the
# other primaries, primary[1] cannot rebuild the frag but it can let
# primary[0] know that its non-durable frag can be made durable
self.reconstructor.once(number=self.config_number(onodes[1]))
# first primary now has a *durable* *newer* frag - it *was* useful to
# sync the non-durable!
headers, frag_etag = self.assert_direct_get_succeeds(onodes[0], opart)
self.assertEqual(newer_frag_etag, frag_etag)
self.assertEqual('meta-bar-new', headers.get('x-object-meta-bar'))
# revive primaries (in case we want to debug)
for pdev in pdevs[2:]:
self.revive_drive(pdev)
def test_revert_object(self):
# create EC container
headers = {'X-Storage-Policy': self.policy.name}
client.put_container(self.url,
self.token,
self.container_name,
headers=headers)
# get our node lists
opart, onodes = self.object_ring.get_nodes(self.account,
self.container_name,
self.object_name)
hnodes = self.object_ring.get_more_nodes(opart)
# kill 2 a parity count number of primary nodes so we can
# force data onto handoffs, we do that by renaming dev dirs
# to induce 507
p_dev1 = self.device_dir(onodes[0])
p_dev2 = self.device_dir(onodes[1])
self.kill_drive(p_dev1)
self.kill_drive(p_dev2)
# PUT object
contents = Body()
headers = {'x-object-meta-foo': 'meta-foo'}
headers_post = {'x-object-meta-bar': 'meta-bar'}
client.put_object(self.url,
self.token,
self.container_name,
self.object_name,
contents=contents,
headers=headers)
client.post_object(self.url,
self.token,
self.container_name,
self.object_name,
headers=headers_post)
# (Some versions of?) swiftclient will mutate the headers dict on post
headers_post.pop('X-Auth-Token', None)
# these primaries can't serve the data any more, we expect 507
# here and not 404 because we're using mount_check to kill nodes
for onode in (onodes[0], onodes[1]):
try:
self.direct_get(onode, opart)
except direct_client.DirectClientException as err:
self.assertEqual(err.http_status, 507)
else:
self.fail('Node data on %r was not fully destroyed!' %
(onode, ))
# now take out another primary
p_dev3 = self.device_dir(onodes[2])
self.kill_drive(p_dev3)
# this node can't servce the data any more
try:
self.direct_get(onodes[2], opart)
except direct_client.DirectClientException as err:
self.assertEqual(err.http_status, 507)
else:
self.fail('Node data on %r was not fully destroyed!' % (onode, ))
# make sure we can still GET the object and its correct
# we're now pulling from handoffs and reconstructing
etag = self.proxy_get()
self.assertEqual(etag, contents.etag)
# rename the dev dirs so they don't 507 anymore
self.revive_drive(p_dev1)
self.revive_drive(p_dev2)
self.revive_drive(p_dev3)
# fire up reconstructor on handoff nodes only
for hnode in hnodes:
hnode_id = (hnode['port'] - 6000) // 10
self.reconstructor.once(number=hnode_id)
# first three primaries have data again
for onode in (onodes[0], onodes[2]):
self.direct_get(onode, opart)
# check meta
meta = client.head_object(self.url, self.token, self.container_name,
self.object_name)
for key in headers_post:
self.assertIn(key, meta)
self.assertEqual(meta[key], headers_post[key])
# handoffs are empty
for hnode in hnodes:
try:
self.direct_get(hnode, opart)
except direct_client.DirectClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail('Node data on %r was not fully destroyed!' %
(hnode, ))
def test_revert_object(self):
# create EC container
headers = {"X-Storage-Policy": self.policy.name}
client.put_container(self.url, self.token, self.container_name, headers=headers)
# get our node lists
opart, onodes = self.object_ring.get_nodes(self.account, self.container_name, self.object_name)
hnodes = self.object_ring.get_more_nodes(opart)
# kill 2 a parity count number of primary nodes so we can
# force data onto handoffs, we do that by renaming dev dirs
# to induce 507
p_dev1 = self.device_dir("object", onodes[0])
p_dev2 = self.device_dir("object", onodes[1])
self.kill_drive(p_dev1)
self.kill_drive(p_dev2)
# PUT object
contents = Body()
headers = {"x-object-meta-foo": "meta-foo"}
headers_post = {"x-object-meta-bar": "meta-bar"}
client.put_object(
self.url, self.token, self.container_name, self.object_name, contents=contents, headers=headers
)
client.post_object(self.url, self.token, self.container_name, self.object_name, headers=headers_post)
del headers_post["X-Auth-Token"] # WTF, where did this come from?
# these primaries can't servce the data any more, we expect 507
# here and not 404 because we're using mount_check to kill nodes
for onode in (onodes[0], onodes[1]):
try:
self.direct_get(onode, opart)
except direct_client.DirectClientException as err:
self.assertEqual(err.http_status, 507)
else:
self.fail("Node data on %r was not fully destoryed!" % (onode,))
# now take out another primary
p_dev3 = self.device_dir("object", onodes[2])
self.kill_drive(p_dev3)
# this node can't servce the data any more
try:
self.direct_get(onodes[2], opart)
except direct_client.DirectClientException as err:
self.assertEqual(err.http_status, 507)
else:
self.fail("Node data on %r was not fully destoryed!" % (onode,))
# make sure we can still GET the object and its correct
# we're now pulling from handoffs and reconstructing
etag = self.proxy_get()
self.assertEqual(etag, contents.etag)
# rename the dev dirs so they don't 507 anymore
self.revive_drive(p_dev1)
self.revive_drive(p_dev2)
self.revive_drive(p_dev3)
# fire up reconstructor on handoff nodes only
for hnode in hnodes:
hnode_id = (hnode["port"] - 6000) / 10
self.reconstructor.once(number=hnode_id)
# first threee primaries have data again
for onode in (onodes[0], onodes[2]):
self.direct_get(onode, opart)
# check meta
meta = client.head_object(self.url, self.token, self.container_name, self.object_name)
for key in headers_post:
self.assertTrue(key in meta)
self.assertEqual(meta[key], headers_post[key])
# handoffs are empty
for hnode in hnodes:
try:
self.direct_get(hnode, opart)
except direct_client.DirectClientException as err:
self.assertEqual(err.http_status, 404)
else:
self.fail("Node data on %r was not fully destoryed!" % (hnode,))
def test_sync_unexpired_object_metadata(self):
# verify that metadata can be sync'd to a frag that has missed a POST
# and consequently that frag appears to be expired, when in fact the
# POST removed the x-delete-at header
client.put_container(self.url,
self.token,
self.container_name,
headers={'x-storage-policy': self.policy.name})
opart, onodes = self.object_ring.get_nodes(self.account,
self.container_name,
self.object_name)
delete_at = int(time.time() + 3)
contents = ('body-%s' % uuid.uuid4()).encode()
headers = {'x-delete-at': delete_at}
client.put_object(self.url,
self.token,
self.container_name,
self.object_name,
headers=headers,
contents=contents)
# fail a primary
post_fail_node = random.choice(onodes)
post_fail_path = self.device_dir(post_fail_node)
self.kill_drive(post_fail_path)
# post over w/o x-delete-at
client.post_object(self.url, self.token, self.container_name,
self.object_name, {'content-type': 'something-new'})
# revive failed primary
self.revive_drive(post_fail_path)
# wait for the delete_at to pass, and check that it thinks the object
# is expired
timeout = time.time() + 5
err = None
while time.time() < timeout:
try:
direct_client.direct_head_object(
post_fail_node,
opart,
self.account,
self.container_name,
self.object_name,
headers={
'X-Backend-Storage-Policy-Index': int(self.policy)
})
except direct_client.ClientException as client_err:
if client_err.http_status != 404:
raise
err = client_err
break
else:
time.sleep(0.1)
else:
self.fail('Failed to get a 404 from node with expired object')
self.assertEqual(err.http_status, 404)
self.assertIn('X-Backend-Timestamp', err.http_headers)
# but from the proxy we've got the whole story
headers, body = client.get_object(self.url, self.token,
self.container_name,
self.object_name)
self.assertNotIn('X-Delete-At', headers)
self.reconstructor.once()
# ... and all the nodes have the final unexpired state
for node in onodes:
headers = direct_client.direct_head_object(
node,
opart,
self.account,
self.container_name,
self.object_name,
headers={'X-Backend-Storage-Policy-Index': int(self.policy)})
self.assertNotIn('X-Delete-At', headers)
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
args = ("http://www.test.com", "asdf", "asdf", "asdf", {})
c.post_object(*args)
def post_object(self, account_id, token, s_type, container, name, headers):
cnx = self.get_cnx(account_id, s_type)
sclient.post_object("", token, container, name, headers, http_conn=cnx)
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', {})
c.post_object(*args)
def post_object(self, account_id, token, s_type, container, name, headers):
cnx = self.get_cnx(account_id, s_type)
sclient.post_object("", token, container, name, headers, http_conn=cnx)
def test_ok(self):
c.http_connection = self.fake_http_connection(200)
args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', {})
value = c.post_object(*args)