def test_main(self): # Create container container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) # Kill container servers excepting two of the primaries cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] kill_nonprimary_server(cnodes, self.ipport2server, self.pids) kill_server((cnode['ip'], cnode['port']), self.ipport2server, self.pids) # Create container/obj obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, '') # Restart other primary server start_server((cnode['ip'], cnode['port']), self.ipport2server, self.pids) # Assert it does not know about container/obj self.assert_(not direct_client.direct_get_container( cnode, cpart, self.account, container)[1]) # Run the object-updaters Manager(['object-updater']).once() # Assert the other primary server now knows about container/obj objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] self.assert_(obj in objs)
def put_storlet_object(url, token, storlet_name, storlet_path, dependency, main_class): # Delete previous storlet # resp = dict() '''try: c.delete_object(url, token, 'storlet', storlet_name, None, None, None, None, resp) except Exception as e: if (resp.get('status')== 404): print 'Nothing to delete' ''' metadata = {'X-Object-Meta-Storlet-Language': 'Java', 'X-Object-Meta-Storlet-Interface-Version': '1.0', 'X-Object-Meta-Storlet-Dependency': dependency, 'X-Object-Meta-Storlet-Object-Metadata': 'no', 'X-Object-Meta-Storlet-Main': main_class} f = open('%s/%s' % (storlet_path, storlet_name), 'r') content_length = None response = dict() c.put_object(url, token, 'storlet', storlet_name, f, content_length, None, None, "application/octet-stream", metadata, None, None, None, response) f.close() status = response.get('status') assert (status == 200 or status == 201)
def createEmptyResource(self, name): client.put_object(self.storage_url, self.auth_token, self.container, sanitize(name), http_conn=self.http_connection) return ObjectResource(self.container, name, self.environ, self.objects)
def uploadNote(self, note, title): if len(title) == 0: title = self._generateObjectTitle(note.getTitle()) metaManager = MetaManager(self._storage_url, self._token, title) currentCreateDate = metaManager.getCreateDate() content = note.getContent().encode('ascii', 'ignore') put_object( self._storage_url, self._token, Configuration.container_name, title, content) lastModifiedDate = MetaManager.dateNow() # currentCreateDate may be None because note may be new # this comment is just a companion for the one above, he felt lonely if currentCreateDate is None: currentCreateDate = lastModifiedDate metaManager.setCreateDate(currentCreateDate) metaManager.setLastModifiedDate(lastModifiedDate) # we only need to keep the previous tags, NONE IS CORRECT DO NOT CHANGE metaManager.setTags(None) metaManager.commitMeta() print title return title
def test_async_update_after_PUT(self): cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content) meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse(direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1]) # Run the object-updaters to be sure updates are done Manager(['object-updater']).once() # check the re-started container server has update with override values obj = direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1][0] self.assertEqual(meta['etag'], obj['hash']) self.assertEqual(len(content), obj['bytes'])
def handleCopy(self, destPath, depthInfinity): dst = '/'.join(destPath.split('/')[2:]) try: client.head_object(self.storage_url, self.auth_token, self.container, dst, http_conn=self.http_connection) except client.ClientException: pass headers = {'X-Copy-From': self.path} try: client.put_object(self.storage_url, self.auth_token, self.container, dst, headers=headers, http_conn=self.http_connection) if self.environ.get("HTTP_OVERWRITE", '') != "T": raise DAVError(HTTP_CREATED) return True except client.ClientException: return False
def create_pseudofolder_from_prefix(storage_url, auth_token, container, prefix, prefixlist): #Recursively creates pseudofolders from a given prefix, if the #prefix is not included in the prefixlist subprefix = '/'.join(prefix.split('/')[0:-1]) if subprefix == '' or prefix in prefixlist: return prefixlist prefixlist = create_pseudofolder_from_prefix( storage_url, auth_token, container, subprefix, prefixlist) content_type = 'application/directory' obj = None client.put_object( storage_url, auth_token, container, prefix + '/', obj, content_type=content_type) prefixlist.append(prefix) return prefixlist
def put_SLO(self): # Create temp files assembly = [] for i in range(1, 10): oname = '/tmp/slo_chunk_%d' % i f = open(oname, 'r') content_length = None response = dict() c.put_object(self.url, self.token, self.container, oname, f, content_length, None, None, "application/octet-stream", None, None, None, None, response) f.close() status = response.get('status') self.assertTrue(status >= 200 and status < 300) headers = response.get('headers') segment = dict() segment['path'] = 'myobjects/%s' % oname segment['size_bytes'] = 1048576 segment['etag'] = headers['etag'] assembly.append(segment) content_length = None response = dict() headers = {'x-object-meta-prop1': 'val1'} c.put_object(self.url, self.token, self.container, 'assembly', json.dumps(assembly), content_length=None, etag=None, chunk_size=None, headers=headers, query_string='multipart-manifest=put', response_dict=response) status = response.get('status') self.assertTrue(status >= 200 and status < 300)
def _test_sync(self, object_post_as_copy): source_container, dest_container = self._setup_synced_containers() # upload to source object_name = 'object-%s' % uuid.uuid4() put_headers = {'X-Object-Meta-Test': 'put_value'} client.put_object(self.url, self.token, source_container, object_name, 'test-body', headers=put_headers) # cycle container-sync Manager(['container-sync']).once() resp_headers, body = client.get_object(self.url, self.token, dest_container, object_name) self.assertEqual(body, 'test-body') self.assertIn('x-object-meta-test', resp_headers) self.assertEqual('put_value', resp_headers['x-object-meta-test']) # update metadata with a POST, using an internal client so we can # vary the object_post_as_copy setting - first use post-as-copy post_headers = {'Content-Type': 'image/jpeg', 'X-Object-Meta-Test': 'post_value'} int_client = self.make_internal_client( object_post_as_copy=object_post_as_copy) int_client.set_object_metadata(self.account, source_container, object_name, post_headers) # sanity checks... resp_headers = client.head_object( self.url, self.token, source_container, object_name) self.assertIn('x-object-meta-test', resp_headers) self.assertEqual('post_value', resp_headers['x-object-meta-test']) self.assertEqual('image/jpeg', resp_headers['content-type']) # cycle container-sync Manager(['container-sync']).once() # verify that metadata changes were sync'd resp_headers, body = client.get_object(self.url, self.token, dest_container, object_name) self.assertEqual(body, 'test-body') self.assertIn('x-object-meta-test', resp_headers) self.assertEqual('post_value', resp_headers['x-object-meta-test']) self.assertEqual('image/jpeg', resp_headers['content-type']) # delete the object client.delete_object( self.url, self.token, source_container, object_name) with self.assertRaises(ClientException) as cm: client.get_object( self.url, self.token, source_container, object_name) self.assertEqual(404, cm.exception.http_status) # sanity check # cycle container-sync Manager(['container-sync']).once() # verify delete has been sync'd with self.assertRaises(ClientException) as cm: client.get_object( self.url, self.token, dest_container, object_name) self.assertEqual(404, cm.exception.http_status) # sanity check
def setUp(self): super(TestAccountReaper, self).setUp() self.all_objects = [] # upload some containers body = 'test-body' for policy in ENABLED_POLICIES: container = 'container-%s-%s' % (policy.name, uuid.uuid4()) client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': policy.name}) obj = 'object-%s' % uuid.uuid4() client.put_object(self.url, self.token, container, obj, body) self.all_objects.append((policy, container, obj)) policy.load_ring('/etc/swift') Manager(['container-updater']).once() headers = client.head_account(self.url, self.token) self.assertEqual(int(headers['x-account-container-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-object-count']), len(ENABLED_POLICIES)) self.assertEqual(int(headers['x-account-bytes-used']), len(ENABLED_POLICIES) * len(body)) part, nodes = self.account_ring.get_nodes(self.account) for node in nodes: direct_delete_account(node, part, self.account)
def test_main(self): # Create container container = "container-%s" % uuid4() client.put_container(self.url, self.token, container) # Kill container servers excepting two of the primaries cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] kill_nonprimary_server(cnodes, self.ipport2server) kill_server((cnode["ip"], cnode["port"]), self.ipport2server) # Create container/obj obj = "object-%s" % uuid4() client.put_object(self.url, self.token, container, obj, "") # Restart other primary server start_server((cnode["ip"], cnode["port"]), self.ipport2server) # Assert it does not know about container/obj self.assertFalse(direct_client.direct_get_container(cnode, cpart, self.account, container)[1]) # Run the object-updaters Manager(["object-updater"]).once() # Assert the other primary server now knows about container/obj objs = [o["name"] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]] self.assertIn(obj, objs)
def test_raw_upload(self): # Raw upload happens when content_length is passed to put_object conn = c.http_connection(u'http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request astring = 'asdf' astring_len = len(astring) mock_file = six.StringIO(astring) c.put_object(url='http://www.test.com', http_conn=conn, contents=mock_file, content_length=astring_len) self.assertTrue(isinstance(resp.requests_params['data'], swiftclient.utils.LengthWrapper)) self.assertEqual(astring_len, len(resp.requests_params['data'].read())) mock_file = six.StringIO(astring) c.put_object(url='http://www.test.com', http_conn=conn, headers={'Content-Length': str(astring_len)}, contents=mock_file) self.assertTrue(isinstance(resp.requests_params['data'], swiftclient.utils.LengthWrapper)) self.assertEqual(astring_len, len(resp.requests_params['data'].read()))
def testTestStorlet(url, token): print "Deploying test storlet" put_storlet_object(url, token, TEST_STORLET_NAME, "%s/TestStorlet/bin/" % PATH_TO_STORLETS, '', 'com.ibm.storlet.test.test1') print "uploading object to execute test upon" c.put_object(url, token, 'myobjects', 'test_object', 'some content') print "Invoking test storlet to print" invokeTestStorlet(url, token, "print", False) print "Invoking test storlet to crash" invokeTestStorlet(url, token, "crash") print "Invoking test storlet to hold" invokeTestStorlet(url, token, "hold") print "Invoking test storlet to print" invokeTestStorlet(url, token, "print", False) print "Invoking test storlet in parallel to print" invokeTestStorletinParallel(url, token)
def test_main(self): # Create container # Kill container servers excepting two of the primaries # Create container/obj # Restart other primary server # Assert it does not know about container/obj # Run the object-updaters # Assert the other primary server now knows about container/obj container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnode['port'], self.port2server, self.pids) obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, '') start_server(cnode['port'], self.port2server, self.pids) self.assert_(not direct_client.direct_get_container( cnode, cpart, self.account, container)[1]) processes = [] for node in xrange(1, 5): processes.append(Popen(['swift-object-updater', '/etc/swift/object-server/%d.conf' % node, 'once'])) for process in processes: process.wait() objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] self.assert_(obj in objs)
def test_delete_propogate(self): # create EC container headers = {'X-Storage-Policy': self.policy.name} client.put_container(self.url, self.token, self.container_name, headers=headers) # get our node lists opart, onodes = self.object_ring.get_nodes( self.account, self.container_name, self.object_name) hnodes = self.object_ring.get_more_nodes(opart) p_dev2 = self.device_dir('object', onodes[1]) # PUT object contents = Body() client.put_object(self.url, self.token, self.container_name, self.object_name, contents=contents) # now lets shut one down self.kill_drive(p_dev2) # delete on the ones that are left client.delete_object(self.url, self.token, self.container_name, self.object_name) # spot check a node try: self.direct_get(onodes[0], opart) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 404) else: self.fail('Node data on %r was not fully destoryed!' % (onodes[0],)) # enable the first node again self.revive_drive(p_dev2) # propogate the delete... # fire up reconstructor on handoff nodes only for hnode in hnodes: hnode_id = (hnode['port'] - 6000) / 10 self.reconstructor.once(number=hnode_id) # check the first node to make sure its gone try: self.direct_get(onodes[1], opart) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 404) else: self.fail('Node data on %r was not fully destoryed!' % (onodes[0])) # make sure proxy get can't find it try: self.proxy_get() except Exception as err: self.assertEqual(err.http_status, 404) else: self.fail('Node data on %r was not fully destoryed!' % (onodes[0]))
def test_sync_lazy_dkey(self): # Create synced containers, but with no key at dest source_container, dest_container =\ self._setup_synced_containers('secret', None) # upload to source object_name = 'object-%s' % uuid.uuid4() client.put_object(self.url, self.token, source_container, object_name, 'test-body') # cycle container-sync, nothing should happen Manager(['container-sync']).once() with self.assertRaises(ClientException) as err: _junk, body = client.get_object(self.url, self.token, dest_container, object_name) self.assertEqual(err.exception.http_status, HTTP_NOT_FOUND) # amend dest key dest_headers = {'X-Container-Sync-Key': 'secret'} client.put_container(self.url, self.token, dest_container, headers=dest_headers) # cycle container-sync, should replicate Manager(['container-sync']).once() _junk, body = client.get_object(self.url, self.token, dest_container, object_name) self.assertEqual(body, 'test-body')
def get_segment_number(file_name, request, container, prefix=None): '''Return the segment number a given file should create next. If it is 0, create a pseudo folder for the file. The folder is created if it doesn't exist. ''' if prefix: foldername = prefix + '/' + file_name + '_segments' else: foldername = file_name + '_segments' storage_url = request.session.get('storage_url', '') auth_token = request.session.get('auth_token', '') foldername = os.path.normpath(foldername) foldername = foldername.strip('/') foldername += '/' content_type = 'application/directory' obj = None client.put_object(storage_url, auth_token, container, foldername, obj, content_type=content_type) meta, objects = client.get_container(storage_url, auth_token, container, delimiter='/', prefix=foldername) pseudofolders, objs = pseudofolder_object_list(objects, prefix) return get_first_nonconsecutive(objs)
def test_one_node_fails(self): # Create container1 # Kill container1 servers excepting two of the primaries # Delete container1 # Restart other container1 primary server # Create container1/object1 (allowed because at least server thinks the # container exists) # Get to a final state # Assert all container1 servers indicate container1 is alive and # well with object1 # Assert account level also indicates container1 is alive and # well with object1 container1 = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container1) client.put_container(self.url, self.token, container1) kill_nonprimary_server(cnodes, self.port2server, self.pids) kill_server(cnodes[0]['port'], self.port2server, self.pids) client.delete_container(self.url, self.token, container1) start_server(cnodes[0]['port'], self.port2server, self.pids) client.put_object(self.url, self.token, container1, 'object1', '123') get_to_final_state() for cnode in cnodes: self.assertEquals( [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container1)[1]], ['object1']) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '3')
def convert(vid_format, videos, local_file, container, storage_url, auth_token, this_id, id_original, c): temp_file = videos[vid_format]['temp_file'] headers = _get_video_headers(storage_url, auth_token, container, this_id) conv = c.convert(local_file, temp_file, videos[vid_format]['options']) for ftimecode in conv: timecode = int(ftimecode * 100) if timecode == 100: break video_keys[id_original][vid_format + 'status'] = int(timecode) headers['X-Object-Meta-Status'] = timecode client.post_object(storage_url, auth_token, container, this_id, headers=headers) print "Converting %s (%f) ...%s\n" % (vid_format, timecode, local_file) if os.path.exists(temp_file): size = os.path.getsize(temp_file) print '\n\nSIZE: ', size, '\n\n' headers = _get_video_headers(storage_url, auth_token, container, this_id) headers['X-Object-Meta-Content-Type'] = 'video/' + vid_format headers['X-Object-Meta-Status'] = 100 try: with open(temp_file, 'r+b') as f: client.put_object(storage_url, auth_token, container, this_id, f, headers=headers) except swift_exception.ClientException as e: storage_url, auth_token = _reauthorize(request) with open(temp_file, 'r+b') as f: client.put_object(storage_url, auth_token, container, this_id, f, headers=headers) video_keys[id_original][vid_format + 'status'] = 100 if os.path.exists(temp_file): os.remove(temp_file)
def put_file_as_storlet_input_object(url, token, local_path, local_file): resp = dict() f = open("%s/%s" % (local_path, local_file), "r") c.put_object(url, token, "myobjects", local_file, f, content_type="application/octet-stream", response_dict=resp) f.close() status = resp.get("status") assert status == 200 or status == 201
def create_pseudofolder(request, container, prefix=None): """ Creates a pseudofolder (empty object of type application/directory) """ storage_url = request.session.get('storage_url', '') auth_token = request.session.get('auth_token', '') form = PseudoFolderForm(request.POST) if form.is_valid(): foldername = request.POST.get('foldername', None) if prefix: foldername = prefix + '/' + foldername foldername = os.path.normpath(foldername) foldername = foldername.strip('/') foldername += '/' content_type = 'application/directory' obj = None try: client.put_object(storage_url, auth_token, container, foldername, obj, content_type=content_type) messages.add_message( request, messages.SUCCESS, _( "The folder " + request.POST.get('foldername', None) + " was created.") ) except client.ClientException: messages.add_message(request, messages.ERROR, _("Access denied.")) return JsonResponse({})
def test_main(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) apart, anodes = self.account_ring.get_nodes(self.account) anode = anodes[0] cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] kill(self.pids[self.port2server[cnode['port']]], SIGTERM) obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, '') self.pids[self.port2server[cnode['port']]] = \ Popen(['swift-container-server', '/etc/swift/container-server/%d.conf' % ((cnode['port'] - 6001) / 10)]).pid sleep(2) self.assert_(not direct_client.direct_get_container(cnode, cpart, self.account, container)[1]) ps = [] for n in xrange(1, 5): ps.append(Popen(['swift-object-updater', '/etc/swift/object-server/%d.conf' % n, 'once'])) for p in ps: p.wait() objs = [o['name'] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]] self.assert_(obj in objs)
def _work(self, op): running_ops = self.outstanding self.outstanding += 1 t1 = time() try: # client = Connection(authurl='http://192.168.16.12:8080/auth/v1.0', user='******', key='testing') # client.put_object(self.container,'%s-%s' % (self.base, op), self.data) put_object(self.storage_url, token=self.token, container=self.container, name='%s-%s' % (self.base, op), contents=self.data) t2 = time() elapsed = (t2-t1) * 1000 self.outstanding -= 1 LOG.info("Operation #%d took %.2f ms (%.2f MB/s, %d ops outstanding on arrival)" % (op, elapsed, (self.data_size / (1024.*1024)) / (t2 - t1) , running_ops)) entry = { "Operation": op, "Arrival-time": t1, "Completion-time": t2, "Elapsed": elapsed, "Outstanding-on-arrival": running_ops, "Outstanding-on-completion": self.outstanding } print '%d, %f, %f, %.2f, %d, %d' % (op, t1, t2, elapsed, running_ops, self.outstanding) return entry except KeyboardInterrupt: self.outstanding -= 1 self.running = False return None except Exception: self.errors += 1 raise
def createCollection(self, name): """Create a pseudo-folder.""" if self.path: tmp = self.path.split('/') name = '/'.join(tmp[2:]) + '/' + name name = name.strip('/') try: client.head_object(self.storage_url, self.auth_token, self.container, name, http_conn=self.http_connection) raise dav_error.DAVError(dav_error.HTTP_METHOD_NOT_ALLOWED) except client.ClientException: pass try: client.head_object(self.storage_url, self.auth_token, self.container, name + '/', http_conn=self.http_connection) raise dav_error.DAVError(dav_error.HTTP_METHOD_NOT_ALLOWED) except client.ClientException: pass client.put_object(self.storage_url, self.auth_token, self.container, sanitize(name).rstrip('/') + '/', content_type='application/directory', http_conn=self.http_connection)
def copyMoveSingle(self, destPath, isMove): src = '/'.join(self.path.split('/')[2:]) dst = '/'.join(destPath.split('/')[2:]) src_cont = self.path.split('/')[1] dst_cont = destPath.split('/')[1] if self.is_subdir(src): client.put_object(self.storage_url, self.auth_token, dst_cont, dst.rstrip('/') + '/', content_type='application/directory', http_conn=self.http_connection) return headers = {'X-Copy-From': self.path} try: client.put_object(self.storage_url, self.auth_token, dst_cont, dst, headers=headers, http_conn=self.http_connection) if isMove: client.delete_object(self.storage_url, self.auth_token, src_cont, src, http_conn=self.http_connection) except client.ClientException: pass
def create_pseudofolder(request, container, prefix=None): """ Creates a pseudofolder (empty object of type application/directory) """ storage_url = request.session.get('storage_url', '') auth_token = request.session.get('auth_token', '') form = PseudoFolderForm(request.POST) if form.is_valid(): foldername = request.POST.get('foldername', None) if prefix: foldername = prefix + '/' + foldername foldername = os.path.normpath(foldername) foldername = foldername.strip('/') foldername += '/' content_type = 'application/directory' obj = None try: client.put_object(storage_url, auth_token, container, foldername, obj, content_type=content_type) messages.add_message(request, messages.INFO, _("Pseudofolder created.")) except client.ClientException: traceback.print_exc() messages.add_message(request, messages.ERROR, _("Access denied.")) if prefix: return redirect(objectview, container=container, prefix=prefix) return redirect(objectview, container=container) return render_to_response('create_pseudofolder.html', { 'container': container, 'prefix': prefix, }, context_instance=RequestContext(request))
def ingest(self): ''' Upload a file when creating an id, generally used with swift''' put_container(self.url, self.token, self.swift_info["container"]) put_object(self.url, token=self.token, container=self.swift_info["container"], name=self.swift_info["object"], contents=open(self.path))
def _small_file_to_swift(im, size, original_name, uid, storage_url, auth_token, response_dict): id_image = uuid.uuid1() im_type = im.format main_header = _make_header(False, im_type, original_name, size, 'original-thumbnail', None) with open('temp', 'r+') as f: client.put_object(storage_url, auth_token, image_container, str(id_image), f, headers=main_header) response_dict[uid] = {'original-thumb-key': id_image} return
def test_no_content_type(self): conn = c.http_connection(u'http://www.test.com/') resp = MockHttpResponse(status=200) conn[1].getresponse = resp.fake_response conn[1]._request = resp._fake_request c.put_object(url='http://www.test.com', http_conn=conn) request_header = resp.requests_params['headers'] self.assertEqual(request_header['content-type'], b'')
def test_server_error(self): body = 'c' * 60 c.http_connection = self.fake_http_connection(500, body=body) args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') self.assertRaises(c.ClientException, c.put_object, *args) try: c.put_object(*args) except c.ClientException as e: self.assertEqual(e.http_response_content, body)
def put_object(self, account_id, token, s_type, container, name, content): cnx = self.get_cnx(account_id, s_type) sclient.put_object("", token, container, name, content, http_conn=cnx)
def test_update_during_POST_only(self): # verify correct update values when PUT update is missed but then a # POST update succeeds *before* the PUT async pending update is sent cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content, content_type='test/ctype') meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse( direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1]) int_client = self.make_internal_client() int_client.set_object_metadata(self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'}) self.assertEqual('Tomato', int_client.get_object_metadata( self.account, 'c1', 'o1')['x-object-meta-fruit']) # sanity # check the re-started container server got same update as others. # we cannot assert the actual etag value because it may be encrypted listing_etags = set() for cnode in cnodes: listing = direct_client.direct_get_container( cnode, cpart, self.account, 'c1')[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual('test/ctype', listing[0]['content_type']) listing_etags.add(listing[0]['hash']) self.assertEqual(1, len(listing_etags)) # check that listing meta returned to client is consistent with object # meta returned to client hdrs, listing = client.get_container(self.url, self.token, 'c1') self.assertEqual(1, len(listing)) self.assertEqual('o1', listing[0]['name']) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual(meta['etag'], listing[0]['hash']) self.assertEqual('test/ctype', listing[0]['content_type']) # Run the object-updaters to send the async pending from the PUT Manager(['object-updater']).once() # check container listing metadata is still correct for cnode in cnodes: listing = direct_client.direct_get_container( cnode, cpart, self.account, 'c1')[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual('test/ctype', listing[0]['content_type']) listing_etags.add(listing[0]['hash']) self.assertEqual(1, len(listing_etags))
def test_sync_unexpired_object_metadata(self): # verify that metadata can be sync'd to a frag that has missed a POST # and consequently that frag appears to be expired, when in fact the # POST removed the x-delete-at header client.put_container(self.url, self.token, self.container_name, headers={'x-storage-policy': self.policy.name}) opart, onodes = self.object_ring.get_nodes(self.account, self.container_name, self.object_name) delete_at = int(time.time() + 3) contents = ('body-%s' % uuid.uuid4()).encode() headers = {'x-delete-at': delete_at} client.put_object(self.url, self.token, self.container_name, self.object_name, headers=headers, contents=contents) # fail a primary post_fail_node = random.choice(onodes) post_fail_path = self.device_dir(post_fail_node) self.kill_drive(post_fail_path) # post over w/o x-delete-at client.post_object(self.url, self.token, self.container_name, self.object_name, {'content-type': 'something-new'}) # revive failed primary self.revive_drive(post_fail_path) # wait for the delete_at to pass, and check that it thinks the object # is expired timeout = time.time() + 5 err = None while time.time() < timeout: try: direct_client.direct_head_object( post_fail_node, opart, self.account, self.container_name, self.object_name, headers={ 'X-Backend-Storage-Policy-Index': int(self.policy) }) except direct_client.ClientException as client_err: if client_err.http_status != 404: raise err = client_err break else: time.sleep(0.1) else: self.fail('Failed to get a 404 from node with expired object') self.assertEqual(err.http_status, 404) self.assertIn('X-Backend-Timestamp', err.http_headers) # but from the proxy we've got the whole story headers, body = client.get_object(self.url, self.token, self.container_name, self.object_name) self.assertNotIn('X-Delete-At', headers) self.reconstructor.once() # ... and all the nodes have the final unexpired state for node in onodes: headers = direct_client.direct_head_object( node, opart, self.account, self.container_name, self.object_name, headers={'X-Backend-Storage-Policy-Index': int(self.policy)}) self.assertNotIn('X-Delete-At', headers)
def test_main(self): # Create one account, container and object file. # Find node with account, container and object replicas. # Delete all directories and files from this node (device). # Wait 60 seconds and check replication results. # Delete directories and files in objects storage without # deleting file "hashes.pkl". # Check, that files not replicated. # Delete file "hashes.pkl". # Check, that all files were replicated. path_list = [] data_dir = get_data_dir(self.policy) # Figure out where the devices are for node_id in range(1, 5): conf = readconf(self.configs['object-server'][node_id]) device_path = conf['app:object-server']['devices'] for dev in self.object_ring.devs: if dev['port'] == int(conf['app:object-server']['bind_port']): device = dev['device'] path_list.append(os.path.join(device_path, device)) # Put data to storage nodes container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': self.policy.name}) obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, 'VERIFY') # Get all data file information (files_list, dir_list) = collect_info(path_list) num = find_max_occupancy_node(dir_list) test_node = path_list[num] test_node_files_list = [] for files in files_list[num]: if not files.endswith('.pending'): test_node_files_list.append(files) test_node_dir_list = [] for d in dir_list[num]: if not d.startswith('tmp'): test_node_dir_list.append(d) # Run all replicators try: self.replicators.start() # Delete some files for directory in os.listdir(test_node): shutil.rmtree(os.path.join(test_node, directory)) self.assertFalse(os.listdir(test_node)) # We will keep trying these tests until they pass for up to 60s begin = time.time() while True: (new_files_list, new_dir_list) = collect_info([test_node]) try: # Check replicate files and dir for files in test_node_files_list: self.assertIn(files, new_files_list[0]) for dir in test_node_dir_list: self.assertIn(dir, new_dir_list[0]) break except Exception: if time.time() - begin > 60: raise time.sleep(1) # Delete directories and files in objects storage without # deleting file "hashes.pkl". # Check, that files not replicated. for directory in os.listdir(os.path.join(test_node, data_dir)): for input_dir in os.listdir( os.path.join(test_node, data_dir, directory)): if os.path.isdir( os.path.join(test_node, data_dir, directory, input_dir)): shutil.rmtree( os.path.join(test_node, data_dir, directory, input_dir)) # We will keep trying these tests until they pass for up to 60s begin = time.time() while True: try: for directory in os.listdir( os.path.join(test_node, data_dir)): for input_dir in os.listdir( os.path.join(test_node, data_dir, directory)): self.assertFalse( os.path.isdir( os.path.join(test_node, data_dir, directory, input_dir))) break except Exception: if time.time() - begin > 60: raise time.sleep(1) # Now, delete file "hashes.pkl". # Check, that all files were replicated. for directory in os.listdir(os.path.join(test_node, data_dir)): os.remove( os.path.join(test_node, data_dir, directory, 'hashes.pkl')) # We will keep trying these tests until they pass for up to 60s begin = time.time() while True: try: (new_files_list, new_dir_list) = collect_info([test_node]) # Check replicate files and dirs for files in test_node_files_list: self.assertIn(files, new_files_list[0]) for directory in test_node_dir_list: self.assertIn(directory, new_dir_list[0]) break except Exception: if time.time() - begin > 60: raise time.sleep(1) finally: self.replicators.stop()
def test_query_string(self): c.http_connection = self.fake_http_connection(200, query_string="hello=20") c.put_object('http://www.test.com', 'asdf', 'asdf', 'asdf', query_string="hello=20")
def test_ok(self): c.http_connection = self.fake_http_connection(200) args = ('http://www.test.com', 'asdf', 'asdf', 'asdf', 'asdf') value = c.put_object(*args) self.assertTrue(isinstance(value, basestring))
def put_object(self, headers=None): """ issue put for zero byte test object """ client.put_object(self.url, self.token, self.container_name, self.object_name, headers=headers)
def test_main(self): # Create container container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': self.policy.name}) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] # Kill one container/obj primary server kill_server(onode['port'], self.port2server, self.pids) # Delete the default data directory for objects on the primary server obj_dir = '%s/%s' % (self._get_objects_dir(onode), get_data_dir(self.policy)) shutil.rmtree(obj_dir, True) self.assertFalse(os.path.exists(obj_dir)) # Create container/obj (goes to two primary servers and one handoff) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill other two container/obj primary servers # to ensure GET handoff works for node in onodes[1:]: kill_server(node['port'], self.port2server, self.pids) # Indirectly through proxy assert we can get container/obj odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Restart those other two container/obj primary servers for node in onodes[1:]: start_server(node['port'], self.port2server, self.pids) self.assertFalse(os.path.exists(obj_dir)) # We've indirectly verified the handoff node has the object, but # let's directly verify it. # Directly to handoff server assert we can get container/obj another_onode = self.object_ring.get_more_nodes(opart).next() odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # Assert container listing (via proxy and directly) has container/obj objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') timeout = time.time() + 5 found_objs_on_cnode = [] while time.time() < timeout: for cnode in [c for c in cnodes if cnodes not in found_objs_on_cnode]: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj in objs: found_objs_on_cnode.append(cnode) if len(found_objs_on_cnode) >= len(cnodes): break time.sleep(0.3) if len(found_objs_on_cnode) < len(cnodes): missing = ['%s:%s' % (cnode['ip'], cnode['port']) for cnode in cnodes if cnode not in found_objs_on_cnode] raise Exception('Container servers %r did not know about object' % missing) # Bring the first container/obj primary server back up start_server(onode['port'], self.port2server, self.pids) # Assert that it doesn't have container/obj yet self.assertFalse(os.path.exists(obj_dir)) try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEquals(err.http_status, 404) self.assertFalse(os.path.exists(obj_dir)) else: self.fail("Expected ClientException but didn't get it") try: port_num = onode['replication_port'] except KeyError: port_num = onode['port'] try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] # Run object replication for first container/obj primary server num = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=num) # Run object replication for handoff node another_num = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_num) # Assert the first container/obj primary server now has container/obj odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # Assert the handoff server no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it")
def test_reconcile_manifest(self): manifest_data = [] def write_part(i): body = 'VERIFY%0.2d' % i + '\x00' * 1048576 part_name = 'manifest_part_%0.2d' % i manifest_entry = { "path": "/%s/%s" % (self.container_name, part_name), "etag": md5(body).hexdigest(), "size_bytes": len(body), } client.put_object(self.url, self.token, self.container_name, part_name, contents=body) manifest_data.append(manifest_entry) # get an old container stashed self.brain.stop_primary_half() policy = random.choice(list(POLICIES)) self.brain.put_container(policy.idx) self.brain.start_primary_half() # write some parts for i in range(10): write_part(i) self.brain.stop_handoff_half() wrong_policy = random.choice([p for p in POLICIES if p is not policy]) self.brain.put_container(wrong_policy.idx) # write some more parts for i in range(10, 20): write_part(i) # write manifest try: client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') except ClientException as err: # so as it works out, you can't really upload a multi-part # manifest for objects that are currently misplaced - you have to # wait until they're all available - which is about the same as # some other failure that causes data to be unavailable to the # proxy at the time of upload self.assertEqual(err.http_status, 400) # but what the heck, we'll sneak one in just to see what happens... direct_manifest_name = self.object_name + '-direct-test' object_ring = POLICIES.get_object_ring(wrong_policy.idx, '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, direct_manifest_name) container_part = self.container_ring.get_part(self.account, self.container_name) def translate_direct(data): return { 'hash': data['etag'], 'bytes': data['size_bytes'], 'name': data['path'], } direct_manifest_data = map(translate_direct, manifest_data) headers = { 'x-container-host': ','.join('%s:%s' % (n['ip'], n['port']) for n in self.container_ring.devs), 'x-container-device': ','.join(n['device'] for n in self.container_ring.devs), 'x-container-partition': container_part, 'X-Backend-Storage-Policy-Index': wrong_policy.idx, 'X-Static-Large-Object': 'True', } for node in nodes: direct_client.direct_put_object( node, part, self.account, self.container_name, direct_manifest_name, contents=utils.json.dumps(direct_manifest_data), headers=headers) break # one should do it... self.brain.start_handoff_half() get_to_final_state() Manager(['container-reconciler']).once() # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # let's see how that direct upload worked out... metadata, body = client.get_object( self.url, self.token, self.container_name, direct_manifest_name, query_string='multipart-manifest=get') self.assertEqual(metadata['x-static-large-object'].lower(), 'true') for i, entry in enumerate(utils.json.loads(body)): for key in ('hash', 'bytes', 'name'): self.assertEquals(entry[key], direct_manifest_data[i][key]) metadata, body = client.get_object(self.url, self.token, self.container_name, direct_manifest_name) self.assertEqual(metadata['x-static-large-object'].lower(), 'true') self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data)) self.assertEqual( body, ''.join('VERIFY%0.2d' % i + '\x00' * 1048576 for i in range(20))) # and regular upload should work now too client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data))
def test_reconstruct_from_reverted_fragment_archive(self): headers = {'X-Storage-Policy': self.policy.name} client.put_container(self.url, self.token, self.container_name, headers=headers) # get our node lists opart, onodes = self.object_ring.get_nodes(self.account, self.container_name, self.object_name) # find a primary server that only has one of it's devices in the # primary node list group_nodes_by_config = defaultdict(list) for n in onodes: group_nodes_by_config[self.config_number(n)].append(n) for config_number, node_list in group_nodes_by_config.items(): if len(node_list) == 1: break else: self.fail('ring balancing did not use all available nodes') primary_node = node_list[0] # ... and 507 it's device primary_device = self.device_dir(primary_node) self.kill_drive(primary_device) # PUT object contents = Body() etag = client.put_object(self.url, self.token, self.container_name, self.object_name, contents=contents) self.assertEqual(contents.etag, etag) # fix the primary device and sanity GET self.revive_drive(primary_device) _headers, actual_etag = self.proxy_get() self.assertEqual(etag, actual_etag) # find a handoff holding the fragment for hnode in self.object_ring.get_more_nodes(opart): try: _hdrs, reverted_fragment_etag = self.direct_get(hnode, opart) except direct_client.DirectClientException as err: if err.http_status != 404: raise else: break else: self.fail('Unable to find handoff fragment!') # we'll force the handoff device to revert instead of potentially # racing with rebuild by deleting any other fragments that may be on # the same server handoff_fragment_etag = None for node in onodes: if self.is_local_to(node, hnode): # we'll keep track of the etag of this fragment we're removing # in case we need it later (queue forshadowing music)... try: _hdrs, handoff_fragment_etag = self.direct_get(node, opart) except direct_client.DirectClientException as err: if err.http_status != 404: raise # this just means our handoff device was on the same # machine as the primary! continue # use the primary nodes device - not the hnode device part_dir = self.storage_dir(node, part=opart) shutil.rmtree(part_dir, True) # revert from handoff device with reconstructor self.reconstructor.once(number=self.config_number(hnode)) # verify fragment reverted to primary server self.assertEqual(reverted_fragment_etag, self.direct_get(primary_node, opart)[1]) # now we'll remove some data on one of the primary node's partners partner = random.choice( reconstructor._get_partners(primary_node['index'], onodes)) try: _hdrs, rebuilt_fragment_etag = self.direct_get(partner, opart) except direct_client.DirectClientException as err: if err.http_status != 404: raise # partner already had it's fragment removed if (handoff_fragment_etag is not None and self.is_local_to(hnode, partner)): # oh, well that makes sense then... rebuilt_fragment_etag = handoff_fragment_etag else: # I wonder what happened? self.fail('Partner inexplicably missing fragment!') part_dir = self.storage_dir(partner, part=opart) shutil.rmtree(part_dir, True) # sanity, it's gone try: self.direct_get(partner, opart) except direct_client.DirectClientException as err: if err.http_status != 404: raise else: self.fail('successful GET of removed partner fragment archive!?') # and force the primary node to do a rebuild self.reconstructor.once(number=self.config_number(primary_node)) # and validate the partners rebuilt_fragment_etag try: self.assertEqual(rebuilt_fragment_etag, self.direct_get(partner, opart)[1]) except direct_client.DirectClientException as err: if err.http_status != 404: raise else: self.fail('Did not find rebuilt fragment on partner node')
def upload(request, container, prefix=None): """ Display upload form using swift formpost """ storage_url = request.session.get('storage_url', '') auth_token = request.session.get('auth_token', '') #redirect_url = get_base_url(request) redirect_url = reverse('objectview', kwargs={ 'container': container, }) swift_url = storage_url + '/' + container + '/' if prefix: swift_url += prefix redirect_url += prefix url_parts = urlparse.urlparse(swift_url) path = url_parts.path max_file_size = 5 * 1024 * 1024 * 1024 max_file_count = 1 expires = int(time.time() + 15 * 60) key = get_temp_key(storage_url, auth_token) if not key: messages.add_message(request, messages.ERROR, _("Access denied.")) if prefix: return redirect(objectview, container=container, prefix=prefix) else: return redirect(objectview, container=container) hmac_body = '%s\n%s\n%s\n%s\n%s' % (path, redirect_url, max_file_size, max_file_count, expires) signature = hmac.new(str(key), str(hmac_body), sha1).hexdigest() prefixes = prefix_list(prefix) if request.method == 'POST': post_file = request.FILES.values().pop() if prefix: post_file_name = prefix + post_file.name else: post_file_name = post_file.name try: client.put_object(storage_url, auth_token, container, post_file_name, post_file) if prefix: return redirect(objectview, container=container, prefix=prefix) else: return redirect(objectview, container=container) except client.ClientException: traceback.print_exc() messages.add_message(request, messages.ERROR, _("Access denied.")) return render_to_response('upload_form.html', { 'swift_url': swift_url, 'redirect_url': redirect_url, 'max_file_size': max_file_size, 'max_file_count': max_file_count, 'expires': expires, 'signature': signature, 'container': container, 'prefix': prefix, 'prefixes': prefixes, }, context_instance=RequestContext(request))
def put_object(self, container, obj_name, content=None, content_type=None, *args): return swift_client.put_object(self.storage_url, self.token, container, obj_name, content, content_type, *args)
def test_handoff_non_durable(self): # verify that reconstructor reverts non-durable frags from handoff to # primary (and also durable frag of same object on same handoff) and # cleans up non-durable data files on handoffs after revert headers = {'X-Storage-Policy': self.policy.name} client.put_container(self.url, self.token, self.container_name, headers=headers) # get our node lists opart, onodes = self.object_ring.get_nodes(self.account, self.container_name, self.object_name) pdevs = [self.device_dir(onode) for onode in onodes] hnodes = list( itertools.islice(self.object_ring.get_more_nodes(opart), 2)) # kill a primary nodes so we can force data onto a handoff self.kill_drive(pdevs[0]) # PUT object at t1 contents = Body(total=3.5 * 2**20) headers = {'x-object-meta-foo': 'meta-foo'} headers_post = {'x-object-meta-bar': 'meta-bar'} client.put_object(self.url, self.token, self.container_name, self.object_name, contents=contents, headers=headers) client.post_object(self.url, self.token, self.container_name, self.object_name, headers=headers_post) # (Some versions of?) swiftclient will mutate the headers dict on post headers_post.pop('X-Auth-Token', None) # this primary can't serve the data; we expect 507 here and not 404 # because we're using mount_check to kill nodes self.assert_direct_get_fails(onodes[0], opart, 507) # these primaries and first handoff do have the data for onode in (onodes[1:]): self.assert_direct_get_succeeds(onode, opart) _hdrs, older_frag_etag = self.assert_direct_get_succeeds( hnodes[0], opart) self.assert_direct_get_fails(hnodes[1], opart, 404) # make sure we can GET the object; there's 5 primaries and 1 handoff headers, older_obj_etag = self.proxy_get() self.assertEqual(contents.etag, older_obj_etag) self.assertEqual('meta-bar', headers.get('x-object-meta-bar')) # PUT object at t2; make all frags non-durable so that the previous # durable frags at t1 remain on object server; use InternalClient so # that x-backend-no-commit is passed through internal_client = self.make_internal_client() contents2 = Body(total=2.5 * 2**20) # different content self.assertNotEqual(contents2.etag, older_obj_etag) # sanity check headers = { 'x-backend-no-commit': 'True', 'x-object-meta-bar': 'meta-bar-new' } internal_client.upload_object(contents2, self.account, self.container_name.decode('utf8'), self.object_name.decode('utf8'), headers) # GET should still return the older durable object headers, obj_etag = self.proxy_get() self.assertEqual(older_obj_etag, obj_etag) self.assertEqual('meta-bar', headers.get('x-object-meta-bar')) # on handoff we have older durable and newer non-durable _hdrs, frag_etag = self.assert_direct_get_succeeds(hnodes[0], opart) self.assertEqual(older_frag_etag, frag_etag) _hdrs, newer_frag_etag = self.assert_direct_get_succeeds( hnodes[0], opart, require_durable=False) self.assertNotEqual(older_frag_etag, newer_frag_etag) # now make all the newer frags durable only on the 5 primaries self.assertEqual(5, self.make_durable(onodes[1:], opart)) # now GET will return the newer object headers, newer_obj_etag = self.proxy_get() self.assertEqual(contents2.etag, newer_obj_etag) self.assertNotEqual(older_obj_etag, newer_obj_etag) self.assertEqual('meta-bar-new', headers.get('x-object-meta-bar')) # fix the 507'ing primary self.revive_drive(pdevs[0]) # fire up reconstructor on handoff node only; nondurable_purge_delay is # set to zero to ensure the nondurable handoff frag is purged hnode_id = (hnodes[0]['port'] % 100) // 10 self.run_custom_daemon(ObjectReconstructor, 'object-reconstructor', hnode_id, {'nondurable_purge_delay': '0'}) # primary now has only the newer non-durable frag self.assert_direct_get_fails(onodes[0], opart, 404) _hdrs, frag_etag = self.assert_direct_get_succeeds( onodes[0], opart, require_durable=False) self.assertEqual(newer_frag_etag, frag_etag) # handoff has only the older durable _hdrs, frag_etag = self.assert_direct_get_succeeds(hnodes[0], opart) self.assertEqual(older_frag_etag, frag_etag) headers, frag_etag = self.assert_direct_get_succeeds( hnodes[0], opart, require_durable=False) self.assertEqual(older_frag_etag, frag_etag) self.assertEqual('meta-bar', headers.get('x-object-meta-bar')) # fire up reconstructor on handoff node only, again self.reconstructor.once(number=hnode_id) # primary now has the newer non-durable frag and the older durable frag headers, frag_etag = self.assert_direct_get_succeeds(onodes[0], opart) self.assertEqual(older_frag_etag, frag_etag) self.assertEqual('meta-bar', headers.get('x-object-meta-bar')) headers, frag_etag = self.assert_direct_get_succeeds( onodes[0], opart, require_durable=False) self.assertEqual(newer_frag_etag, frag_etag) self.assertEqual('meta-bar-new', headers.get('x-object-meta-bar')) # handoff has nothing self.assert_direct_get_fails(hnodes[0], opart, 404, require_durable=False) # kill all but first two primaries for pdev in pdevs[2:]: self.kill_drive(pdev) # fire up reconstructor on the remaining primary[1]; without the # other primaries, primary[1] cannot rebuild the frag but it can let # primary[0] know that its non-durable frag can be made durable self.reconstructor.once(number=self.config_number(onodes[1])) # first primary now has a *durable* *newer* frag - it *was* useful to # sync the non-durable! headers, frag_etag = self.assert_direct_get_succeeds(onodes[0], opart) self.assertEqual(newer_frag_etag, frag_etag) self.assertEqual('meta-bar-new', headers.get('x-object-meta-bar')) # revive primaries (in case we want to debug) for pdev in pdevs[2:]: self.revive_drive(pdev)
def test_delete_propagate(self): # create EC container headers = {'X-Storage-Policy': self.policy.name} client.put_container(self.url, self.token, self.container_name, headers=headers) # get our node lists opart, onodes = self.object_ring.get_nodes(self.account, self.container_name, self.object_name) hnodes = list( itertools.islice(self.object_ring.get_more_nodes(opart), 2)) # PUT object contents = Body() client.put_object(self.url, self.token, self.container_name, self.object_name, contents=contents) # now lets shut down a couple of primaries failed_nodes = random.sample(onodes, 2) for node in failed_nodes: self.kill_drive(self.device_dir(node)) # Write tombstones over the nodes that are still online client.delete_object(self.url, self.token, self.container_name, self.object_name) # spot check the primary nodes that are still online delete_timestamp = None for node in onodes: if node in failed_nodes: continue try: self.direct_get(node, opart) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 404) delete_timestamp = err.http_headers['X-Backend-Timestamp'] else: self.fail('Node data on %r was not fully destroyed!' % (node, )) # run the reconstructor on the handoff node multiple times until # tombstone is pushed out - each handoff node syncs to a few # primaries each time iterations = 0 while iterations < 52: self.reconstructor.once(number=self.config_number(hnodes[0])) iterations += 1 # see if the tombstone is reverted try: self.direct_get(hnodes[0], opart) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 404) if 'X-Backend-Timestamp' not in err.http_headers: # this means the tombstone is *gone* so it's reverted break else: self.fail('Still found tombstone on %r after %s iterations' % (hnodes[0], iterations)) # tombstone is still on the *second* handoff try: self.direct_get(hnodes[1], opart) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 404) self.assertEqual(err.http_headers['X-Backend-Timestamp'], delete_timestamp) else: self.fail('Found obj data on %r' % hnodes[1]) # repair the primaries self.revive_drive(self.device_dir(failed_nodes[0])) self.revive_drive(self.device_dir(failed_nodes[1])) # run reconstructor on second handoff self.reconstructor.once(number=self.config_number(hnodes[1])) # verify tombstone is reverted on the first pass try: self.direct_get(hnodes[1], opart) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 404) self.assertNotIn('X-Backend-Timestamp', err.http_headers) else: self.fail('Found obj data on %r' % hnodes[1]) # sanity make sure proxy get can't find it try: self.proxy_get() except Exception as err: self.assertEqual(err.http_status, 404) else: self.fail('Node data on %r was not fully destroyed!' % (onodes[0]))
def test_slo_async_delete(self): if not self.cluster_info.get('slo', {}).get('allow_async_delete'): raise unittest.SkipTest('allow_async_delete not enabled') segment_container = self.container_name + '_segments' client.put_container(self.url, self.token, self.container_name, {}) client.put_container(self.url, self.token, segment_container, {}) client.put_object(self.url, self.token, segment_container, 'segment_1', b'1234') client.put_object(self.url, self.token, segment_container, 'segment_2', b'5678') client.put_object(self.url, self.token, self.container_name, 'slo', json.dumps([ { 'path': segment_container + '/segment_1' }, { 'data': 'Cg==' }, { 'path': segment_container + '/segment_2' }, ]), query_string='multipart-manifest=put') _, body = client.get_object(self.url, self.token, self.container_name, 'slo') self.assertEqual(body, b'1234\n5678') client.delete_object( self.url, self.token, self.container_name, 'slo', query_string='multipart-manifest=delete&async=true') # Object's deleted _, objects = client.get_container(self.url, self.token, self.container_name) self.assertEqual(objects, []) with self.assertRaises(client.ClientException) as caught: client.get_object(self.url, self.token, self.container_name, 'slo') self.assertEqual(404, caught.exception.http_status) # But segments are still around and accessible _, objects = client.get_container(self.url, self.token, segment_container) self.assertEqual([o['name'] for o in objects], ['segment_1', 'segment_2']) _, body = client.get_object(self.url, self.token, segment_container, 'segment_1') self.assertEqual(body, b'1234') _, body = client.get_object(self.url, self.token, segment_container, 'segment_2') self.assertEqual(body, b'5678') # make sure auto-created expirer-queue containers get in the account # listing so the expirer can find them Manager(['container-updater']).once() self.expirer.once() # Now the expirer has cleaned up the segments _, objects = client.get_container(self.url, self.token, segment_container) self.assertEqual(objects, []) with self.assertRaises(client.ClientException) as caught: client.get_object(self.url, self.token, segment_container, 'segment_1') self.assertEqual(404, caught.exception.http_status) with self.assertRaises(client.ClientException) as caught: client.get_object(self.url, self.token, segment_container, 'segment_2') self.assertEqual(404, caught.exception.http_status)
def test_main(self): # Create container1 and container2 # Assert account level sees them # Create container2/object1 # Assert account level doesn't see it yet # Get to final state # Assert account level now sees the container2/object1 # Kill account servers excepting two of the primaries # Delete container1 # Assert account level knows container1 is gone but doesn't know about # container2/object2 yet # Put container2/object2 # Run container updaters # Assert account level now knows about container2/object2 # Restart other primary account server # Assert that server doesn't know about container1's deletion or the # new container2/object2 yet # Get to final state # Assert that server is now up to date container1 = 'container1' client.put_container(self.url, self.token, container1) container2 = 'container2' client.put_container(self.url, self.token, container2) headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) self.assert_(found1) self.assert_(found2) client.put_object(self.url, self.token, container2, 'object1', '1234') headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) self.assert_(found1) self.assert_(found2) get_to_final_state() headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(found1) self.assert_(found2) apart, anodes = self.account_ring.get_nodes(self.account) kill_nonprimary_server(anodes, self.port2server, self.pids) kill_server(anodes[0]['port'], self.port2server, self.pids) client.delete_container(self.url, self.token, container1) client.put_object(self.url, self.token, container2, 'object2', '12345') headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(not found1) self.assert_(found2) processes = [] for node in xrange(1, 5): processes.append( Popen([ 'swift-container-updater', '/etc/swift/container-server/%d.conf' % node, 'once' ])) for process in processes: process.wait() headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 2) self.assertEquals(container['bytes'], 9) self.assert_(not found1) self.assert_(found2) start_server(anodes[0]['port'], self.port2server, self.pids) headers, containers = \ direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(found1) self.assert_(found2) get_to_final_state() headers, containers = \ direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 2) self.assertEquals(container['bytes'], 9) self.assert_(not found1) self.assert_(found2)
def test_revert_object(self): # create EC container headers = {'X-Storage-Policy': self.policy.name} client.put_container(self.url, self.token, self.container_name, headers=headers) # get our node lists opart, onodes = self.object_ring.get_nodes( self.account, self.container_name, self.object_name) hnodes = self.object_ring.get_more_nodes(opart) # kill 2 a parity count number of primary nodes so we can # force data onto handoffs, we do that by renaming dev dirs # to induce 507 p_dev1 = self.device_dir('object', onodes[0]) p_dev2 = self.device_dir('object', onodes[1]) self.kill_drive(p_dev1) self.kill_drive(p_dev2) # PUT object contents = Body() headers = {'x-object-meta-foo': 'meta-foo'} headers_post = {'x-object-meta-bar': 'meta-bar'} client.put_object(self.url, self.token, self.container_name, self.object_name, contents=contents, headers=headers) client.post_object(self.url, self.token, self.container_name, self.object_name, headers=headers_post) del headers_post['X-Auth-Token'] # WTF, where did this come from? # these primaries can't servce the data any more, we expect 507 # here and not 404 because we're using mount_check to kill nodes for onode in (onodes[0], onodes[1]): try: self.direct_get(onode, opart) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 507) else: self.fail('Node data on %r was not fully destoryed!' % (onode,)) # now take out another primary p_dev3 = self.device_dir('object', onodes[2]) self.kill_drive(p_dev3) # this node can't servce the data any more try: self.direct_get(onodes[2], opart) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 507) else: self.fail('Node data on %r was not fully destoryed!' % (onode,)) # make sure we can still GET the object and its correct # we're now pulling from handoffs and reconstructing etag = self.proxy_get() self.assertEqual(etag, contents.etag) # rename the dev dirs so they don't 507 anymore self.revive_drive(p_dev1) self.revive_drive(p_dev2) self.revive_drive(p_dev3) # fire up reconstructor on handoff nodes only for hnode in hnodes: hnode_id = (hnode['port'] - 6000) / 10 self.reconstructor.once(number=hnode_id) # first three primaries have data again for onode in (onodes[0], onodes[2]): self.direct_get(onode, opart) # check meta meta = client.head_object(self.url, self.token, self.container_name, self.object_name) for key in headers_post: self.assertTrue(key in meta) self.assertEqual(meta[key], headers_post[key]) # handoffs are empty for hnode in hnodes: try: self.direct_get(hnode, opart) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 404) else: self.fail('Node data on %r was not fully destoryed!' % (hnode,))
def test_main(self): # Create container1 and container2 container1 = 'container1' client.put_container(self.url, self.token, container1) container2 = 'container2' client.put_container(self.url, self.token, container2) # Assert account level sees them headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) self.assert_(found1) self.assert_(found2) # Create container2/object1 client.put_object(self.url, self.token, container2, 'object1', '1234') # Assert account level doesn't see it yet headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '0') self.assertEquals(headers['x-account-bytes-used'], '0') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) self.assert_(found1) self.assert_(found2) # Get to final state self.get_to_final_state() # Assert account level now sees the container2/object1 headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True self.assertEquals(container['count'], 0) self.assertEquals(container['bytes'], 0) elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(found1) self.assert_(found2) apart, anodes = self.account_ring.get_nodes(self.account) kill_nonprimary_server(anodes, self.port2server, self.pids) kill_server(anodes[0]['port'], self.port2server, self.pids) # Kill account servers excepting two of the primaries # Delete container1 client.delete_container(self.url, self.token, container1) # Put container2/object2 client.put_object(self.url, self.token, container2, 'object2', '12345') # Assert account level knows container1 is gone but doesn't know about # container2/object2 yet headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(not found1) self.assert_(found2) # Run container updaters Manager(['container-updater']).once() # Assert account level now knows about container2/object2 headers, containers = client.get_account(self.url, self.token) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 2) self.assertEquals(container['bytes'], 9) self.assert_(not found1) self.assert_(found2) # Restart other primary account server start_server(anodes[0]['port'], self.port2server, self.pids) # Assert that server doesn't know about container1's deletion or the # new container2/object2 yet headers, containers = \ direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEquals(headers['x-account-container-count'], '2') self.assertEquals(headers['x-account-object-count'], '1') self.assertEquals(headers['x-account-bytes-used'], '4') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 1) self.assertEquals(container['bytes'], 4) self.assert_(found1) self.assert_(found2) # Get to final state self.get_to_final_state() # Assert that server is now up to date headers, containers = \ direct_client.direct_get_account(anodes[0], apart, self.account) self.assertEquals(headers['x-account-container-count'], '1') self.assertEquals(headers['x-account-object-count'], '2') self.assertEquals(headers['x-account-bytes-used'], '9') found1 = False found2 = False for container in containers: if container['name'] == container1: found1 = True elif container['name'] == container2: found2 = True self.assertEquals(container['count'], 2) self.assertEquals(container['bytes'], 9) self.assert_(not found1) self.assert_(found2)
def test_missing_container(self): # In this test, we need to put container at handoff devices, so we # need container devices more than replica count if len(self.container_ring.devs) <= self.container_ring.replica_count: raise SkipTest("Need devices more that replica count") container = 'container-%s' % uuid4() cpart, cnodes = self.container_ring.get_nodes(self.account, container) # Kill all primary container servers for cnode in cnodes: kill_server((cnode['ip'], cnode['port']), self.ipport2server) # Create container, and all of its replicas are placed at handoff # device try: client.put_container(self.url, self.token, container) except ClientException as err: # if the cluster doesn't have enough devices, swift may return # error (ex. When we only have 4 devices in 3-replica cluster). self.assertEqual(err.http_status, 503) # Assert handoff device has a container replica another_cnode = next(self.container_ring.get_more_nodes(cpart)) direct_client.direct_get_container(another_cnode, cpart, self.account, container) # Restart all primary container servers for cnode in cnodes: start_server((cnode['ip'], cnode['port']), self.ipport2server) # Create container/obj obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, '') # Run the object-updater Manager(['object-updater']).once() # Run the container-replicator, and now, container replicas # at handoff device get moved to primary servers Manager(['container-replicator']).once() # Assert container replicas in primary servers, just moved by # replicator don't know about the object for cnode in cnodes: self.assertFalse( direct_client.direct_get_container(cnode, cpart, self.account, container)[1]) # since the container is empty - we can delete it! client.delete_container(self.url, self.token, container) # Re-run the object-updaters and now container replicas in primary # container servers should get updated Manager(['object-updater']).once() # Assert all primary container servers know about container/obj for cnode in cnodes: objs = [ o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1] ] self.assertIn(obj, objs)
def test_main(self): # Create container container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': self.policy.name}) # Kill one container/obj primary server cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes(self.account, container, obj) onode = onodes[0] kill_server((onode['ip'], onode['port']), self.ipport2server, self.pids) # Create container/obj (goes to two primary servers and one handoff) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill other two container/obj primary servers # to ensure GET handoff works for node in onodes[1:]: kill_server((node['ip'], node['port']), self.ipport2server, self.pids) # Indirectly through proxy assert we can get container/obj odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Restart those other two container/obj primary servers for node in onodes[1:]: start_server((node['ip'], node['port']), self.ipport2server, self.pids) # We've indirectly verified the handoff node has the container/object, # but let's directly verify it. another_onode = next(self.object_ring.get_more_nodes(opart)) odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # Assert container listing (via proxy and directly) has container/obj objs = [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: objs = [ o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1] ] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % (cnode['ip'], cnode['port'])) # Bring the first container/obj primary server back up start_server((onode['ip'], onode['port']), self.ipport2server, self.pids) # Assert that it doesn't have container/obj yet try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=node_id) try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] another_num = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_num) # Assert the first container/obj primary server now has container/obj odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # Assert the handoff server no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Kill the first container/obj primary server again (we have two # primaries and the handoff up now) kill_server((onode['ip'], onode['port']), self.ipport2server, self.pids) # Delete container/obj try: client.delete_object(self.url, self.token, container, obj) except client.ClientException as err: if self.object_ring.replica_count > 2: raise # Object DELETE returning 503 for (404, 204) # remove this with fix for # https://bugs.launchpad.net/swift/+bug/1318375 self.assertEqual(503, err.http_status) # Assert we can't head container/obj try: client.head_object(self.url, self.token, container, obj) except client.ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Assert container/obj is not in the container listing, both indirectly # and directly objs = [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ] if obj in objs: raise Exception('Container listing still knew about object') for cnode in cnodes: objs = [ o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1] ] if obj in objs: raise Exception( 'Container server %s:%s still knew about object' % (cnode['ip'], cnode['port'])) # Restart the first container/obj primary server again start_server((onode['ip'], onode['port']), self.ipport2server, self.pids) # Assert it still has container/obj direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx}) # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=node_id) another_node_id = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_node_id) # Assert primary node no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it")
def _test_main(self, cancel=False): container = 'container-%s' % uuid4() obj = 'object-%s' % uuid4() obj2 = 'object-%s' % uuid4() # Create container headers = {'X-Storage-Policy': self.policy.name} client.put_container(self.url, self.token, container, headers=headers) # Create a new object client.put_object(self.url, self.token, container, obj, self.data) client.head_object(self.url, self.token, container, obj) # Prepare partition power increase builder = RingBuilder.load(self.builder_file) builder.prepare_increase_partition_power() builder.save(self.builder_file) ring_data = builder.get_ring() ring_data.save(self.ring_file) # Ensure the proxy uses the changed ring Manager(['proxy']).restart() # Ensure object is still accessible client.head_object(self.url, self.token, container, obj) # Relink existing objects for device in self.devices: self.assertEqual(0, relink(skip_mount_check=True, devices=device)) # Create second object after relinking and ensure it is accessible client.put_object(self.url, self.token, container, obj2, self.data) client.head_object(self.url, self.token, container, obj2) # Remember the original object locations org_locations = self._find_objs_ondisk(container, obj) org_locations += self._find_objs_ondisk(container, obj2) # Remember the new object locations new_locations = [] for loc in org_locations: new_locations.append( replace_partition_in_path(str(loc), self.object_ring.part_power + 1)) # Overwrite existing object - to ensure that older timestamp files # will be cleaned up properly later client.put_object(self.url, self.token, container, obj, self.data) # Ensure objects are still accessible client.head_object(self.url, self.token, container, obj) client.head_object(self.url, self.token, container, obj2) # Increase partition power builder = RingBuilder.load(self.builder_file) if not cancel: builder.increase_partition_power() else: builder.cancel_increase_partition_power() builder.save(self.builder_file) ring_data = builder.get_ring() ring_data.save(self.ring_file) # Ensure the proxy uses the changed ring Manager(['proxy']).restart() # Ensure objects are still accessible client.head_object(self.url, self.token, container, obj) client.head_object(self.url, self.token, container, obj2) # Overwrite existing object - to ensure that older timestamp files # will be cleaned up properly later client.put_object(self.url, self.token, container, obj, self.data) # Cleanup old objects in the wrong location for device in self.devices: self.assertEqual(0, cleanup(skip_mount_check=True, devices=device)) # Ensure objects are still accessible client.head_object(self.url, self.token, container, obj) client.head_object(self.url, self.token, container, obj2) # Ensure data in old or relinked object locations is removed if not cancel: for fn in org_locations: self.assertFalse(os.path.exists(fn)) else: for fn in new_locations: self.assertFalse(os.path.exists(fn))
def test_ec_handoff_overwrite(self): container_name = 'container-%s' % uuid4() object_name = 'object-%s' % uuid4() # create EC container headers = {'X-Storage-Policy': self.policy.name} client.put_container(self.url, self.token, container_name, headers=headers) # PUT object old_contents = Body() client.put_object(self.url, self.token, container_name, object_name, contents=old_contents) # get our node lists opart, onodes = self.object_ring.get_nodes(self.account, container_name, object_name) # shutdown one of the primary data nodes failed_primary = random.choice(onodes) failed_primary_device_path = self.device_dir('object', failed_primary) self.kill_drive(failed_primary_device_path) # overwrite our object with some new data new_contents = Body() client.put_object(self.url, self.token, container_name, object_name, contents=new_contents) self.assertNotEqual(new_contents.etag, old_contents.etag) # restore failed primary device self.revive_drive(failed_primary_device_path) # sanity - failed node has old contents req_headers = {'X-Backend-Storage-Policy-Index': int(self.policy)} headers = direct_client.direct_head_object(failed_primary, opart, self.account, container_name, object_name, headers=req_headers) self.assertEqual(headers['X-Object-Sysmeta-EC-Etag'], old_contents.etag) # we have 1 primary with wrong old etag, and we should have 5 with # new etag plus a handoff with the new etag, so killing 2 other # primaries forces proxy to try to GET from all primaries plus handoff. other_nodes = [n for n in onodes if n != failed_primary] random.shuffle(other_nodes) for node in other_nodes[:2]: self.kill_drive(self.device_dir('object', node)) # sanity, after taking out two primaries we should be down to # only four primaries, one of which has the old etag - but we # also have a handoff with the new etag out there found_frags = defaultdict(int) req_headers = {'X-Backend-Storage-Policy-Index': int(self.policy)} for node in onodes + list(self.object_ring.get_more_nodes(opart)): try: headers = direct_client.direct_head_object(node, opart, self.account, container_name, object_name, headers=req_headers) except Exception: continue found_frags[headers['X-Object-Sysmeta-EC-Etag']] += 1 self.assertEqual( found_frags, { new_contents.etag: 4, # this should be enough to rebuild! old_contents.etag: 1, }) # clear node error limiting Manager(['proxy']).restart() resp_etag = self.get_object(container_name, object_name) self.assertEqual(resp_etag, new_contents.etag)
def test_reconcile_symlink(self): if 'symlink' not in self.cluster_info: raise unittest.SkipTest("Symlink not enabled in proxy; can't test " "symlink reconciliation") wrong_policy = random.choice(ENABLED_POLICIES) policy = random.choice( [p for p in ENABLED_POLICIES if p is not wrong_policy]) # get an old container stashed self.brain.stop_primary_half() self.brain.put_container(int(policy)) self.brain.start_primary_half() # write some target data client.put_object(self.url, self.token, self.container_name, 'target', contents=b'this is the target data') # write the symlink self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) client.put_object(self.url, self.token, self.container_name, 'symlink', headers={ 'X-Symlink-Target': '%s/target' % self.container_name, 'Content-Type': 'application/symlink', }) # at this point we have a broken symlink (the container_info has the # proxy looking for the target in the wrong policy) with self.assertRaises(ClientException) as ctx: client.get_object(self.url, self.token, self.container_name, 'symlink') self.assertEqual(ctx.exception.http_status, 404) # of course the symlink itself is fine metadata, body = client.get_object(self.url, self.token, self.container_name, 'symlink', query_string='symlink=get') self.assertEqual(metadata['x-symlink-target'], '%s/target' % self.container_name) self.assertEqual(metadata['content-type'], 'application/symlink') self.assertEqual(body, b'') # ... although in the wrong policy object_ring = POLICIES.get_object_ring(int(wrong_policy), '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, 'symlink') for node in nodes: metadata = direct_client.direct_head_object( node, part, self.account, self.container_name, 'symlink', headers={'X-Backend-Storage-Policy-Index': int(wrong_policy)}) self.assertEqual(metadata['X-Object-Sysmeta-Symlink-Target'], '%s/target' % self.container_name) # let the reconciler run self.brain.start_handoff_half() self.get_to_final_state() Manager(['container-reconciler']).once() # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # now the symlink works metadata, body = client.get_object(self.url, self.token, self.container_name, 'symlink') self.assertEqual(body, b'this is the target data') # and it's in the correct policy object_ring = POLICIES.get_object_ring(int(policy), '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, 'symlink') for node in nodes: metadata = direct_client.direct_head_object( node, part, self.account, self.container_name, 'symlink', headers={'X-Backend-Storage-Policy-Index': int(policy)}) self.assertEqual(metadata['X-Object-Sysmeta-Symlink-Target'], '%s/target' % self.container_name)
def test_sync_with_stale_container_rows(self): source_container, dest_container = self._setup_synced_containers() brain = BrainSplitter(self.url, self.token, source_container, None, 'container') # upload to source object_name = 'object-%s' % uuid.uuid4() client.put_object(self.url, self.token, source_container, object_name, 'test-body') # check source container listing _, listing = client.get_container( self.url, self.token, source_container) for expected_obj_dict in listing: if expected_obj_dict['name'] == object_name: break else: self.fail('Failed to find source object %r in container listing %r' % (object_name, listing)) # stop all container servers brain.stop_primary_half() brain.stop_handoff_half() # upload new object content to source - container updates will fail client.put_object(self.url, self.token, source_container, object_name, 'new-test-body') source_headers = client.head_object( self.url, self.token, source_container, object_name) # start all container servers brain.start_primary_half() brain.start_handoff_half() # sanity check: source container listing should not have changed _, listing = client.get_container( self.url, self.token, source_container) for actual_obj_dict in listing: if actual_obj_dict['name'] == object_name: self.assertDictEqual(expected_obj_dict, actual_obj_dict) break else: self.fail('Failed to find source object %r in container listing %r' % (object_name, listing)) # cycle container-sync - object should be correctly sync'd despite # stale info in container row Manager(['container-sync']).once() # verify sync'd object has same content and headers dest_headers, body = client.get_object(self.url, self.token, dest_container, object_name) self.assertEqual(body, 'new-test-body') mismatched_headers = [] for k in ('etag', 'content-length', 'content-type', 'x-timestamp', 'last-modified'): if source_headers[k] == dest_headers[k]: continue mismatched_headers.append((k, source_headers[k], dest_headers[k])) if mismatched_headers: msg = '\n'.join([('Mismatched header %r, expected %r but got %r' % item) for item in mismatched_headers]) self.fail(msg)
def invoke_storlet(self, op, params=None, global_params=None, header_parameters=False): headers = {'X-Run-Storlet': self.storlet_name} headers.update(self.additional_headers) if params is not None: if header_parameters: querystring = None count = 1 for key in params: headers['X-Storlet-Parameter-' + str(count)] = \ key + ':' + params[key] count = count + 1 else: querystring = '&'.join( ['%s=%s' % (k, v) for k, v in params.items()]) else: querystring = None if op == 'GET': # Get original object original_h, original_c = c.get_object(self.url, self.token, self.container, self.storlet_file, response_dict=dict()) # print original_headers file_length = int(original_h['content-length']) processed_h, returned_c = c.get_object(self.url, self.token, self.container, self.storlet_file, query_string=querystring, response_dict=dict(), headers=headers, resp_chunk_size=file_length) processed_c = b'' for chunk in returned_c: if chunk: processed_c += chunk if params is not None and params.get('execute', None) is not None: mdv = processed_h['X-Object-Meta-Execution-Result'.lower()] self.assertEqual('42', mdv) if params is not None and params.get('double', None) == 'true': self.assertEqual(original_c, processed_c[:file_length]) self.assertEqual(original_c, processed_c[file_length:]) else: self.assertEqual(original_c, processed_c) self.assertEqual(original_h['X-Object-Meta-Testkey'.lower()], processed_h['X-Object-Meta-Testkey'.lower()]) if op == 'PUT': # PUT a random file response = dict() uploaded_c = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(1024)).encode('ascii') random_md = ''.join( random.choice(string.ascii_uppercase + string.digits) for _ in range(32)) content_length = None headers['X-Object-Meta-Testkey'] = random_md c.put_object(self.url, self.token, self.container, 'identity_random_source', uploaded_c, content_length, None, None, "application/octet-stream", headers, None, None, querystring, response) resp_headers, saved_c = c.get_object(self.url, self.token, self.container, 'identity_random_source', response_dict=dict()) if params is not None and params.get('double', None) == 'true': assert (uploaded_c == saved_c[:1024]) assert (uploaded_c == saved_c[1024:]) else: assert (uploaded_c == saved_c) if params is not None and params.get('execute', None) is not None: mdv = resp_headers['X-Object-Meta-Execution-Result'.lower()] self.assertEqual('42', mdv) self.assertEqual(random_md, resp_headers['X-Object-Meta-Testkey'.lower()])
def test_sync_slo_manifest(self): # Verify that SLO manifests are sync'd even if their segments can not # be found in the destination account at time of sync'ing. # Create source and dest containers for manifest in separate accounts. dest_account = self.account_2 source_container, dest_container = self._setup_synced_containers( dest_overrides=dest_account ) # Create source and dest containers for segments in separate accounts. # These containers must have same name for the destination SLO manifest # to be able to resolve segments. Initially the destination has no sync # key so segments will not sync. segs_container = 'segments-%s' % uuid.uuid4() dest_segs_info = dict(dest_account) dest_segs_info.update({'name': segs_container, 'sync_key': None}) self._setup_synced_containers( source_overrides={'name': segs_container, 'sync_key': 'segs_key'}, dest_overrides=dest_segs_info) # upload a segment to source segment_name = 'segment-%s' % uuid.uuid4() segment_data = 'segment body' # it's ok for first segment to be small segment_etag = client.put_object( self.url, self.token, segs_container, segment_name, segment_data) manifest = [{'etag': segment_etag, 'size_bytes': len(segment_data), 'path': '/%s/%s' % (segs_container, segment_name)}] manifest_name = 'manifest-%s' % uuid.uuid4() put_headers = {'X-Object-Meta-Test': 'put_value'} client.put_object( self.url, self.token, source_container, manifest_name, json.dumps(manifest), headers=put_headers, query_string='multipart-manifest=put') resp_headers, manifest_body = client.get_object( self.url, self.token, source_container, manifest_name, query_string='multipart-manifest=get') int_manifest = json.loads(manifest_body) # cycle container-sync Manager(['container-sync']).once() # verify manifest was sync'd resp_headers, dest_listing = client.get_container( dest_account['url'], dest_account['token'], dest_container) self.assertFalse(dest_listing[1:]) self.assertEqual(manifest_name, dest_listing[0]['name']) # verify manifest body resp_headers, body = client.get_object( dest_account['url'], dest_account['token'], dest_container, manifest_name, query_string='multipart-manifest=get') self.assertEqual(int_manifest, json.loads(body)) self.assertIn('x-object-meta-test', resp_headers) self.assertEqual('put_value', resp_headers['x-object-meta-test']) # attempt to GET the SLO will fail because the segment wasn't sync'd with self.assertRaises(ClientException) as cm: client.get_object(dest_account['url'], dest_account['token'], dest_container, manifest_name) self.assertEqual(409, cm.exception.http_status) # now set sync key on destination segments container client.put_container( dest_account['url'], dest_account['token'], segs_container, headers={'X-Container-Sync-Key': 'segs_key'}) # cycle container-sync Manager(['container-sync']).once() # sanity check - verify manifest body resp_headers, body = client.get_object( dest_account['url'], dest_account['token'], dest_container, manifest_name, query_string='multipart-manifest=get') self.assertEqual(int_manifest, json.loads(body)) self.assertIn('x-object-meta-test', resp_headers) self.assertEqual('put_value', resp_headers['x-object-meta-test']) # verify GET of SLO manifest now succeeds resp_headers, body = client.get_object( dest_account['url'], dest_account['token'], dest_container, manifest_name) self.assertEqual(segment_data, body)
def test_delete_propogate(self): # create EC container headers = {'X-Storage-Policy': self.policy.name} client.put_container(self.url, self.token, self.container_name, headers=headers) # get our node lists opart, onodes = self.object_ring.get_nodes(self.account, self.container_name, self.object_name) hnodes = self.object_ring.get_more_nodes(opart) p_dev2 = self.device_dir('object', onodes[1]) # PUT object contents = Body() client.put_object(self.url, self.token, self.container_name, self.object_name, contents=contents) # now lets shut one down self.kill_drive(p_dev2) # delete on the ones that are left client.delete_object(self.url, self.token, self.container_name, self.object_name) # spot check a node try: self.direct_get(onodes[0], opart) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 404) else: self.fail('Node data on %r was not fully destoryed!' % (onodes[0], )) # enable the first node again self.revive_drive(p_dev2) # propagate the delete... # fire up reconstructor on handoff nodes only for hnode in hnodes: hnode_id = (hnode['port'] - 6000) / 10 self.reconstructor.once(number=hnode_id) # check the first node to make sure its gone try: self.direct_get(onodes[1], opart) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 404) else: self.fail('Node data on %r was not fully destoryed!' % (onodes[0])) # make sure proxy get can't find it try: self.proxy_get() except Exception as err: self.assertEqual(err.http_status, 404) else: self.fail('Node data on %r was not fully destoryed!' % (onodes[0]))
def test_metadata_sync(self): # Create container container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': self.policy.name, 'X-Container-Meta-A': '1', 'X-Container-Meta-B': '1', 'X-Container-Meta-C': '1'}) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes.pop() # 2 of 3 container servers are temporarily down for node in cnodes: kill_server((node['ip'], node['port']), self.ipport2server) # Put some meta on the lone server, to make sure it's merged properly # This will 503 (since we don't have a quorum), but we don't care (!) try: client.post_container(self.url, self.token, container, headers={'X-Container-Meta-A': '2', 'X-Container-Meta-B': '2', 'X-Container-Meta-D': '2'}) except ClientException: pass # object updates come to only one container server for _ in range(self.object_puts): obj = 'object-%s' % uuid4() client.put_object(self.url, self.token, container, obj, 'VERIFY') # 2 container servers make comeback for node in cnodes: start_server((node['ip'], node['port']), self.ipport2server) # But, container-server which got object updates is down kill_server((cnode['ip'], cnode['port']), self.ipport2server) # Metadata update will be applied to 2 container servers # (equal to quorum) client.post_container(self.url, self.token, container, headers={'X-Container-Meta-B': '3', 'X-Container-Meta-E': '3'}) # container-server which got object updates makes comeback start_server((cnode['ip'], cnode['port']), self.ipport2server) # other nodes have no objects for node in cnodes: resp_headers = direct_client.direct_head_container( node, cpart, self.account, container) self.assertIn(resp_headers.get('x-container-object-count'), (None, '0', 0)) # If container-replicator on the node which got the object updates # runs in first, db file may be replicated by rsync to other # containers. In that case, the db file does not information about # metadata, so metadata should be synced before replication Manager(['container-replicator']).once( number=self.config_number(cnode)) expected_meta = { 'x-container-meta-a': '2', 'x-container-meta-b': '3', 'x-container-meta-c': '1', 'x-container-meta-d': '2', 'x-container-meta-e': '3', } # node that got the object updates now has the meta resp_headers = direct_client.direct_head_container( cnode, cpart, self.account, container) for header, value in expected_meta.items(): self.assertIn(header, resp_headers) self.assertEqual(value, resp_headers[header]) self.assertNotIn(resp_headers.get('x-container-object-count'), (None, '0', 0)) # other nodes still have the meta, as well as objects for node in cnodes: resp_headers = direct_client.direct_head_container( node, cpart, self.account, container) for header, value in expected_meta.items(): self.assertIn(header, resp_headers) self.assertEqual(value, resp_headers[header]) self.assertNotIn(resp_headers.get('x-container-object-count'), (None, '0', 0)) # and after full pass on remaining nodes for node in cnodes: Manager(['container-replicator']).once( number=self.config_number(node)) # ... all is right for node in cnodes + [cnode]: resp_headers = direct_client.direct_head_container( node, cpart, self.account, container) for header, value in expected_meta.items(): self.assertIn(header, resp_headers) self.assertEqual(value, resp_headers[header]) self.assertNotIn(resp_headers.get('x-container-object-count'), (None, '0', 0))
def test_reconcile_manifest(self): info_url = "%s://%s/info" % (urlparse( self.url).scheme, urlparse(self.url).netloc) proxy_conn = client.http_connection(info_url) cluster_info = client.get_capabilities(proxy_conn) if 'slo' not in cluster_info: raise SkipTest("SLO not enabled in proxy; " "can't test manifest reconciliation") # this test is not only testing a split brain scenario on # multiple policies with mis-placed objects - it even writes out # a static large object directly to the storage nodes while the # objects are unavailably mis-placed from *behind* the proxy and # doesn't know how to do that for EC_POLICY (clayg: why did you # guys let me write a test that does this!?) - so we force # wrong_policy (where the manifest gets written) to be one of # any of your configured REPL_POLICY (we know you have one # because this is a ReplProbeTest) wrong_policy = random.choice(POLICIES_BY_TYPE[REPL_POLICY]) policy = random.choice( [p for p in ENABLED_POLICIES if p is not wrong_policy]) manifest_data = [] def write_part(i): body = 'VERIFY%0.2d' % i + '\x00' * 1048576 part_name = 'manifest_part_%0.2d' % i manifest_entry = { "path": "/%s/%s" % (self.container_name, part_name), "etag": md5(body).hexdigest(), "size_bytes": len(body), } client.put_object(self.url, self.token, self.container_name, part_name, contents=body) manifest_data.append(manifest_entry) # get an old container stashed self.brain.stop_primary_half() self.brain.put_container(int(policy)) self.brain.start_primary_half() # write some parts for i in range(10): write_part(i) self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) # write some more parts for i in range(10, 20): write_part(i) # write manifest with self.assertRaises(ClientException) as catcher: client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') # so as it works out, you can't really upload a multi-part # manifest for objects that are currently misplaced - you have to # wait until they're all available - which is about the same as # some other failure that causes data to be unavailable to the # proxy at the time of upload self.assertEqual(catcher.exception.http_status, 400) # but what the heck, we'll sneak one in just to see what happens... direct_manifest_name = self.object_name + '-direct-test' object_ring = POLICIES.get_object_ring(wrong_policy.idx, '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, direct_manifest_name) container_part = self.container_ring.get_part(self.account, self.container_name) def translate_direct(data): return { 'hash': data['etag'], 'bytes': data['size_bytes'], 'name': data['path'], } direct_manifest_data = map(translate_direct, manifest_data) headers = { 'x-container-host': ','.join('%s:%s' % (n['ip'], n['port']) for n in self.container_ring.devs), 'x-container-device': ','.join(n['device'] for n in self.container_ring.devs), 'x-container-partition': container_part, 'X-Backend-Storage-Policy-Index': wrong_policy.idx, 'X-Static-Large-Object': 'True', } for node in nodes: direct_client.direct_put_object( node, part, self.account, self.container_name, direct_manifest_name, contents=utils.json.dumps(direct_manifest_data), headers=headers) break # one should do it... self.brain.start_handoff_half() self.get_to_final_state() Manager(['container-reconciler']).once() # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # let's see how that direct upload worked out... metadata, body = client.get_object( self.url, self.token, self.container_name, direct_manifest_name, query_string='multipart-manifest=get') self.assertEqual(metadata['x-static-large-object'].lower(), 'true') for i, entry in enumerate(utils.json.loads(body)): for key in ('hash', 'bytes', 'name'): self.assertEqual(entry[key], direct_manifest_data[i][key]) metadata, body = client.get_object(self.url, self.token, self.container_name, direct_manifest_name) self.assertEqual(metadata['x-static-large-object'].lower(), 'true') self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data)) self.assertEqual( body, ''.join('VERIFY%0.2d' % i + '\x00' * 1048576 for i in range(20))) # and regular upload should work now too client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data))
def test_revert_object(self): # create EC container headers = {'X-Storage-Policy': self.policy.name} client.put_container(self.url, self.token, self.container_name, headers=headers) # get our node lists opart, onodes = self.object_ring.get_nodes(self.account, self.container_name, self.object_name) hnodes = self.object_ring.get_more_nodes(opart) # kill 2 a parity count number of primary nodes so we can # force data onto handoffs, we do that by renaming dev dirs # to induce 507 p_dev1 = self.device_dir(onodes[0]) p_dev2 = self.device_dir(onodes[1]) self.kill_drive(p_dev1) self.kill_drive(p_dev2) # PUT object contents = Body() headers = {'x-object-meta-foo': 'meta-foo'} headers_post = {'x-object-meta-bar': 'meta-bar'} client.put_object(self.url, self.token, self.container_name, self.object_name, contents=contents, headers=headers) client.post_object(self.url, self.token, self.container_name, self.object_name, headers=headers_post) # (Some versions of?) swiftclient will mutate the headers dict on post headers_post.pop('X-Auth-Token', None) # these primaries can't serve the data any more, we expect 507 # here and not 404 because we're using mount_check to kill nodes for onode in (onodes[0], onodes[1]): self.assert_direct_get_fails(onode, opart, 507) # now take out another primary p_dev3 = self.device_dir(onodes[2]) self.kill_drive(p_dev3) # this node can't serve the data any more self.assert_direct_get_fails(onodes[2], opart, 507) # make sure we can still GET the object and its correct # we're now pulling from handoffs and reconstructing _headers, etag = self.proxy_get() self.assertEqual(etag, contents.etag) # rename the dev dirs so they don't 507 anymore self.revive_drive(p_dev1) self.revive_drive(p_dev2) self.revive_drive(p_dev3) # fire up reconstructor on handoff nodes only for hnode in hnodes: hnode_id = (hnode['port'] % 100) // 10 self.reconstructor.once(number=hnode_id) # first three primaries have data again for onode in (onodes[0], onodes[2]): self.assert_direct_get_succeeds(onode, opart) # check meta meta = client.head_object(self.url, self.token, self.container_name, self.object_name) for key in headers_post: self.assertIn(key, meta) self.assertEqual(meta[key], headers_post[key]) # handoffs are empty for hnode in hnodes: self.assert_direct_get_fails(hnode, opart, 404)