def getMember(self, objectname): """Get member for this ObjectCollection. Checks if requested name is a subdir (see above) """ if self.prefix and self.prefix not in objectname: objectname = self.prefix + objectname if self.is_subdir(objectname): return ObjectCollection(self.container, self.environ, prefix=objectname) if self.environ.get('REQUEST_METHOD') in ['PUT']: return ObjectResource(self.container, objectname, self.environ, self.objects) try: client.head_object(self.storage_url, self.auth_token, self.container, objectname, http_conn=self.http_connection) return ObjectResource(self.container, objectname, self.environ, self.objects) except client.ClientException: pass return None
def handleCopy(self, destPath, depthInfinity): dst = '/'.join(destPath.split('/')[2:]) try: client.head_object(self.storage_url, self.auth_token, self.container, dst, http_conn=self.http_connection) except client.ClientException: pass headers = {'X-Copy-From': self.path} try: client.put_object(self.storage_url, self.auth_token, self.container, dst, headers=headers, http_conn=self.http_connection) if self.environ.get("HTTP_OVERWRITE", '') != "T": raise DAVError(HTTP_CREATED) return True except client.ClientException: return False
def createCollection(self, name): """Create a pseudo-folder.""" if self.path: tmp = self.path.split('/') name = '/'.join(tmp[2:]) + '/' + name name = name.strip('/') try: client.head_object(self.storage_url, self.auth_token, self.container, name, http_conn=self.http_connection) raise dav_error.DAVError(dav_error.HTTP_METHOD_NOT_ALLOWED) except client.ClientException: pass try: client.head_object(self.storage_url, self.auth_token, self.container, name + '/', http_conn=self.http_connection) raise dav_error.DAVError(dav_error.HTTP_METHOD_NOT_ALLOWED) except client.ClientException: pass client.put_object(self.storage_url, self.auth_token, self.container, name + '/', content_type='application/directory', http_conn=self.http_connection)
def createCollection(self, name): """Create a pseudo-folder.""" if self.path: tmp = self.path.split('/') name = '/'.join(tmp[2:]) + '/' + name name = name.strip('/') try: client.head_object(self.storage_url, self.auth_token, self.container, name, http_conn=self.http_connection) raise dav_error.DAVError(dav_error.HTTP_METHOD_NOT_ALLOWED) except client.ClientException: pass try: client.head_object(self.storage_url, self.auth_token, self.container, name + '/', http_conn=self.http_connection) raise dav_error.DAVError(dav_error.HTTP_METHOD_NOT_ALLOWED) except client.ClientException: pass client.put_object(self.storage_url, self.auth_token, self.container, sanitize(name).rstrip('/') + '/', content_type='application/directory', http_conn=self.http_connection)
def get_status(request, container, object_name): """Download an object from Swift :param container: container of swift where object is stored :param object_name: ID of object :return: object bytestream """ try: if not request.session.get('storage_url') and not request.session.get('auth_token'): return Response('Please Contact your administrator', status=status.HTTP_401_UNAUTHORIZED) storage_url, auth_token = _get_auth_data(request.session) obj = {object_name: {}} # TODO: Currently using swift as first option, switch to in-memory headers = client.head_object(storage_url, auth_token, container, object_name) vid_format = 'mp4' id_header = 'x-object-meta-' + vid_format + '-id' if id_header in headers: video_headers = client.head_object(storage_url, auth_token, container, headers[id_header]) obj[object_name][vid_format + 'status'] = video_headers['x-object-meta-status'] if 'x-object-meta-status' in video_headers else 0 else: print "DOESN'T EXIST" # print video_keys obj[object_name]['status'] = 'DONE' return Response(obj, status=status.HTTP_200_OK) return Response(obj) # if object_name in video_keys: # keys = video_keys[object_name] # obj = {object_name: keys} # print obj # return Response(obj) except swift_exception.ClientException as e: print e return Response('Please Contact Your Administrator', status=status.HTTP_500_INTERNAL_SERVER_ERROR) except Exception as e: print e return Response(e.message, status=status.HTTP_500_INTERNAL_SERVER_ERROR)
def exists(self, name): """ Returns True if a file referenced by the given name already exists in the storage system, or False if the name is available for a new file. """ try: (host, port, path, is_ssl) = self.connection.connection_args storage_url = 'https://%s:%d/%s' % (host, port, path) head_object(storage_url, self.connection.token, self.container_name, name) return True except ClientException: return False
def get_headers(self): """Execute HEAD object request. Since this info is used in different methods (see below), do it once and then use this info. """ if self.headers is None: data = self.objects.get(self.objectname) if data: self.headers = { 'content-length': data.get('bytes'), 'etag': data.get('hash'), } else: try: self.headers = client.head_object( self.storage_url, self.auth_token, self.container, self.objectname, http_conn=self.http_connection) except client.ClientException: self.headers = {} pass
def _set_headers(storage_url, auth_token, container, object_name, deleted): try: header = client.head_object(storage_url, auth_token, container, object_name) new_header = { 'X-Object-Meta-Deleted': deleted, 'X-Object-Meta-Format': header['x-object-meta-format'], 'X-Object-Meta-Resolution': header['x-object-meta-resolution'], 'X-Object-Meta-Name': header['x-object-meta-name'], 'X-Object-Meta-Type': header['x-object-meta-type'] } copy_of = None if header['x-object-meta-type'] == 'thumbnail': new_header['X-Object-Meta-Original'] = header[ 'x-object-meta-original'] copy_of = header['x-object-meta-original'] if header['x-object-meta-type'] == 'original': new_header['X-Object-Meta-Thumb'] = header['x-object-meta-thumb'] copy_of = header['x-object-meta-thumb'] client.post_object(storage_url, auth_token, container, object_name, headers=new_header) return copy_of except swift_exception.ClientException as e: raise e except Exception as e: raise e
def get_headers(self): """Execute HEAD object request. Since this info is used in different methods (see below), do it once and then use this info. """ if self.headers is None: data = self.objects.get(self.objectname) if data: self.headers = {'content-length': data.get('bytes'), 'etag': data.get('hash'), 'last_modified': data.get('last_modified'), } else: try: self.headers = client.head_object( self.storage_url, self.auth_token, self.container, self.objectname, http_conn=self.http_connection) except client.ClientException: self.headers = {} pass
def test_async_updates_after_PUT_and_POST(self): # verify correct update values when PUT update and POST updates are # missed but then async updates are sent cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # PUT and POST to object while one container server is stopped so that # we force async updates to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content, content_type='test/ctype') meta = client.head_object(self.url, self.token, 'c1', 'o1') # use internal client for POST so we can force fast-post mode int_client = self.make_internal_client(object_post_as_copy=False) int_client.set_object_metadata(self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'}) self.assertEqual('Tomato', int_client.get_object_metadata( self.account, 'c1', 'o1')['x-object-meta-fruit']) # sanity # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse( direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1]) # Run the object-updaters to send the async pendings Manager(['object-updater']).once() # check the re-started container server got same update as others. # we cannot assert the actual etag value because it may be encrypted listing_etags = set() for cnode in cnodes: listing = direct_client.direct_get_container( cnode, cpart, self.account, 'c1')[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual('test/ctype', listing[0]['content_type']) listing_etags.add(listing[0]['hash']) self.assertEqual(1, len(listing_etags)) # check that listing meta returned to client is consistent with object # meta returned to client hdrs, listing = client.get_container(self.url, self.token, 'c1') self.assertEqual(1, len(listing)) self.assertEqual('o1', listing[0]['name']) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual(meta['etag'], listing[0]['hash']) self.assertEqual('test/ctype', listing[0]['content_type'])
def _test_sync(self, object_post_as_copy): source_container, dest_container = self._setup_synced_containers() # upload to source object_name = 'object-%s' % uuid.uuid4() put_headers = {'X-Object-Meta-Test': 'put_value'} client.put_object(self.url, self.token, source_container, object_name, 'test-body', headers=put_headers) # cycle container-sync Manager(['container-sync']).once() resp_headers, body = client.get_object(self.url, self.token, dest_container, object_name) self.assertEqual(body, 'test-body') self.assertIn('x-object-meta-test', resp_headers) self.assertEqual('put_value', resp_headers['x-object-meta-test']) # update metadata with a POST, using an internal client so we can # vary the object_post_as_copy setting - first use post-as-copy post_headers = {'Content-Type': 'image/jpeg', 'X-Object-Meta-Test': 'post_value'} int_client = self.make_internal_client( object_post_as_copy=object_post_as_copy) int_client.set_object_metadata(self.account, source_container, object_name, post_headers) # sanity checks... resp_headers = client.head_object( self.url, self.token, source_container, object_name) self.assertIn('x-object-meta-test', resp_headers) self.assertEqual('post_value', resp_headers['x-object-meta-test']) self.assertEqual('image/jpeg', resp_headers['content-type']) # cycle container-sync Manager(['container-sync']).once() # verify that metadata changes were sync'd resp_headers, body = client.get_object(self.url, self.token, dest_container, object_name) self.assertEqual(body, 'test-body') self.assertIn('x-object-meta-test', resp_headers) self.assertEqual('post_value', resp_headers['x-object-meta-test']) self.assertEqual('image/jpeg', resp_headers['content-type']) # delete the object client.delete_object( self.url, self.token, source_container, object_name) with self.assertRaises(ClientException) as cm: client.get_object( self.url, self.token, source_container, object_name) self.assertEqual(404, cm.exception.http_status) # sanity check # cycle container-sync Manager(['container-sync']).once() # verify delete has been sync'd with self.assertRaises(ClientException) as cm: client.get_object( self.url, self.token, dest_container, object_name) self.assertEqual(404, cm.exception.http_status) # sanity check
def _check_node(self, node, part, etag, headers_post): # get fragment archive etag fragment_archive_etag = self.direct_get(node, part) # remove data from the selected node part_dir = self.storage_dir('object', node, part=part) shutil.rmtree(part_dir, True) # this node can't servce the data any more try: self.direct_get(node, part) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 404) else: self.fail('Node data on %r was not fully destoryed!' % (node, )) # make sure we can still GET the object and its correct, the # proxy is doing decode on remaining fragments to get the obj self.assertEqual(etag, self.proxy_get()) # fire up reconstructor self.reconstructor.once() # fragment is rebuilt exactly as it was before! self.assertEqual(fragment_archive_etag, self.direct_get(node, part)) # check meta meta = client.head_object(self.url, self.token, self.container_name, self.object_name) for key in headers_post: self.assertIn(key, meta) self.assertEqual(meta[key], headers_post[key])
def test_copy_from(self): resp = dict() objname = self.storlet_file + '-copy' req_headers = { 'X-Run-Storlet': self.storlet_name, 'X-Copy-From': '%s/%s' % (self.container, self.storlet_file) } client.put_object(self.url, self.token, self.container, objname, self.content, response_dict=resp, headers=req_headers) self.assertEqual(201, resp['status']) resp_header = resp['headers'] self.assertEqual('%s/%s' % (self.container, self.storlet_file), resp_header['x-storlet-generated-from']) self.assertEqual(self.acct, resp_header['x-storlet-generated-from-account']) self.assertIn('x-storlet-generated-from-last-modified', resp_header) headers = client.head_object(self.url, self.token, self.container, objname) self.assertEqual(str(len(self.content)), headers['content-length']) resp = dict() client.delete_object(self.url, self.token, self.container, objname, response_dict=resp) self.assertEqual(204, resp['status'])
def _check_node(self, node, part, etag, headers_post): # get fragment archive etag fragment_archive_etag = self.direct_get(node, part) # remove the .durable from the selected node part_dir = self.storage_dir('object', node, part=part) for dirs, subdirs, files in os.walk(part_dir): for fname in files: if fname.endswith('.durable'): durable = os.path.join(dirs, fname) os.remove(durable) break try: os.remove(os.path.join(part_dir, 'hashes.pkl')) except OSError as e: if e.errno != errno.ENOENT: raise # fire up reconstructor to propogate the .durable self.reconstructor.once() # fragment is still exactly as it was before! self.assertEqual(fragment_archive_etag, self.direct_get(node, part)) # check meta meta = client.head_object(self.url, self.token, self.container_name, self.object_name) for key in headers_post: self.assertTrue(key in meta) self.assertEqual(meta[key], headers_post[key])
def get_all(request, container): """ Returns List of all Objects present in specified container :param container: Name of Swift Container :return: meta data and id's of all the objects """ try: response_dict = dict() storage_url, auth_token = _get_auth_data(request.session) data_container = client.get_container(storage_url, auth_token, container) for obj in data_container[1]: meta_object = client.head_object(storage_url, auth_token, container, obj['name']) if not meta_object['x-object-meta-deleted']: if meta_object['x-object-meta-type'] in ['thumbnail', 'original-thumbnail']: if meta_object['x-object-meta-format'] not in response_dict: form = meta_object['x-object-meta-format'] response_dict[form] = [] if meta_object['x-object-meta-format'] in response_dict: new_obj = {'thumbnail_id': obj['name'], 'name': meta_object['x-object-meta-name'], 'type': meta_object['x-object-meta-type'], 'resolution': meta_object['x-object-meta-resolution']} if meta_object['x-object-meta-format'] is 'thumbnail': new_obj['original_id'] = meta_object['x-object-meta-original'] response_dict[form].append(new_obj) return Response(response_dict) except swift_exception.ClientException as e: print e return Response('Please contact your admininstrator', status=status.HTTP_500_INTERNAL_SERVER_ERROR) except Exception as e: print e return Response('Please contact your admininstrator', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
def test_copy_dest(self): # No COPY in swiftclient. Using urllib instead... url = os.path.join(self.url, self.container, self.storlet_file) objname = self.storlet_file + '-copy-ex' headers = { 'X-Auth-Token': self.token, 'X-Run-Storlet': self.storlet_name, 'Destination': '%s/%s' % (self.container, objname) } headers.update(self.additional_headers) req = urllib2.Request(url, headers=headers) req.get_method = lambda: 'COPY' conn = urllib2.urlopen(req, timeout=10) self.assertEqual(201, conn.getcode()) self.assertEqual('%s/%s' % (self.container, self.storlet_file), conn.info()['x-storlet-generated-from']) self.assertEqual(self.acct, conn.info()['x-storlet-generated-from-account']) self.assertIn('x-storlet-generated-from-last-modified', conn.info()) headers = client.head_object(self.url, self.token, self.container, objname) self.assertEqual(str(len(self.content)), headers['content-length']) resp = dict() client.delete_object(self.url, self.token, self.container, objname, response_dict=resp) self.assertEqual(204, resp['status'])
def invoke_storlet_on_copy_from(self): headers = {'X-Run-Storlet': self.storlet_name, 'X-Object-Meta-Name': 'thumbnail', 'X-Copy-From': '%s/%s' % (self.container, self.storlet_file)} headers.update(self.additional_headers) resp = dict() c.put_object(self.url, self.token, self.container, 'gen_thumb_on_copy.jpg', '', headers=headers, response_dict=resp) status = resp.get('status') self.assertIn(status, [201, 202]) rh = resp['headers'] self.assertEqual(rh['x-storlet-generated-from'], '%s/%s' % (self.container, self.storlet_file)) self.assertEqual(rh['x-storlet-generated-from-account'], self.acct) self.assertIn('x-storlet-generated-from-last-modified', rh) headers = c.head_object(self.url, self.token, self.container, 'gen_thumb_on_copy.jpg') self.assertEqual('49032', headers['content-length']) self.assertEqual('thumbnail', headers['x-object-meta-name']) self.assertTrue('x-object-meta-x-timestamp' not in headers) self.assertTrue('x-timestamp' in headers)
def test_async_update_after_PUT(self): cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content) meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse(direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1]) # Run the object-updaters to be sure updates are done Manager(['object-updater']).once() # check the re-started container server has update with override values obj = direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1][0] self.assertEqual(meta['etag'], obj['hash']) self.assertEqual(len(content), obj['bytes'])
def test_async_update_after_PUT(self): cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content) meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse( direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1]) # Run the object-updaters to be sure updates are done Manager(['object-updater']).once() # check the re-started container server has update with override values obj = direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1][0] self.assertEqual(meta['etag'], obj['hash']) self.assertEqual(len(content), obj['bytes'])
def _check_node(self, node, part, etag, headers_post): # get fragment archive etag fragment_archive_etag = self.direct_get(node, part) # remove data from the selected node part_dir = self.storage_dir('object', node, part=part) shutil.rmtree(part_dir, True) # this node can't servce the data any more try: self.direct_get(node, part) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 404) else: self.fail('Node data on %r was not fully destoryed!' % (node,)) # make sure we can still GET the object and its correct, the # proxy is doing decode on remaining fragments to get the obj self.assertEqual(etag, self.proxy_get()) # fire up reconstructor self.reconstructor.once() # fragment is rebuilt exactly as it was before! self.assertEqual(fragment_archive_etag, self.direct_get(node, part)) # check meta meta = client.head_object(self.url, self.token, self.container_name, self.object_name) for key in headers_post: self.assertTrue(key in meta) self.assertEqual(meta[key], headers_post[key])
def read_temp_url(url, token, container, object): head = swift_client.head_object(url=url, token=token, container=container, name=object) print head try: print head['x-object-meta-tempurl'] return head['x-object-meta-tempurl'], head['x-object-meta-tempurlexp'] except KeyError: return False, False
def test_put_ctype_replicated_when_subsequent_post(self): # primary half handoff half # ------------ ------------ # t0.data: ctype = foo # t1.data: ctype = bar # t2.meta: # # ...run replicator and expect... # # t1.data: ctype = bar # t2.meta: self.brain.put_container() # incomplete write self.brain.stop_handoff_half() self.container_brain.stop_handoff_half() self._put_object(headers={'Content-Type': 'foo'}) self.brain.start_handoff_half() self.container_brain.start_handoff_half() # handoff write self.brain.stop_primary_half() self.container_brain.stop_primary_half() self._put_object(headers={'Content-Type': 'bar'}) self.brain.start_primary_half() self.container_brain.start_primary_half() # metadata update with newest data unavailable self.brain.stop_handoff_half() self.container_brain.stop_handoff_half() self._post_object(headers={'X-Object-Meta-Color': 'Blue'}) self.brain.start_handoff_half() self.container_brain.start_handoff_half() self.get_to_final_state() # check object metadata metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) # check container listing metadata container_metadata, objs = client.get_container(self.url, self.token, self.container_name) for obj in objs: if obj['name'] == self.object_name: break else: self.fail('obj not found in container listing') expected = 'bar' self.assertEqual(obj['content_type'], expected) self.assertEqual(metadata['x-object-meta-color'], 'Blue') self._assert_object_metadata_matches_listing(obj, metadata) self._assert_consistent_container_dbs() self._assert_consistent_object_metadata() self._assert_consistent_suffix_hashes()
def test_put_ctype_replicated_when_subsequent_post(self): # primary half handoff half # ------------ ------------ # t0.data: ctype = foo # t1.data: ctype = bar # t2.meta: # # ...run replicator and expect... # # t1.data: ctype = bar # t2.meta: self.brain.put_container(policy_index=0) # incomplete write self.brain.stop_handoff_half() self.container_brain.stop_handoff_half() self._put_object(headers={'Content-Type': 'foo'}) self.brain.start_handoff_half() self.container_brain.start_handoff_half() # handoff write self.brain.stop_primary_half() self.container_brain.stop_primary_half() self._put_object(headers={'Content-Type': 'bar'}) self.brain.start_primary_half() self.container_brain.start_primary_half() # metadata update with newest data unavailable self.brain.stop_handoff_half() self.container_brain.stop_handoff_half() self._post_object(headers={'X-Object-Meta-Color': 'Blue'}) self.brain.start_handoff_half() self.container_brain.start_handoff_half() self.get_to_final_state() # check object metadata metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) # check container listing metadata container_metadata, objs = client.get_container(self.url, self.token, self.container_name) for obj in objs: if obj['name'] == self.object_name: break else: self.fail('obj not found in container listing') expected = 'bar' self.assertEqual(obj['content_type'], expected) self.assertEqual(metadata['x-object-meta-color'], 'Blue') self._assert_object_metadata_matches_listing(obj, metadata) self._assert_consistent_container_dbs() self._assert_consistent_object_metadata() self._assert_consistent_suffix_hashes()
def test_update_during_POST_only(self): # verify correct update values when PUT update is missed but then a # POST update succeeds *before* the PUT async pending update is sent cpart, cnodes = self.container_ring.get_nodes(self.account, "c1") client.put_container(self.url, self.token, "c1", headers={"X-Storage-Policy": self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server) content = u"stuff" client.put_object(self.url, self.token, "c1", "o1", contents=content, content_type="test/ctype") meta = client.head_object(self.url, self.token, "c1", "o1") # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server) self.assertFalse(direct_client.direct_get_container(cnodes[0], cpart, self.account, "c1")[1]) # use internal client for POST so we can force fast-post mode int_client = self.make_internal_client(object_post_as_copy=False) int_client.set_object_metadata(self.account, "c1", "o1", {"X-Object-Meta-Fruit": "Tomato"}) self.assertEqual( "Tomato", int_client.get_object_metadata(self.account, "c1", "o1")["x-object-meta-fruit"] ) # sanity # check the re-started container server got same update as others. # we cannot assert the actual etag value because it may be encrypted listing_etags = set() for cnode in cnodes: listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]["bytes"]) self.assertEqual("test/ctype", listing[0]["content_type"]) listing_etags.add(listing[0]["hash"]) self.assertEqual(1, len(listing_etags)) # check that listing meta returned to client is consistent with object # meta returned to client hdrs, listing = client.get_container(self.url, self.token, "c1") self.assertEqual(1, len(listing)) self.assertEqual("o1", listing[0]["name"]) self.assertEqual(len(content), listing[0]["bytes"]) self.assertEqual(meta["etag"], listing[0]["hash"]) self.assertEqual("test/ctype", listing[0]["content_type"]) # Run the object-updaters to send the async pending from the PUT Manager(["object-updater"]).once() # check container listing metadata is still correct for cnode in cnodes: listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]["bytes"]) self.assertEqual("test/ctype", listing[0]["content_type"]) listing_etags.add(listing[0]["hash"]) self.assertEqual(1, len(listing_etags))
def _set_headers(storage_url, auth_token, container, object_name, deleted, request): try: try: header = client.head_object(storage_url, auth_token, container, object_name) except swift_exception.ClientException as e: storage_url, auth_token = _reauthorize(request) header = client.head_object(storage_url, auth_token, container, object_name) new_header = {'X-Object-Meta-Deleted': deleted, 'X-Object-Meta-Format': header['x-object-meta-format'], 'X-Object-Meta-Resolution': header['x-object-meta-resolution'], 'X-Object-Meta-Name': header['x-object-meta-name'], 'X-Object-Meta-Type': header['x-object-meta-type'] } copy_of = None if (header['x-object-meta-format'] in video_types) and (header['x-object-meta-type'] == 'original'): new_header['X-Object-Meta-Mp4-Id'] = header['x-object-meta-mp4-id'] if 'x-object-meta-status' in header: new_header['X-Object-Meta-Status'] = header['x-object-meta-status'] if header['x-object-meta-type'] == 'thumbnail': new_header['X-Object-Meta-Original'] = header['x-object-meta-original'] copy_of = header['x-object-meta-original'] if header['x-object-meta-type'] == 'original': new_header['X-Object-Meta-Thumb'] = header['x-object-meta-thumb'] if container == 'Video': copy_of = [header['x-object-meta-thumb'], header['x-object-meta-mp4-id']] else: copy_of = header['x-object-meta-thumb'] if 'x-object-meta-preview-id' in header: new_header['X-Object-Meta-Preview-Id'] = header['x-object-meta-preview-id'] copy_of = [header['x-object-meta-thumb'], header['x-object-meta-preview-id']] try: client.post_object(storage_url, auth_token, container, object_name, headers=new_header) except swift_exception.ClientException as e: storage_url, auth_token = _reauthorize(request) client.post_object(storage_url, auth_token, container, object_name, headers=new_header) return copy_of except swift_exception.ClientException as e: raise e except Exception as e: raise e
def test_async_updates_after_PUT_and_POST(self): # verify correct update values when PUT update and POST updates are # missed but then async updates are sent cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # PUT and POST to object while one container server is stopped so that # we force async updates to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content, content_type='test/ctype') meta = client.head_object(self.url, self.token, 'c1', 'o1') # use internal client for POST so we can force fast-post mode int_client = self.make_internal_client(object_post_as_copy=False) int_client.set_object_metadata( self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'}) self.assertEqual( 'Tomato', int_client.get_object_metadata(self.account, 'c1', 'o1') ['x-object-meta-fruit']) # sanity # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse(direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1]) # Run the object-updaters to send the async pendings Manager(['object-updater']).once() # check the re-started container server got same update as others. # we cannot assert the actual etag value because it may be encrypted listing_etags = set() for cnode in cnodes: listing = direct_client.direct_get_container( cnode, cpart, self.account, 'c1')[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual('test/ctype', listing[0]['content_type']) listing_etags.add(listing[0]['hash']) self.assertEqual(1, len(listing_etags)) # check that listing meta returned to client is consistent with object # meta returned to client hdrs, listing = client.get_container(self.url, self.token, 'c1') self.assertEqual(1, len(listing)) self.assertEqual('o1', listing[0]['name']) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual(meta['etag'], listing[0]['hash']) self.assertEqual('test/ctype', listing[0]['content_type'])
def test_post_ctype_replicated_when_previous_incomplete_puts(self): # primary half handoff half # ------------ ------------ # t0.data: ctype = foo # t1.data: ctype = bar # t2.meta: ctype = baz # # ...run replicator and expect... # # t1.data: # t2.meta: ctype = baz self.brain.put_container() # incomplete write to primary half self.brain.stop_handoff_half() self.container_brain.stop_handoff_half() self._put_object(headers={'Content-Type': 'foo'}) self.brain.start_handoff_half() self.container_brain.start_handoff_half() # handoff write self.brain.stop_primary_half() self.container_brain.stop_primary_half() self._put_object(headers={'Content-Type': 'bar'}) self.brain.start_primary_half() self.container_brain.start_primary_half() # content-type update to primary half self.brain.stop_handoff_half() self.container_brain.stop_handoff_half() self._post_object(headers={'Content-Type': 'baz'}) self.brain.start_handoff_half() self.container_brain.start_handoff_half() self.get_to_final_state() # check object metadata metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) # check container listing metadata container_metadata, objs = client.get_container(self.url, self.token, self.container_name) for obj in objs: if obj['name'] == self.object_name: break expected = 'baz' self.assertEqual(obj['content_type'], expected) self._assert_object_metadata_matches_listing(obj, metadata) self._assert_consistent_container_dbs() self._assert_consistent_object_metadata() self._assert_consistent_suffix_hashes()
def test_post_ctype_replicated_when_previous_incomplete_puts(self): # primary half handoff half # ------------ ------------ # t0.data: ctype = foo # t1.data: ctype = bar # t2.meta: ctype = baz # # ...run replicator and expect... # # t1.data: # t2.meta: ctype = baz self.brain.put_container(policy_index=0) # incomplete write to primary half self.brain.stop_handoff_half() self.container_brain.stop_handoff_half() self._put_object(headers={'Content-Type': 'foo'}) self.brain.start_handoff_half() self.container_brain.start_handoff_half() # handoff write self.brain.stop_primary_half() self.container_brain.stop_primary_half() self._put_object(headers={'Content-Type': 'bar'}) self.brain.start_primary_half() self.container_brain.start_primary_half() # content-type update to primary half self.brain.stop_handoff_half() self.container_brain.stop_handoff_half() self._post_object(headers={'Content-Type': 'baz'}) self.brain.start_handoff_half() self.container_brain.start_handoff_half() self.get_to_final_state() # check object metadata metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) # check container listing metadata container_metadata, objs = client.get_container(self.url, self.token, self.container_name) for obj in objs: if obj['name'] == self.object_name: break expected = 'baz' self.assertEqual(obj['content_type'], expected) self._assert_object_metadata_matches_listing(obj, metadata) self._assert_consistent_container_dbs() self._assert_consistent_object_metadata() self._assert_consistent_suffix_hashes()
def test_async_update_after_PUT(self): cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content, content_type='test/ctype') meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse( direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1]) # Run the object-updaters to be sure updates are done Manager(['object-updater']).once() # check the re-started container server got same update as others. # we cannot assert the actual etag value because it may be encrypted listing_etags = set() for cnode in cnodes: listing = direct_client.direct_get_container( cnode, cpart, self.account, 'c1')[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual('test/ctype', listing[0]['content_type']) listing_etags.add(listing[0]['hash']) self.assertEqual(1, len(listing_etags)) # check that listing meta returned to client is consistent with object # meta returned to client hdrs, listing = client.get_container(self.url, self.token, 'c1') self.assertEqual(1, len(listing)) self.assertEqual('o1', listing[0]['name']) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual(meta['etag'], listing[0]['hash']) self.assertEqual('test/ctype', listing[0]['content_type'])
def _check_node(self, node, part, etag, headers_post): # get fragment archive etag headers, fragment_archive_etag = self.direct_get(node, part) self.assertIn('X-Backend-Durable-Timestamp', headers) # sanity check durable_timestamp = headers['X-Backend-Durable-Timestamp'] # make the data file non-durable on the selected node part_dir = self.storage_dir('object', node, part=part) for dirs, subdirs, files in os.walk(part_dir): for fname in files: if fname.endswith('.data'): non_durable_fname = fname.replace('#d', '') os.rename(os.path.join(dirs, fname), os.path.join(dirs, non_durable_fname)) try: os.remove(os.path.join(part_dir, 'hashes.pkl')) except OSError as e: if e.errno != errno.ENOENT: raise # sanity check that fragment is no longer durable headers = direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={ 'X-Backend-Storage-Policy-Index': int(self.policy), 'X-Backend-Fragment-Preferences': json.dumps([]) }) self.assertNotIn('X-Backend-Durable-Timestamp', headers) # fire up reconstructor to propagate durable state self.reconstructor.once() # fragment is still exactly as it was before! headers, fragment_archive_etag_2 = self.direct_get(node, part) self.assertEqual(fragment_archive_etag, fragment_archive_etag_2) self.assertIn('X-Backend-Durable-Timestamp', headers) self.assertEqual(durable_timestamp, headers['X-Backend-Durable-Timestamp']) # check meta meta = client.head_object(self.url, self.token, self.container_name, self.object_name) for key in headers_post: self.assertIn(key, meta) self.assertEqual(meta[key], headers_post[key])
def invoke_storlet_on_copy_dest(self): # No COPY in swiftclient. Using urllib instead... url = '%s/%s/%s' % (self.url, 'myobjects', self.storlet_file) headers = {'X-Auth-Token': self.token, 'X-Run-Storlet': self.storlet_name, 'Destination': 'myobjects/gen_thumb_on_copy_.jpg'} req = urllib2.Request(url, headers=headers) req.get_method = lambda: 'COPY' conn = urllib2.urlopen(req, timeout=10) status = conn.getcode() self.assertTrue(status in [201, 202]) headers = c.head_object(self.url, self.token, 'myobjects', 'gen_thumb_on_copy_.jpg') self.assertEqual(headers['content-length'], '49032')
def invoke_storlet_on_put(self): headers = {'X-Run-Storlet': self.storlet_name} resp = dict() source_file = '%s/%s' % (self.path_to_bundle, self.storlet_file) with open(source_file, 'r') as f: c.put_object(self.url, self.token, 'myobjects', 'gen_thumb_on_put.jpg', f, headers=headers, response_dict=resp) status = resp.get('status') self.assertTrue(status in [201, 202]) headers = c.head_object(self.url, self.token, 'myobjects', 'gen_thumb_on_put.jpg') self.assertEqual(headers['content-length'], '49032')
def _get_video_headers(storage_url, auth_token, container, object_name): try: header = client.head_object(storage_url, auth_token, container, object_name) new_header = {'X-Object-Meta-Deleted': header['x-object-meta-deleted'], 'X-Object-Meta-Format': header['x-object-meta-format'], 'X-Object-Meta-Resolution': header['x-object-meta-resolution'], 'X-Object-Meta-Name': header['x-object-meta-name'], 'X-Object-Meta-Type': header['x-object-meta-type'] } # if 'x-delete-after' in header: # new_header['X-Delete-After'] = header['x-delete-after'] return new_header except swift_exception.ClientException as e: raise e except Exception as e: raise e
def test_update_during_POST_only(self): # verify correct update values when PUT update is missed but then a # POST update succeeds *before* the PUT async pending update is sent cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content) meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse( direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1]) # use internal client for POST so we can force fast-post mode int_client = self.make_internal_client(object_post_as_copy=False) int_client.set_object_metadata(self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'}) self.assertEqual('Tomato', int_client.get_object_metadata( self.account, 'c1', 'o1')['x-object-meta-fruit']) # sanity # check the re-started container server has update with override values obj = direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1][0] self.assertEqual(meta['etag'], obj['hash']) self.assertEqual(len(content), obj['bytes']) # Run the object-updaters to send the async pending from the PUT Manager(['object-updater']).once() # check container listing metadata is still correct obj = direct_client.direct_get_container(cnodes[0], cpart, self.account, 'c1')[1][0] self.assertEqual(meta['etag'], obj['hash']) self.assertEqual(len(content), obj['bytes'])
def test_update_during_POST_only(self): # verify correct update values when PUT update is missed but then a # POST update succeeds *before* the PUT async pending update is sent cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content) meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse(direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1]) # use internal client for POST so we can force fast-post mode int_client = self.make_internal_client(object_post_as_copy=False) int_client.set_object_metadata( self.account, 'c1', 'o1', {'X-Object-Meta-Fruit': 'Tomato'}) self.assertEqual( 'Tomato', int_client.get_object_metadata(self.account, 'c1', 'o1') ['x-object-meta-fruit']) # sanity # check the re-started container server has update with override values obj = direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1][0] self.assertEqual(meta['etag'], obj['hash']) self.assertEqual(len(content), obj['bytes']) # Run the object-updaters to send the async pending from the PUT Manager(['object-updater']).once() # check container listing metadata is still correct obj = direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1][0] self.assertEqual(meta['etag'], obj['hash']) self.assertEqual(len(content), obj['bytes'])
def invoke_storlet_on_put(self): headers = {'X-Run-Storlet': self.storlet_name, 'x-object-meta-name': 'thumbnail'} headers.update(self.additional_headers) resp = dict() source_file = '%s/%s' % (self.path_to_bundle, self.storlet_file) with open(source_file, 'r') as f: c.put_object(self.url, self.token, self.container, 'gen_thumb_on_put.jpg', f, headers=headers, response_dict=resp) status = resp.get('status') self.assertIn(status, [201, 202]) headers = c.head_object(self.url, self.token, self.container, 'gen_thumb_on_put.jpg') self.assertEqual('49032', headers['content-length']) self.assertEqual('thumbnail', headers['x-object-meta-name'])
def test_async_update_after_PUT(self): cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1') client.put_container(self.url, self.token, 'c1', headers={'X-Storage-Policy': self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) content = u'stuff' client.put_object(self.url, self.token, 'c1', 'o1', contents=content, content_type='test/ctype') meta = client.head_object(self.url, self.token, 'c1', 'o1') # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server) self.assertFalse(direct_client.direct_get_container( cnodes[0], cpart, self.account, 'c1')[1]) # Run the object-updaters to be sure updates are done Manager(['object-updater']).once() # check the re-started container server got same update as others. # we cannot assert the actual etag value because it may be encrypted listing_etags = set() for cnode in cnodes: listing = direct_client.direct_get_container( cnode, cpart, self.account, 'c1')[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual('test/ctype', listing[0]['content_type']) listing_etags.add(listing[0]['hash']) self.assertEqual(1, len(listing_etags)) # check that listing meta returned to client is consistent with object # meta returned to client hdrs, listing = client.get_container(self.url, self.token, 'c1') self.assertEqual(1, len(listing)) self.assertEqual('o1', listing[0]['name']) self.assertEqual(len(content), listing[0]['bytes']) self.assertEqual(meta['etag'], listing[0]['hash']) self.assertEqual('test/ctype', listing[0]['content_type'])
def get_all(request, container): """ Returns List of all Objects present in specified container :param container: Name of Swift Container :return: meta data and id's of all the objects """ try: response_dict = dict() storage_url, auth_token = _get_auth_data(request.session) data_container = client.get_container(storage_url, auth_token, container) for obj in data_container[1]: meta_object = client.head_object(storage_url, auth_token, container, obj['name']) if not meta_object['x-object-meta-deleted']: if meta_object['x-object-meta-type'] in [ 'thumbnail', 'original-thumbnail' ]: if meta_object[ 'x-object-meta-format'] not in response_dict: form = meta_object['x-object-meta-format'] response_dict[form] = [] if meta_object['x-object-meta-format'] in response_dict: new_obj = { 'thumbnail_id': obj['name'], 'name': meta_object['x-object-meta-name'], 'type': meta_object['x-object-meta-type'], 'resolution': meta_object['x-object-meta-resolution'] } if meta_object['x-object-meta-format'] is 'thumbnail': new_obj['original_id'] = meta_object[ 'x-object-meta-original'] response_dict[form].append(new_obj) return Response(response_dict) except swift_exception.ClientException as e: print e return Response('Please contact your admininstrator', status=status.HTTP_501_NOT_IMPLEMENTED) except Exception as e: print e return Response('Please contact your admininstrator', status=status.HTTP_501_NOT_IMPLEMENTED)
def register(self, filepath, protocol, extra=None): ''' Reigster filepath and protocol with the ID service. If there is an interface specified use that. The interface parameter is useful when testing on the same host as the ID service and you want to use the VM IP Auth ''' if protocol == "swift": path_parts = filepath.split("/") object_name = path_parts.pop() url = "%s//%s/%s/%s" % ((path_parts[0], ) + tuple(path_parts[2:5])) container = "/".join(path_parts[5:]) if self.swift_auth_url == self.os_auth_url: token = self.auth_token else: _, token = get_auth( self.swift_auth_url, "%s:%s" % (self.swift_tenant, self.swift_username), self.swift_password) size = head_object(url, token, container, object_name)["content-length"] swift = { "swift": { "url": url, "container": container, "object": object_name, "auth_url": self.swift_auth_url }, "filesize": size } if extra: extra.update(swift) else: extra = swift record = {"filepath": filepath, "protocol": protocol} if extra: record.update(extra) return self.http_post("%s%s" % (self.id_service, self.ID_PATH), json.dumps(record)).text
def invoke_storlet_on_copy_dest(self): # No COPY in swiftclient. Using urllib instead... url = '%s/%s/%s' % (self.url, self.container, self.storlet_file) headers = {'X-Auth-Token': self.token, 'X-Run-Storlet': self.storlet_name, 'X-Object-Meta-Name': 'thumbnail', 'Destination': '%s/gen_thumb_on_copy_.jpg' % self.container} headers.update(self.additional_headers) req = urllib2.Request(url, headers=headers) req.get_method = lambda: 'COPY' conn = urllib2.urlopen(req, timeout=10) status = conn.getcode() self.assertIn(status, [201, 202]) headers = c.head_object(self.url, self.token, self.container, 'gen_thumb_on_copy_.jpg') self.assertEqual('49032', headers['content-length']) self.assertEqual('thumbnail', headers['x-object-meta-name']) self.assertTrue('x-object-meta-x-timestamp' not in headers) self.assertTrue('x-timestamp' in headers)
def invoke_storlet_on_copy_from(self): headers = {'X-Run-Storlet': self.storlet_name, 'X-Copy-From': 'myobjects/%s' % self.storlet_file} resp = dict() c.put_object(self.url, self.token, 'myobjects', 'gen_thumb_on_copy.jpg', '', headers=headers, response_dict=resp) status = resp.get('status') self.assertTrue(status in [201, 202]) rh = resp['headers'] self.assertEqual(rh['x-storlet-generated-from'], 'myobjects/%s' % self.storlet_file) self.assertEqual(rh['x-storlet-generated-from-account'], self.acct) self.assertTrue('x-storlet-generated-from-last-modified' in rh) headers = c.head_object(self.url, self.token, 'myobjects', 'gen_thumb_on_copy.jpg') self.assertEqual(headers['content-length'], '49032')
def register(self, filepath, protocol, extra=None): ''' Reigster filepath and protocol with the ID service. If there is an interface specified use that. The interface parameter is useful when testing on the same host as the ID service and you want to use the VM IP Auth ''' if protocol == "swift": path_parts = filepath.split("/") object_name = path_parts.pop() url = "%s//%s/%s/%s" % ((path_parts[0],) + tuple(path_parts[2:5])) container = "/".join(path_parts[5:]) if self.swift_auth_url == self.os_auth_url: token = self.auth_token else: _, token = get_auth(self.swift_auth_url, "%s:%s" % ( self.swift_tenant, self.swift_username), self.swift_password) size = head_object(url, token, container, object_name)["content-length"] swift = {"swift": { "url": url, "container": container, "object": object_name, "auth_url": self.swift_auth_url }, "filesize": size } if extra: extra.update(swift) else: extra = swift record = {"filepath": filepath, "protocol": protocol} if extra: record.update(extra) return self.http_post("%s%s" % (self.id_service, self.ID_PATH), json.dumps(record)).text
def _set_headers(storage_url, auth_token, container, object_name, deleted): try: header = client.head_object(storage_url, auth_token, container, object_name) new_header = {'X-Object-Meta-Deleted': deleted, 'X-Object-Meta-Format': header['x-object-meta-format'], 'X-Object-Meta-Resolution': header['x-object-meta-resolution'], 'X-Object-Meta-Name': header['x-object-meta-name'], 'X-Object-Meta-Type': header['x-object-meta-type'] } copy_of = None if header['x-object-meta-type'] == 'thumbnail': new_header['X-Object-Meta-Original'] = header['x-object-meta-original'] copy_of = header['x-object-meta-original'] if header['x-object-meta-type'] == 'original': new_header['X-Object-Meta-Thumb'] = header['x-object-meta-thumb'] copy_of = header['x-object-meta-thumb'] client.post_object(storage_url, auth_token, container, object_name, headers=new_header) return copy_of except swift_exception.ClientException as e: raise e except Exception as e: raise e
def test_async_update_after_PUT(self): cpart, cnodes = self.container_ring.get_nodes(self.account, "c1") client.put_container(self.url, self.token, "c1", headers={"X-Storage-Policy": self.policy.name}) # put an object while one container server is stopped so that we force # an async update to it kill_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server) content = u"stuff" client.put_object(self.url, self.token, "c1", "o1", contents=content, content_type="test/ctype") meta = client.head_object(self.url, self.token, "c1", "o1") # re-start the container server and assert that it does not yet know # about the object start_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server) self.assertFalse(direct_client.direct_get_container(cnodes[0], cpart, self.account, "c1")[1]) # Run the object-updaters to be sure updates are done Manager(["object-updater"]).once() # check the re-started container server got same update as others. # we cannot assert the actual etag value because it may be encrypted listing_etags = set() for cnode in cnodes: listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1] self.assertEqual(1, len(listing)) self.assertEqual(len(content), listing[0]["bytes"]) self.assertEqual("test/ctype", listing[0]["content_type"]) listing_etags.add(listing[0]["hash"]) self.assertEqual(1, len(listing_etags)) # check that listing meta returned to client is consistent with object # meta returned to client hdrs, listing = client.get_container(self.url, self.token, "c1") self.assertEqual(1, len(listing)) self.assertEqual("o1", listing[0]["name"]) self.assertEqual(len(content), listing[0]["bytes"]) self.assertEqual(meta["etag"], listing[0]["hash"]) self.assertEqual("test/ctype", listing[0]["content_type"])
if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) exc = None try: direct_client.direct_get_object(another_onode, opart, self.account, container, obj) except direct_client.ClientException, err: exc = err self.assertEquals(exc.http_status, 404) kill_server(onode['port'], self.port2server, self.pids) client.delete_object(self.url, self.token, container, obj) exc = None try: client.head_object(self.url, self.token, container, obj) except direct_client.ClientException, err: exc = err self.assertEquals(exc.http_status, 404) objs = [ o['name'] for o in client.get_container(self.url, self.token, container)[1] ] if obj in objs: raise Exception('Container listing still knew about object') for cnode in cnodes: objs = [ o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1] ] if obj in objs:
def test_main(self): # Create container container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': self.policy.name}) # Kill one container/obj primary server cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] kill_server((onode['ip'], onode['port']), self.ipport2server) # Create container/obj (goes to two primary servers and one handoff) client.put_object(self.url, self.token, container, obj, b'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != b'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Stash the on disk data from a primary for future comparison with the # handoff - this may not equal 'VERIFY' if for example the proxy has # crypto enabled direct_get_data = direct_client.direct_get_object( onodes[1], opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] # Kill other two container/obj primary servers # to ensure GET handoff works for node in onodes[1:]: kill_server((node['ip'], node['port']), self.ipport2server) # Indirectly through proxy assert we can get container/obj odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != b'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Restart those other two container/obj primary servers for node in onodes[1:]: start_server((node['ip'], node['port']), self.ipport2server) # We've indirectly verified the handoff node has the container/object, # but let's directly verify it. another_onode = next(self.object_ring.get_more_nodes(opart)) odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] self.assertEqual(direct_get_data, odata) # drop a tempfile in the handoff's datadir, like it might have # had if there was an rsync failure while it was previously a # primary handoff_device_path = self.device_dir(another_onode) data_filename = None for root, dirs, files in os.walk(handoff_device_path): for filename in files: if filename.endswith('.data'): data_filename = filename temp_filename = '.%s.6MbL6r' % data_filename temp_filepath = os.path.join(root, temp_filename) if not data_filename: self.fail('Did not find any data files on %r' % handoff_device_path) open(temp_filepath, 'w') # Assert container listing (via proxy and directly) has container/obj objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % (cnode['ip'], cnode['port'])) # Bring the first container/obj primary server back up start_server((onode['ip'], onode['port']), self.ipport2server) # Assert that it doesn't have container/obj yet try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) // 10 Manager(['object-replicator']).once(number=node_id) try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] another_num = (another_port_num - 6000) // 10 Manager(['object-replicator']).once(number=another_num) # Assert the first container/obj primary server now has container/obj odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] self.assertEqual(direct_get_data, odata) # and that it does *not* have a temporary rsync dropping! found_data_filename = False primary_device_path = self.device_dir(onode) for root, dirs, files in os.walk(primary_device_path): for filename in files: if filename.endswith('.6MbL6r'): self.fail('Found unexpected file %s' % os.path.join(root, filename)) if filename == data_filename: found_data_filename = True self.assertTrue(found_data_filename, 'Did not find data file %r on %r' % ( data_filename, primary_device_path)) # Assert the handoff server no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Kill the first container/obj primary server again (we have two # primaries and the handoff up now) kill_server((onode['ip'], onode['port']), self.ipport2server) # Delete container/obj try: client.delete_object(self.url, self.token, container, obj) except client.ClientException as err: if self.object_ring.replica_count > 2: raise # Object DELETE returning 503 for (404, 204) # remove this with fix for # https://bugs.launchpad.net/swift/+bug/1318375 self.assertEqual(503, err.http_status) # Assert we can't head container/obj try: client.head_object(self.url, self.token, container, obj) except client.ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Assert container/obj is not in the container listing, both indirectly # and directly objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj in objs: raise Exception('Container listing still knew about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj in objs: raise Exception( 'Container server %s:%s still knew about object' % (cnode['ip'], cnode['port'])) # Restart the first container/obj primary server again start_server((onode['ip'], onode['port']), self.ipport2server) # Assert it still has container/obj direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) // 10 Manager(['object-replicator']).once(number=node_id) another_node_id = (another_port_num - 6000) // 10 Manager(['object-replicator']).once(number=another_node_id) # Assert primary node no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it")
def test_revert_object(self): # create EC container headers = {'X-Storage-Policy': self.policy.name} client.put_container(self.url, self.token, self.container_name, headers=headers) # get our node lists opart, onodes = self.object_ring.get_nodes(self.account, self.container_name, self.object_name) hnodes = self.object_ring.get_more_nodes(opart) # kill 2 a parity count number of primary nodes so we can # force data onto handoffs, we do that by renaming dev dirs # to induce 507 p_dev1 = self.device_dir(onodes[0]) p_dev2 = self.device_dir(onodes[1]) self.kill_drive(p_dev1) self.kill_drive(p_dev2) # PUT object contents = Body() headers = {'x-object-meta-foo': 'meta-foo'} headers_post = {'x-object-meta-bar': 'meta-bar'} client.put_object(self.url, self.token, self.container_name, self.object_name, contents=contents, headers=headers) client.post_object(self.url, self.token, self.container_name, self.object_name, headers=headers_post) # (Some versions of?) swiftclient will mutate the headers dict on post headers_post.pop('X-Auth-Token', None) # these primaries can't serve the data any more, we expect 507 # here and not 404 because we're using mount_check to kill nodes for onode in (onodes[0], onodes[1]): try: self.direct_get(onode, opart) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 507) else: self.fail('Node data on %r was not fully destroyed!' % (onode, )) # now take out another primary p_dev3 = self.device_dir(onodes[2]) self.kill_drive(p_dev3) # this node can't servce the data any more try: self.direct_get(onodes[2], opart) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 507) else: self.fail('Node data on %r was not fully destroyed!' % (onode, )) # make sure we can still GET the object and its correct # we're now pulling from handoffs and reconstructing etag = self.proxy_get() self.assertEqual(etag, contents.etag) # rename the dev dirs so they don't 507 anymore self.revive_drive(p_dev1) self.revive_drive(p_dev2) self.revive_drive(p_dev3) # fire up reconstructor on handoff nodes only for hnode in hnodes: hnode_id = (hnode['port'] - 6000) // 10 self.reconstructor.once(number=hnode_id) # first three primaries have data again for onode in (onodes[0], onodes[2]): self.direct_get(onode, opart) # check meta meta = client.head_object(self.url, self.token, self.container_name, self.object_name) for key in headers_post: self.assertIn(key, meta) self.assertEqual(meta[key], headers_post[key]) # handoffs are empty for hnode in hnodes: try: self.direct_get(hnode, opart) except direct_client.DirectClientException as err: self.assertEqual(err.http_status, 404) else: self.fail('Node data on %r was not fully destroyed!' % (hnode, ))
def test_rebuild_quarantines_lonely_frag(self): # fail one device while the object is deleted so we are left with one # fragment and some tombstones failed_node = self.onodes[0] device_path = self.device_dir(failed_node) self.kill_drive(device_path) self.assert_direct_get_fails(failed_node, self.opart, 507) # sanity # delete object client.delete_object(self.url, self.token, self.container_name, self.object_name) # check we have tombstones for node in self.onodes[1:]: err = self.assert_direct_get_fails(node, self.opart, 404) self.assertIn('X-Backend-Timestamp', err.http_headers) # run the reconstructor with zero reclaim age to clean up tombstones for conf_index in self.configs['object-reconstructor'].keys(): self.run_custom_daemon(ObjectReconstructor, 'object-reconstructor', conf_index, {'reclaim_age': '0'}) # check we no longer have tombstones for node in self.onodes[1:]: err = self.assert_direct_get_fails(node, self.opart, 404) self.assertNotIn('X-Timestamp', err.http_headers) # revive the failed device and check it has a fragment self.revive_drive(device_path) self.assert_direct_get_succeeds(failed_node, self.opart) # restart proxy to clear error-limiting so that the revived drive # participates again Manager(['proxy-server']).restart() # client GET will fail with 503 ... with self.assertRaises(ClientException) as cm: client.get_object(self.url, self.token, self.container_name, self.object_name) self.assertEqual(503, cm.exception.http_status) # ... but client GET succeeds headers = client.head_object(self.url, self.token, self.container_name, self.object_name) for key in self.headers_post: self.assertIn(key, headers) self.assertEqual(self.headers_post[key], headers[key]) # run the reconstructor without quarantine_threshold set error_lines = [] warning_lines = [] for conf_index in self.configs['object-reconstructor'].keys(): reconstructor = self.run_custom_daemon(ObjectReconstructor, 'object-reconstructor', conf_index, {'quarantine_age': '0'}) logger = reconstructor.logger.logger error_lines.append(logger.get_lines_for_level('error')) warning_lines.append(logger.get_lines_for_level('warning')) # check logs for errors found_lines = False for lines in error_lines: if not lines: continue self.assertFalse(found_lines, error_lines) found_lines = True for line in itertools.islice(lines, 0, 6, 2): self.assertIn( 'Unable to get enough responses (1/4 from 1 ok ' 'responses)', line, lines) for line in itertools.islice(lines, 1, 7, 2): self.assertIn( 'Unable to get enough responses (4 x 404 error ' 'responses)', line, lines) self.assertTrue(found_lines, 'error lines not found') for lines in warning_lines: self.assertEqual([], lines) # check we have still have a single fragment and no tombstones self.assert_direct_get_succeeds(failed_node, self.opart) for node in self.onodes[1:]: err = self.assert_direct_get_fails(node, self.opart, 404) self.assertNotIn('X-Timestamp', err.http_headers) # run the reconstructor to quarantine the lonely frag error_lines = [] warning_lines = [] for conf_index in self.configs['object-reconstructor'].keys(): reconstructor = self.run_custom_daemon( ObjectReconstructor, 'object-reconstructor', conf_index, { 'quarantine_age': '0', 'quarantine_threshold': '1' }) logger = reconstructor.logger.logger error_lines.append(logger.get_lines_for_level('error')) warning_lines.append(logger.get_lines_for_level('warning')) # check logs for errors found_lines = False for index, lines in enumerate(error_lines): if not lines: continue self.assertFalse(found_lines, error_lines) found_lines = True for line in itertools.islice(lines, 0, 6, 2): self.assertIn( 'Unable to get enough responses (1/4 from 1 ok ' 'responses)', line, lines) for line in itertools.islice(lines, 1, 7, 2): self.assertIn( 'Unable to get enough responses (6 x 404 error ' 'responses)', line, lines) self.assertTrue(found_lines, 'error lines not found') # check logs for quarantine warning found_lines = False for lines in warning_lines: if not lines: continue self.assertFalse(found_lines, warning_lines) found_lines = True self.assertEqual(1, len(lines), lines) self.assertIn('Quarantined object', lines[0]) self.assertTrue(found_lines, 'warning lines not found') # check we have nothing for node in self.onodes: err = self.assert_direct_get_fails(node, self.opart, 404) self.assertNotIn('X-Backend-Timestamp', err.http_headers) # client HEAD and GET now both 404 with self.assertRaises(ClientException) as cm: client.get_object(self.url, self.token, self.container_name, self.object_name) self.assertEqual(404, cm.exception.http_status) with self.assertRaises(ClientException) as cm: client.head_object(self.url, self.token, self.container_name, self.object_name) self.assertEqual(404, cm.exception.http_status) # run the reconstructor once more - should see no errors in logs! error_lines = [] warning_lines = [] for conf_index in self.configs['object-reconstructor'].keys(): reconstructor = self.run_custom_daemon( ObjectReconstructor, 'object-reconstructor', conf_index, { 'quarantine_age': '0', 'quarantine_threshold': '1' }) logger = reconstructor.logger.logger error_lines.append(logger.get_lines_for_level('error')) warning_lines.append(logger.get_lines_for_level('warning')) for lines in error_lines: self.assertEqual([], lines) for lines in warning_lines: self.assertEqual([], lines)
def head_object(self, container_name, object_name): return client.head_object(self.url, self.token, container_name, object_name)
def test_reconcile_manifest(self): manifest_data = [] def write_part(i): body = 'VERIFY%0.2d' % i + '\x00' * 1048576 part_name = 'manifest_part_%0.2d' % i manifest_entry = { "path": "/%s/%s" % (self.container_name, part_name), "etag": md5(body).hexdigest(), "size_bytes": len(body), } client.put_object(self.url, self.token, self.container_name, part_name, contents=body) manifest_data.append(manifest_entry) # get an old container stashed self.brain.stop_primary_half() policy = random.choice(list(POLICIES)) self.brain.put_container(policy.idx) self.brain.start_primary_half() # write some parts for i in range(10): write_part(i) self.brain.stop_handoff_half() wrong_policy = random.choice([p for p in POLICIES if p is not policy]) self.brain.put_container(wrong_policy.idx) # write some more parts for i in range(10, 20): write_part(i) # write manifest try: client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') except ClientException as err: # so as it works out, you can't really upload a multi-part # manifest for objects that are currently misplaced - you have to # wait until they're all available - which is about the same as # some other failure that causes data to be unavailable to the # proxy at the time of upload self.assertEqual(err.http_status, 400) # but what the heck, we'll sneak one in just to see what happens... direct_manifest_name = self.object_name + '-direct-test' object_ring = POLICIES.get_object_ring(wrong_policy.idx, '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, direct_manifest_name) container_part = self.container_ring.get_part(self.account, self.container_name) def translate_direct(data): return { 'hash': data['etag'], 'bytes': data['size_bytes'], 'name': data['path'], } direct_manifest_data = map(translate_direct, manifest_data) headers = { 'x-container-host': ','.join('%s:%s' % (n['ip'], n['port']) for n in self.container_ring.devs), 'x-container-device': ','.join(n['device'] for n in self.container_ring.devs), 'x-container-partition': container_part, 'X-Backend-Storage-Policy-Index': wrong_policy.idx, 'X-Static-Large-Object': 'True', } for node in nodes: direct_client.direct_put_object( node, part, self.account, self.container_name, direct_manifest_name, contents=utils.json.dumps(direct_manifest_data), headers=headers) break # one should do it... self.brain.start_handoff_half() get_to_final_state() Manager(['container-reconciler']).once() # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # let's see how that direct upload worked out... metadata, body = client.get_object( self.url, self.token, self.container_name, direct_manifest_name, query_string='multipart-manifest=get') self.assertEqual(metadata['x-static-large-object'].lower(), 'true') for i, entry in enumerate(utils.json.loads(body)): for key in ('hash', 'bytes', 'name'): self.assertEquals(entry[key], direct_manifest_data[i][key]) metadata, body = client.get_object(self.url, self.token, self.container_name, direct_manifest_name) self.assertEqual(metadata['x-static-large-object'].lower(), 'true') self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data)) self.assertEqual( body, ''.join('VERIFY%0.2d' % i + '\x00' * 1048576 for i in range(20))) # and regular upload should work now too client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data))
def test_reconcile_manifest(self): info_url = "%s://%s/info" % (urlparse( self.url).scheme, urlparse(self.url).netloc) proxy_conn = client.http_connection(info_url) cluster_info = client.get_capabilities(proxy_conn) if 'slo' not in cluster_info: raise SkipTest("SLO not enabled in proxy; " "can't test manifest reconciliation") # this test is not only testing a split brain scenario on # multiple policies with mis-placed objects - it even writes out # a static large object directly to the storage nodes while the # objects are unavailably mis-placed from *behind* the proxy and # doesn't know how to do that for EC_POLICY (clayg: why did you # guys let me write a test that does this!?) - so we force # wrong_policy (where the manifest gets written) to be one of # any of your configured REPL_POLICY (we know you have one # because this is a ReplProbeTest) wrong_policy = random.choice(POLICIES_BY_TYPE[REPL_POLICY]) policy = random.choice( [p for p in ENABLED_POLICIES if p is not wrong_policy]) manifest_data = [] def write_part(i): body = 'VERIFY%0.2d' % i + '\x00' * 1048576 part_name = 'manifest_part_%0.2d' % i manifest_entry = { "path": "/%s/%s" % (self.container_name, part_name), "etag": md5(body).hexdigest(), "size_bytes": len(body), } client.put_object(self.url, self.token, self.container_name, part_name, contents=body) manifest_data.append(manifest_entry) # get an old container stashed self.brain.stop_primary_half() self.brain.put_container(int(policy)) self.brain.start_primary_half() # write some parts for i in range(10): write_part(i) self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) # write some more parts for i in range(10, 20): write_part(i) # write manifest with self.assertRaises(ClientException) as catcher: client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') # so as it works out, you can't really upload a multi-part # manifest for objects that are currently misplaced - you have to # wait until they're all available - which is about the same as # some other failure that causes data to be unavailable to the # proxy at the time of upload self.assertEqual(catcher.exception.http_status, 400) # but what the heck, we'll sneak one in just to see what happens... direct_manifest_name = self.object_name + '-direct-test' object_ring = POLICIES.get_object_ring(wrong_policy.idx, '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, direct_manifest_name) container_part = self.container_ring.get_part(self.account, self.container_name) def translate_direct(data): return { 'hash': data['etag'], 'bytes': data['size_bytes'], 'name': data['path'], } direct_manifest_data = map(translate_direct, manifest_data) headers = { 'x-container-host': ','.join('%s:%s' % (n['ip'], n['port']) for n in self.container_ring.devs), 'x-container-device': ','.join(n['device'] for n in self.container_ring.devs), 'x-container-partition': container_part, 'X-Backend-Storage-Policy-Index': wrong_policy.idx, 'X-Static-Large-Object': 'True', } for node in nodes: direct_client.direct_put_object( node, part, self.account, self.container_name, direct_manifest_name, contents=utils.json.dumps(direct_manifest_data), headers=headers) break # one should do it... self.brain.start_handoff_half() self.get_to_final_state() Manager(['container-reconciler']).once() # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # let's see how that direct upload worked out... metadata, body = client.get_object( self.url, self.token, self.container_name, direct_manifest_name, query_string='multipart-manifest=get') self.assertEqual(metadata['x-static-large-object'].lower(), 'true') for i, entry in enumerate(utils.json.loads(body)): for key in ('hash', 'bytes', 'name'): self.assertEqual(entry[key], direct_manifest_data[i][key]) metadata, body = client.get_object(self.url, self.token, self.container_name, direct_manifest_name) self.assertEqual(metadata['x-static-large-object'].lower(), 'true') self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data)) self.assertEqual( body, ''.join('VERIFY%0.2d' % i + '\x00' * 1048576 for i in range(20))) # and regular upload should work now too client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data))
def test_reconcile_manifest(self): info_url = "%s://%s/info" % (urlparse(self.url).scheme, urlparse(self.url).netloc) proxy_conn = client.http_connection(info_url) cluster_info = client.get_capabilities(proxy_conn) if 'slo' not in cluster_info: raise SkipTest("SLO not enabled in proxy; " "can't test manifest reconciliation") # this test is not only testing a split brain scenario on # multiple policies with mis-placed objects - it even writes out # a static large object directly to the storage nodes while the # objects are unavailably mis-placed from *behind* the proxy and # doesn't know how to do that for EC_POLICY (clayg: why did you # guys let me write a test that does this!?) - so we force # wrong_policy (where the manifest gets written) to be one of # any of your configured REPL_POLICY (we know you have one # because this is a ReplProbeTest) wrong_policy = random.choice(POLICIES_BY_TYPE[REPL_POLICY]) policy = random.choice([p for p in ENABLED_POLICIES if p is not wrong_policy]) manifest_data = [] def write_part(i): body = 'VERIFY%0.2d' % i + '\x00' * 1048576 part_name = 'manifest_part_%0.2d' % i manifest_entry = { "path": "/%s/%s" % (self.container_name, part_name), "etag": md5(body).hexdigest(), "size_bytes": len(body), } client.put_object(self.url, self.token, self.container_name, part_name, contents=body) manifest_data.append(manifest_entry) # get an old container stashed self.brain.stop_primary_half() self.brain.put_container(int(policy)) self.brain.start_primary_half() # write some parts for i in range(10): write_part(i) self.brain.stop_handoff_half() self.brain.put_container(int(wrong_policy)) # write some more parts for i in range(10, 20): write_part(i) # write manifest with self.assertRaises(ClientException) as catcher: client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') # so as it works out, you can't really upload a multi-part # manifest for objects that are currently misplaced - you have to # wait until they're all available - which is about the same as # some other failure that causes data to be unavailable to the # proxy at the time of upload self.assertEqual(catcher.exception.http_status, 400) # but what the heck, we'll sneak one in just to see what happens... direct_manifest_name = self.object_name + '-direct-test' object_ring = POLICIES.get_object_ring(wrong_policy.idx, '/etc/swift') part, nodes = object_ring.get_nodes( self.account, self.container_name, direct_manifest_name) container_part = self.container_ring.get_part(self.account, self.container_name) def translate_direct(data): return { 'hash': data['etag'], 'bytes': data['size_bytes'], 'name': data['path'], } direct_manifest_data = map(translate_direct, manifest_data) headers = { 'x-container-host': ','.join('%s:%s' % (n['ip'], n['port']) for n in self.container_ring.devs), 'x-container-device': ','.join(n['device'] for n in self.container_ring.devs), 'x-container-partition': container_part, 'X-Backend-Storage-Policy-Index': wrong_policy.idx, 'X-Static-Large-Object': 'True', } for node in nodes: direct_client.direct_put_object( node, part, self.account, self.container_name, direct_manifest_name, contents=utils.json.dumps(direct_manifest_data), headers=headers) break # one should do it... self.brain.start_handoff_half() self.get_to_final_state() Manager(['container-reconciler']).once() # clear proxy cache client.post_container(self.url, self.token, self.container_name, {}) # let's see how that direct upload worked out... metadata, body = client.get_object( self.url, self.token, self.container_name, direct_manifest_name, query_string='multipart-manifest=get') self.assertEqual(metadata['x-static-large-object'].lower(), 'true') for i, entry in enumerate(utils.json.loads(body)): for key in ('hash', 'bytes', 'name'): self.assertEqual(entry[key], direct_manifest_data[i][key]) metadata, body = client.get_object( self.url, self.token, self.container_name, direct_manifest_name) self.assertEqual(metadata['x-static-large-object'].lower(), 'true') self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data)) self.assertEqual(body, ''.join('VERIFY%0.2d' % i + '\x00' * 1048576 for i in range(20))) # and regular upload should work now too client.put_object(self.url, self.token, self.container_name, self.object_name, contents=utils.json.dumps(manifest_data), query_string='multipart-manifest=put') metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) self.assertEqual(int(metadata['content-length']), sum(part['size_bytes'] for part in manifest_data))
def test_main(self): # Create container container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container, headers={'X-Storage-Policy': self.policy.name}) # Kill one container/obj primary server cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] kill_server((onode['ip'], onode['port']), self.ipport2server) # Create container/obj (goes to two primary servers and one handoff) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill other two container/obj primary servers # to ensure GET handoff works for node in onodes[1:]: kill_server((node['ip'], node['port']), self.ipport2server) # Indirectly through proxy assert we can get container/obj odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Restart those other two container/obj primary servers for node in onodes[1:]: start_server((node['ip'], node['port']), self.ipport2server) # We've indirectly verified the handoff node has the container/object, # but let's directly verify it. another_onode = next(self.object_ring.get_more_nodes(opart)) odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # drop a tempfile in the handoff's datadir, like it might have # had if there was an rsync failure while it was previously a # primary handoff_device_path = self.device_dir('object', another_onode) data_filename = None for root, dirs, files in os.walk(handoff_device_path): for filename in files: if filename.endswith('.data'): data_filename = filename temp_filename = '.%s.6MbL6r' % data_filename temp_filepath = os.path.join(root, temp_filename) if not data_filename: self.fail('Did not find any data files on %r' % handoff_device_path) open(temp_filepath, 'w') # Assert container listing (via proxy and directly) has container/obj objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % (cnode['ip'], cnode['port'])) # Bring the first container/obj primary server back up start_server((onode['ip'], onode['port']), self.ipport2server) # Assert that it doesn't have container/obj yet try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=node_id) try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] another_num = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_num) # Assert the first container/obj primary server now has container/obj odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # and that it does *not* have a temporary rsync dropping! found_data_filename = False primary_device_path = self.device_dir('object', onode) for root, dirs, files in os.walk(primary_device_path): for filename in files: if filename.endswith('.6MbL6r'): self.fail('Found unexpected file %s' % os.path.join(root, filename)) if filename == data_filename: found_data_filename = True self.assertTrue(found_data_filename, 'Did not find data file %r on %r' % ( data_filename, primary_device_path)) # Assert the handoff server no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Kill the first container/obj primary server again (we have two # primaries and the handoff up now) kill_server((onode['ip'], onode['port']), self.ipport2server) # Delete container/obj try: client.delete_object(self.url, self.token, container, obj) except client.ClientException as err: if self.object_ring.replica_count > 2: raise # Object DELETE returning 503 for (404, 204) # remove this with fix for # https://bugs.launchpad.net/swift/+bug/1318375 self.assertEqual(503, err.http_status) # Assert we can't head container/obj try: client.head_object(self.url, self.token, container, obj) except client.ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Assert container/obj is not in the container listing, both indirectly # and directly objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj in objs: raise Exception('Container listing still knew about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj in objs: raise Exception( 'Container server %s:%s still knew about object' % (cnode['ip'], cnode['port'])) # Restart the first container/obj primary server again start_server((onode['ip'], onode['port']), self.ipport2server) # Assert it still has container/obj direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=node_id) another_node_id = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_node_id) # Assert primary node no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEqual(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it")
def test_sync_with_stale_container_rows(self): source_container, dest_container = self._setup_synced_containers() brain = BrainSplitter(self.url, self.token, source_container, None, 'container') # upload to source object_name = 'object-%s' % uuid.uuid4() client.put_object(self.url, self.token, source_container, object_name, 'test-body') # check source container listing _, listing = client.get_container( self.url, self.token, source_container) for expected_obj_dict in listing: if expected_obj_dict['name'] == object_name: break else: self.fail('Failed to find source object %r in container listing %r' % (object_name, listing)) # stop all container servers brain.stop_primary_half() brain.stop_handoff_half() # upload new object content to source - container updates will fail client.put_object(self.url, self.token, source_container, object_name, 'new-test-body') source_headers = client.head_object( self.url, self.token, source_container, object_name) # start all container servers brain.start_primary_half() brain.start_handoff_half() # sanity check: source container listing should not have changed _, listing = client.get_container( self.url, self.token, source_container) for actual_obj_dict in listing: if actual_obj_dict['name'] == object_name: self.assertDictEqual(expected_obj_dict, actual_obj_dict) break else: self.fail('Failed to find source object %r in container listing %r' % (object_name, listing)) # cycle container-sync - object should be correctly sync'd despite # stale info in container row Manager(['container-sync']).once() # verify sync'd object has same content and headers dest_headers, body = client.get_object(self.url, self.token, dest_container, object_name) self.assertEqual(body, b'new-test-body') mismatched_headers = [] for k in ('etag', 'content-length', 'content-type', 'x-timestamp', 'last-modified'): if source_headers[k] == dest_headers[k]: continue mismatched_headers.append((k, source_headers[k], dest_headers[k])) if mismatched_headers: msg = '\n'.join([('Mismatched header %r, expected %r but got %r' % item) for item in mismatched_headers]) self.fail(msg)
def test_main(self): # Create container container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) # Kill one container/obj primary server cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] kill_server(onode['port'], self.port2server, self.pids) # Create container/obj (goes to two primary servers and one handoff) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill other two container/obj primary servers # to ensure GET handoff works for node in onodes[1:]: kill_server(node['port'], self.port2server, self.pids) # Indirectly through proxy assert we can get container/obj odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Restart those other two container/obj primary servers for node in onodes[1:]: start_server(node['port'], self.port2server, self.pids) # We've indirectly verified the handoff node has the container/object, # but let's directly verify it. another_onode = self.object_ring.get_more_nodes(opart).next() odata = direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # Assert container listing (via proxy and directly) has container/obj objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % (cnode['ip'], cnode['port'])) # Bring the first container/obj primary server back up start_server(onode['port'], self.port2server, self.pids) # Assert that it doesn't have container/obj yet try: direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=node_id) try: another_port_num = another_onode['replication_port'] except KeyError: another_port_num = another_onode['port'] another_num = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_num) # Assert the first container/obj primary server now has container/obj odata = direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx})[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) # Assert the handoff server no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Kill the first container/obj primary server again (we have two # primaries and the handoff up now) kill_server(onode['port'], self.port2server, self.pids) # Delete container/obj try: client.delete_object(self.url, self.token, container, obj) except client.ClientException as err: if self.object_ring.replica_count > 2: raise # Object DELETE returning 503 for (404, 204) # remove this with fix for # https://bugs.launchpad.net/swift/+bug/1318375 self.assertEqual(503, err.http_status) # Assert we can't head container/obj try: client.head_object(self.url, self.token, container, obj) except client.ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it") # Assert container/obj is not in the container listing, both indirectly # and directly objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj in objs: raise Exception('Container listing still knew about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj in objs: raise Exception( 'Container server %s:%s still knew about object' % (cnode['ip'], cnode['port'])) # Restart the first container/obj primary server again start_server(onode['port'], self.port2server, self.pids) # Assert it still has container/obj direct_client.direct_get_object( onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) # Run object replication, ensuring we run the handoff node last so it # will remove its extra handoff partition for node in onodes: try: port_num = node['replication_port'] except KeyError: port_num = node['port'] node_id = (port_num - 6000) / 10 Manager(['object-replicator']).once(number=node_id) another_node_id = (another_port_num - 6000) / 10 Manager(['object-replicator']).once(number=another_node_id) # Assert primary node no longer has container/obj try: direct_client.direct_get_object( another_onode, opart, self.account, container, obj, headers={ 'X-Backend-Storage-Policy-Index': self.policy.idx}) except ClientException as err: self.assertEquals(err.http_status, 404) else: self.fail("Expected ClientException but didn't get it")
def test_main(self): container = 'container-%s' % uuid4() client.put_container(self.url, self.token, container) apart, anodes = self.account_ring.get_nodes(self.account) cpart, cnodes = self.container_ring.get_nodes(self.account, container) cnode = cnodes[0] obj = 'object-%s' % uuid4() opart, onodes = self.object_ring.get_nodes( self.account, container, obj) onode = onodes[0] kill(self.pids[self.port2server[onode['port']]], SIGTERM) client.put_object(self.url, self.token, container, obj, 'VERIFY') odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) # Kill all primaries to ensure GET handoff works for node in onodes[1:]: kill(self.pids[self.port2server[node['port']]], SIGTERM) odata = client.get_object(self.url, self.token, container, obj)[-1] if odata != 'VERIFY': raise Exception('Object GET did not return VERIFY, instead it ' 'returned: %s' % repr(odata)) for node in onodes[1:]: self.pids[self.port2server[node['port']]] = Popen([ 'swift-object-server', '/etc/swift/object-server/%d.conf' % ((node['port'] - 6000) / 10)]).pid sleep(2) # We've indirectly verified the handoff node has the object, but let's # directly verify it. another_onode = self.object_ring.get_more_nodes(opart).next() odata = direct_client.direct_get_object(another_onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj not in objs: raise Exception('Container listing did not know about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container(cnode, cpart, self.account, container)[1]] if obj not in objs: raise Exception( 'Container server %s:%s did not know about object' % (cnode['ip'], cnode['port'])) self.pids[self.port2server[onode['port']]] = Popen([ 'swift-object-server', '/etc/swift/object-server/%d.conf' % ((onode['port'] - 6000) / 10)]).pid sleep(2) exc = False try: direct_client.direct_get_object(onode, opart, self.account, container, obj) except Exception: exc = True if not exc: raise Exception('Previously downed object server had test object') # Run the extra server last so it'll remove its extra partition ps = [] for n in onodes: ps.append(Popen(['swift-object-replicator', '/etc/swift/object-server/%d.conf' % ((n['port'] - 6000) / 10), 'once'])) for p in ps: p.wait() call(['swift-object-replicator', '/etc/swift/object-server/%d.conf' % ((another_onode['port'] - 6000) / 10), 'once']) odata = direct_client.direct_get_object(onode, opart, self.account, container, obj)[-1] if odata != 'VERIFY': raise Exception('Direct object GET did not return VERIFY, instead ' 'it returned: %s' % repr(odata)) exc = False try: direct_client.direct_get_object(another_onode, opart, self.account, container, obj) except Exception: exc = True if not exc: raise Exception('Handoff object server still had test object') # Because POST has changed to a COPY by default, POSTs will succeed on all up # nodes now if at least one up node has the object. # kill(self.pids[self.port2server[onode['port']]], SIGTERM) # client.post_object(self.url, self.token, container, obj, # headers={'x-object-meta-probe': 'value'}) # oheaders = client.head_object(self.url, self.token, container, obj) # if oheaders.get('x-object-meta-probe') != 'value': # raise Exception('Metadata incorrect, was %s' % repr(oheaders)) # exc = False # try: # direct_client.direct_get_object(another_onode, opart, self.account, # container, obj) # except Exception: # exc = True # if not exc: # raise Exception('Handoff server claimed it had the object when ' # 'it should not have it') # self.pids[self.port2server[onode['port']]] = Popen([ # 'swift-object-server', # '/etc/swift/object-server/%d.conf' % # ((onode['port'] - 6000) / 10)]).pid # sleep(2) # oheaders = direct_client.direct_get_object(onode, opart, self.account, # container, obj)[0] # if oheaders.get('x-object-meta-probe') == 'value': # raise Exception('Previously downed object server had the new ' # 'metadata when it should not have it') # # Run the extra server last so it'll remove its extra partition # ps = [] # for n in onodes: # ps.append(Popen(['swift-object-replicator', # '/etc/swift/object-server/%d.conf' % # ((n['port'] - 6000) / 10), 'once'])) # for p in ps: # p.wait() # call(['swift-object-replicator', # '/etc/swift/object-server/%d.conf' % # ((another_onode['port'] - 6000) / 10), 'once']) # oheaders = direct_client.direct_get_object(onode, opart, self.account, # container, obj)[0] # if oheaders.get('x-object-meta-probe') != 'value': # raise Exception( # 'Previously downed object server did not have the new metadata') kill(self.pids[self.port2server[onode['port']]], SIGTERM) client.delete_object(self.url, self.token, container, obj) exc = False try: client.head_object(self.url, self.token, container, obj) except Exception: exc = True if not exc: raise Exception('Regular object HEAD was still successful') objs = [o['name'] for o in client.get_container(self.url, self.token, container)[1]] if obj in objs: raise Exception('Container listing still knew about object') for cnode in cnodes: objs = [o['name'] for o in direct_client.direct_get_container( cnode, cpart, self.account, container)[1]] if obj in objs: raise Exception( 'Container server %s:%s still knew about object' % (cnode['ip'], cnode['port'])) self.pids[self.port2server[onode['port']]] = Popen([ 'swift-object-server', '/etc/swift/object-server/%d.conf' % ((onode['port'] - 6000) / 10)]).pid sleep(2) direct_client.direct_get_object(onode, opart, self.account, container, obj) # Run the extra server last so it'll remove its extra partition ps = [] for n in onodes: ps.append(Popen(['swift-object-replicator', '/etc/swift/object-server/%d.conf' % ((n['port'] - 6000) / 10), 'once'])) for p in ps: p.wait() call(['swift-object-replicator', '/etc/swift/object-server/%d.conf' % ((another_onode['port'] - 6000) / 10), 'once']) exc = False try: direct_client.direct_get_object(another_onode, opart, self.account, container, obj) except Exception: exc = True if not exc: raise Exception('Handoff object server still had the object')
def test_merge_storage_policy_index(self): # generic split brain self.brain.stop_primary_half() self.brain.put_container() self.brain.start_primary_half() self.brain.stop_handoff_half() self.brain.put_container() self.brain.put_object() self.brain.start_handoff_half() # make sure we have some manner of split brain container_part, container_nodes = self.container_ring.get_nodes( self.account, self.container_name) head_responses = [] for node in container_nodes: metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) found_policy_indexes = \ set(metadata['X-Backend-Storage-Policy-Index'] for node, metadata in head_responses) self.assert_( len(found_policy_indexes) > 1, 'primary nodes did not disagree about policy index %r' % head_responses) # find our object orig_policy_index = None for policy_index in found_policy_indexes: object_ring = POLICIES.get_object_ring(policy_index, '/etc/swift') part, nodes = object_ring.get_nodes(self.account, self.container_name, self.object_name) for node in nodes: try: direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={ 'X-Backend-Storage-Policy-Index': policy_index }) except direct_client.ClientException as err: continue orig_policy_index = policy_index break if orig_policy_index is not None: break else: self.fail('Unable to find /%s/%s/%s in %r' % (self.account, self.container_name, self.object_name, found_policy_indexes)) get_to_final_state() Manager(['container-reconciler']).once() # validate containers head_responses = [] for node in container_nodes: metadata = direct_client.direct_head_container( node, container_part, self.account, self.container_name) head_responses.append((node, metadata)) found_policy_indexes = \ set(metadata['X-Backend-Storage-Policy-Index'] for node, metadata in head_responses) self.assert_( len(found_policy_indexes) == 1, 'primary nodes disagree about policy index %r' % head_responses) expected_policy_index = found_policy_indexes.pop() self.assertNotEqual(orig_policy_index, expected_policy_index) # validate object placement orig_policy_ring = POLICIES.get_object_ring(orig_policy_index, '/etc/swift') for node in orig_policy_ring.devs: try: direct_client.direct_head_object( node, part, self.account, self.container_name, self.object_name, headers={ 'X-Backend-Storage-Policy-Index': orig_policy_index }) except direct_client.ClientException as err: if err.http_status == HTTP_NOT_FOUND: continue raise else: self.fail('Found /%s/%s/%s in %s' % (self.account, self.container_name, self.object_name, orig_policy_index)) # use proxy to access object (bad container info might be cached...) timeout = time.time() + TIMEOUT while time.time() < timeout: try: metadata = client.head_object(self.url, self.token, self.container_name, self.object_name) except ClientException as err: if err.http_status != HTTP_NOT_FOUND: raise time.sleep(1) else: break else: self.fail('could not HEAD /%s/%s/%s/ from policy %s ' 'after %s seconds.' % (self.account, self.container_name, self.object_name, expected_policy_index, TIMEOUT))