def teardown_package(): global orig_collate locale.setlocale(locale.LC_COLLATE, orig_collate) # clean up containers and objects left behind after running tests conn = Connection(config) conn.authenticate() account = Account(conn, config.get('account', config['username'])) account.delete_containers() global in_process if in_process: try: for server in _test_coros: server.kill() except Exception: pass try: rmtree(os.path.dirname(_testdir)) except Exception: pass utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX = \ orig_hash_path_suff_pref utils.SWIFT_CONF_FILE = orig_swift_conf_name constraints.reload_constraints()
def teardown_package(): global orig_collate locale.setlocale(locale.LC_COLLATE, orig_collate) # clean up containers and objects left behind after running tests global config if config: conn = Connection(config) conn.authenticate() account = Account(conn, config.get('account', config['username'])) account.delete_containers() global in_process global _test_socks if in_process: try: for i, server in enumerate(_test_coros): server.kill() if not server.dead: # kill it from the socket level _test_socks[i].close() except Exception: pass try: rmtree(os.path.dirname(_testdir)) except Exception: pass reset_globals()
def teardown_package(): global orig_collate locale.setlocale(locale.LC_COLLATE, orig_collate) # clean up containers and objects left behind after running tests conn = Connection(config) conn.authenticate() account = Account(conn, config.get('account', config['username'])) account.delete_containers() global in_process if in_process: try: for server in _test_coros: server.kill() except Exception: pass try: rmtree(os.path.dirname(_testdir)) except Exception: pass utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX = \ orig_hash_path_suff_pref utils.SWIFT_CONF_FILE = orig_swift_conf_name constraints.reload_constraints()
def teardown_package(): global orig_collate locale.setlocale(locale.LC_COLLATE, orig_collate) # clean up containers and objects left behind after running tests global config if config: conn = Connection(config) conn.authenticate() account = Account(conn, config.get('account', config['username'])) account.delete_containers() global in_process global _test_socks if in_process: try: for i, server in enumerate(_test_coros): server.kill() if not server.dead: # kill it from the socket level _test_socks[i].close() except Exception: pass try: rmtree(os.path.dirname(_testdir)) except Exception: pass reset_globals()
def test_dlo_referer_on_segment_container(self): if 'username3' not in tf.config: raise tf.SkipTest('Requires user 3') # First the account2 (test3) should fail config2 = tf.config.copy() config2['username'] = tf.config['username3'] config2['password'] = tf.config['password3'] conn2 = Connection(config2) conn2.authenticate() headers = { 'X-Auth-Token': conn2.storage_token, 'Referer': 'http://blah.example.com' } dlo_file = self.env.container.file("mancont2") self.assertRaises(ResponseError, dlo_file.read, hdrs=headers) self.assert_status(403) # Now set the referer on the dlo container only referer_metadata = {'X-Container-Read': '.r:*.example.com,.rlistings'} self.env.container.update_metadata(referer_metadata) self.assertRaises(ResponseError, dlo_file.read, hdrs=headers) self.assert_status(403) # Finally set the referer on the segment container self.env.container2.update_metadata(referer_metadata) contents = dlo_file.read(hdrs=headers) self.assertEqual( contents, b"ffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj")
def test_bucket_listing_with_staticweb(self): if 'staticweb' not in tf.cluster_info: raise tf.SkipTest('Staticweb not enabled') bucket = 'bucket' resp = self.conn.create_bucket(Bucket=bucket) self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) resp = self.conn.list_objects(Bucket=bucket) self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode']) # enable staticweb listings; make publicly-readable conn = Connection(tf.config) conn.authenticate() post_status = conn.make_request('POST', [bucket], hdrs={ 'X-Container-Read': '.r:*,.rlistings', 'X-Container-Meta-Web-Listings': 'true', }) self.assertEqual(post_status, 204) resp = self.conn.list_objects(Bucket=bucket) self.assertEqual(200, resp['ResponseMetadata']['HTTPStatusCode'])
def test_dlo_referer_on_segment_container(self): # First the account2 (test3) should fail config2 = tf.config.copy() config2['username'] = tf.config['username3'] config2['password'] = tf.config['password3'] conn2 = Connection(config2) conn2.authenticate() headers = {'X-Auth-Token': conn2.storage_token, 'Referer': 'http://blah.example.com'} dlo_file = self.env.container.file("mancont2") self.assertRaises(ResponseError, dlo_file.read, hdrs=headers) self.assert_status(403) # Now set the referer on the dlo container only referer_metadata = {'X-Container-Read': '.r:*.example.com,.rlistings'} self.env.container.update_metadata(referer_metadata) self.assertRaises(ResponseError, dlo_file.read, hdrs=headers) self.assert_status(403) # Finally set the referer on the segment container self.env.container2.update_metadata(referer_metadata) contents = dlo_file.read(hdrs=headers) self.assertEqual( contents, "ffffffffffgggggggggghhhhhhhhhhiiiiiiiiiijjjjjjjjjj")
def teardown_package(): global orig_collate locale.setlocale(locale.LC_COLLATE, orig_collate) # clean up containers and objects left behind after running tests global config conn = Connection(config) conn.authenticate() account = Account(conn, config.get("account", config["username"])) account.delete_containers() global in_process global _test_socks if in_process: try: for i, server in enumerate(_test_coros): server.kill() if not server.dead: # kill it from the socket level _test_socks[i].close() except Exception: pass try: rmtree(os.path.dirname(_testdir)) except Exception: pass utils.HASH_PATH_PREFIX, utils.HASH_PATH_SUFFIX = orig_hash_path_suff_pref utils.SWIFT_CONF_FILE = orig_swift_conf_name constraints.reload_constraints() reset_globals()
def get_cluster_info(): # The fallback constraints used for testing will come from the current # effective constraints. eff_constraints = dict(constraints.EFFECTIVE_CONSTRAINTS) # We'll update those constraints based on what the /info API provides, if # anything. global cluster_info global config try: conn = Connection(config) conn.authenticate() cluster_info.update(conn.cluster_info()) except (ResponseError, socket.error): # Failed to get cluster_information via /info API, so fall back on # test.conf data pass else: try: eff_constraints.update(cluster_info['swift']) except KeyError: # Most likely the swift cluster has "expose_info = false" set # in its proxy-server.conf file, so we'll just do the best we # can. print("** Swift Cluster not exposing /info **", file=sys.stderr) # Finally, we'll allow any constraint present in the swift-constraints # section of test.conf to override everything. Note that only those # constraints defined in the constraints module are converted to integers. test_constraints = get_config('swift-constraints') for k in constraints.DEFAULT_CONSTRAINTS: try: test_constraints[k] = int(test_constraints[k]) except KeyError: pass except ValueError: print("Invalid constraint value: %s = %s" % (k, test_constraints[k]), file=sys.stderr) eff_constraints.update(test_constraints) # Just make it look like these constraints were loaded from a /info call, # even if the /info call failed, or when they are overridden by values # from the swift-constraints section of test.conf cluster_info['swift'] = eff_constraints
def get_cluster_info(): # The fallback constraints used for testing will come from the current # effective constraints. eff_constraints = dict(constraints.EFFECTIVE_CONSTRAINTS) # We'll update those constraints based on what the /info API provides, if # anything. global cluster_info global config try: conn = Connection(config) conn.authenticate() cluster_info.update(conn.cluster_info()) except (ResponseError, socket.error): # Failed to get cluster_information via /info API, so fall back on # test.conf data pass else: try: eff_constraints.update(cluster_info['swift']) except KeyError: # Most likely the swift cluster has "expose_info = false" set # in its proxy-server.conf file, so we'll just do the best we # can. print("** Swift Cluster not exposing /info **", file=sys.stderr) # Finally, we'll allow any constraint present in the swift-constraints # section of test.conf to override everything. Note that only those # constraints defined in the constraints module are converted to integers. test_constraints = get_config('swift-constraints') for k in constraints.DEFAULT_CONSTRAINTS: try: test_constraints[k] = int(test_constraints[k]) except KeyError: pass except ValueError: print("Invalid constraint value: %s = %s" % ( k, test_constraints[k]), file=sys.stderr) eff_constraints.update(test_constraints) # Just make it look like these constraints were loaded from a /info call, # even if the /info call failed, or when they are overridden by values # from the swift-constraints section of test.conf cluster_info['swift'] = eff_constraints
class TestSloWithVersioning(unittest2.TestCase): def setUp(self): if 'slo' not in cluster_info: raise SkipTest("SLO not enabled") if tf.in_process: tf.skip_if_no_xattrs() self.conn = Connection(tf.config) self.conn.authenticate() self.account = Account( self.conn, tf.config.get('account', tf.config['username'])) self.account.delete_containers() # create a container with versioning self.versions_container = self.account.container(Utils.create_name()) self.container = self.account.container(Utils.create_name()) self.segments_container = self.account.container(Utils.create_name()) if not self.container.create( hdrs={'X-Versions-Location': self.versions_container.name}): raise ResponseError(self.conn.response) if 'versions' not in self.container.info(): raise SkipTest("Object versioning not enabled") for cont in (self.versions_container, self.segments_container): if not cont.create(): raise ResponseError(self.conn.response) # create some segments self.seg_info = {} for letter, size in (('a', 1024 * 1024), ('b', 1024 * 1024)): seg_name = letter file_item = self.segments_container.file(seg_name) file_item.write(letter * size) self.seg_info[seg_name] = { 'size_bytes': size, 'etag': file_item.md5, 'path': '/%s/%s' % (self.segments_container.name, seg_name)} def _create_manifest(self, seg_name): # create a manifest in the versioning container file_item = self.container.file("my-slo-manifest") file_item.write( json.dumps([self.seg_info[seg_name]]), parms={'multipart-manifest': 'put'}) return file_item def _assert_is_manifest(self, file_item, seg_name): manifest_body = file_item.read(parms={'multipart-manifest': 'get'}) resp_headers = dict(file_item.conn.response.getheaders()) self.assertIn('x-static-large-object', resp_headers) self.assertEqual('application/json; charset=utf-8', file_item.content_type) try: manifest = json.loads(manifest_body) except ValueError: self.fail("GET with multipart-manifest=get got invalid json") self.assertEqual(1, len(manifest)) key_map = {'etag': 'hash', 'size_bytes': 'bytes', 'path': 'name'} for k_client, k_slo in key_map.items(): self.assertEqual(self.seg_info[seg_name][k_client], manifest[0][k_slo]) def _assert_is_object(self, file_item, seg_name): file_contents = file_item.read() self.assertEqual(1024 * 1024, len(file_contents)) self.assertEqual(seg_name, file_contents[0]) self.assertEqual(seg_name, file_contents[-1]) def tearDown(self): # remove versioning to allow simple container delete self.container.update_metadata(hdrs={'X-Versions-Location': ''}) self.account.delete_containers() def test_slo_manifest_version(self): file_item = self._create_manifest('a') # sanity check: read the manifest, then the large object self._assert_is_manifest(file_item, 'a') self._assert_is_object(file_item, 'a') # upload new manifest file_item = self._create_manifest('b') # sanity check: read the manifest, then the large object self._assert_is_manifest(file_item, 'b') self._assert_is_object(file_item, 'b') versions_list = self.versions_container.files() self.assertEqual(1, len(versions_list)) version_file = self.versions_container.file(versions_list[0]) # check the version is still a manifest self._assert_is_manifest(version_file, 'a') self._assert_is_object(version_file, 'a') # delete the newest manifest file_item.delete() # expect the original manifest file to be restored self._assert_is_manifest(file_item, 'a') self._assert_is_object(file_item, 'a')
class TestSloWithVersioning(Base): def setUp(self): if 'slo' not in cluster_info: raise SkipTest("SLO not enabled") self.conn = Connection(tf.config) self.conn.authenticate() self.account = Account(self.conn, tf.config.get('account', tf.config['username'])) self.account.delete_containers() # create a container with versioning self.versions_container = self.account.container(Utils.create_name()) self.container = self.account.container(Utils.create_name()) self.segments_container = self.account.container(Utils.create_name()) if not self.container.create( hdrs={'X-Versions-Location': self.versions_container.name}): raise ResponseError(self.conn.response) if 'versions' not in self.container.info(): raise SkipTest("Object versioning not enabled") for cont in (self.versions_container, self.segments_container): if not cont.create(): raise ResponseError(self.conn.response) # create some segments self.seg_info = {} for letter, size in (('a', 1024 * 1024), ('b', 1024 * 1024)): seg_name = letter file_item = self.segments_container.file(seg_name) file_item.write(letter * size) self.seg_info[seg_name] = { 'size_bytes': size, 'etag': file_item.md5, 'path': '/%s/%s' % (self.segments_container.name, seg_name) } def _create_manifest(self, seg_name): # create a manifest in the versioning container file_item = self.container.file("my-slo-manifest") file_item.write(json.dumps([self.seg_info[seg_name]]), parms={'multipart-manifest': 'put'}) return file_item def _assert_is_manifest(self, file_item, seg_name): manifest_body = file_item.read(parms={'multipart-manifest': 'get'}) resp_headers = dict(file_item.conn.response.getheaders()) self.assertIn('x-static-large-object', resp_headers) self.assertEqual('application/json; charset=utf-8', file_item.content_type) try: manifest = json.loads(manifest_body) except ValueError: self.fail("GET with multipart-manifest=get got invalid json") self.assertEqual(1, len(manifest)) key_map = {'etag': 'hash', 'size_bytes': 'bytes', 'path': 'name'} for k_client, k_slo in key_map.items(): self.assertEqual(self.seg_info[seg_name][k_client], manifest[0][k_slo]) def _assert_is_object(self, file_item, seg_name): file_contents = file_item.read() self.assertEqual(1024 * 1024, len(file_contents)) self.assertEqual(seg_name, file_contents[0]) self.assertEqual(seg_name, file_contents[-1]) def tearDown(self): # remove versioning to allow simple container delete self.container.update_metadata(hdrs={'X-Versions-Location': ''}) self.account.delete_containers() def test_slo_manifest_version(self): file_item = self._create_manifest('a') # sanity check: read the manifest, then the large object self._assert_is_manifest(file_item, 'a') self._assert_is_object(file_item, 'a') # upload new manifest file_item = self._create_manifest('b') # sanity check: read the manifest, then the large object self._assert_is_manifest(file_item, 'b') self._assert_is_object(file_item, 'b') versions_list = self.versions_container.files() self.assertEqual(1, len(versions_list)) version_file = self.versions_container.file(versions_list[0]) # check the version is still a manifest self._assert_is_manifest(version_file, 'a') self._assert_is_object(version_file, 'a') # delete the newest manifest file_item.delete() # expect the original manifest file to be restored self._assert_is_manifest(file_item, 'a') self._assert_is_object(file_item, 'a')
def get_cluster_info(): conn = Connection(config) conn.authenticate() global cluster_info cluster_info = conn.cluster_info()
def test_tempauth_account_acls(self): if skip: raise SkipTest # Determine whether this cluster has account ACLs; if not, skip test conn = Connection(get_config('func_test')) conn.authenticate() status = conn.make_request('GET', '/info', cfg={'verbatim_path': True}) if status // 100 != 2: # Can't tell if account ACLs are enabled; skip tests proactively. raise SkipTest else: cluster_info = json.loads(conn.response.read()) if not cluster_info.get('tempauth', {}).get('account_acls'): raise SkipTest if 'keystoneauth' in cluster_info: # Unfortunate hack -- tempauth (with account ACLs) is expected # to play nice with Keystone (without account ACLs), but Zuul # functest framework doesn't give us an easy way to get a # tempauth user. raise SkipTest def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def put(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('PUT', parsed.path, '', new_headers) return check_response(conn) def delete(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('DELETE', parsed.path, '', new_headers) return check_response(conn) def head(url, token, parsed, conn): conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) def get(url, token, parsed, conn): conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) try: # User1 can POST to their own account (and reset the ACLs) resp = retry(post, headers={'X-Account-Access-Control': '{}'}, use_account=1) resp.read() self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Account-Access-Control'), None) # User1 can GET their own empty account resp = retry(get, use_account=1) resp.read() self.assertEqual(resp.status // 100, 2) self.assertEqual(resp.getheader('X-Account-Access-Control'), None) # User2 can't GET User1's account resp = retry(get, use_account=2, url_account=1) resp.read() self.assertEqual(resp.status, 403) # User1 is swift_owner of their own account, so they can POST an # ACL -- let's do this and make User2 (test_user[1]) an admin acl_user = swift_testing.swift_test_user[1] acl = {'admin': [acl_user]} headers = { 'x-account-access-control': format_acl(version=2, acl_dict=acl) } resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # User1 can see the new header resp = retry(get, use_account=1) resp.read() self.assertEqual(resp.status // 100, 2) data_from_headers = resp.getheader('x-account-access-control') expected = json.dumps(acl, separators=(',', ':')) self.assertEqual(data_from_headers, expected) # Now User2 should be able to GET the account and see the ACL resp = retry(head, use_account=2, url_account=1) resp.read() data_from_headers = resp.getheader('x-account-access-control') self.assertEqual(data_from_headers, expected) # Revoke User2's admin access, grant User2 read-write access acl = {'read-write': [acl_user]} headers = { 'x-account-access-control': format_acl(version=2, acl_dict=acl) } resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # User2 can still GET the account, but not see the ACL # (since it's privileged data) resp = retry(head, use_account=2, url_account=1) resp.read() self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('x-account-access-control'), None) # User2 can PUT and DELETE a container resp = retry(put, use_account=2, url_account=1, resource='%(storage_url)s/mycontainer', headers={}) resp.read() self.assertEqual(resp.status, 201) resp = retry(delete, use_account=2, url_account=1, resource='%(storage_url)s/mycontainer', headers={}) resp.read() self.assertEqual(resp.status, 204) # Revoke User2's read-write access, grant User2 read-only access acl = {'read-only': [acl_user]} headers = { 'x-account-access-control': format_acl(version=2, acl_dict=acl) } resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # User2 can still GET the account, but not see the ACL # (since it's privileged data) resp = retry(head, use_account=2, url_account=1) resp.read() self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('x-account-access-control'), None) # User2 can't PUT a container resp = retry(put, use_account=2, url_account=1, resource='%(storage_url)s/mycontainer', headers={}) resp.read() self.assertEqual(resp.status, 403) finally: # Make sure to clean up even if tests fail -- User2 should not # have access to User1's account in other functional tests! resp = retry(post, headers={'X-Account-Access-Control': '{}'}, use_account=1) resp.read()
def test_tempauth_account_acls(self): if skip: raise SkipTest # Determine whether this cluster has account ACLs; if not, skip test conn = Connection(get_config('func_test')) conn.authenticate() cluster_info = conn.cluster_info() if not cluster_info.get('tempauth', {}).get('account_acls'): raise SkipTest if 'keystoneauth' in cluster_info: # Unfortunate hack -- tempauth (with account ACLs) is expected # to play nice with Keystone (without account ACLs), but Zuul # functest framework doesn't give us an easy way to get a # tempauth user. raise SkipTest def post(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('POST', parsed.path, '', new_headers) return check_response(conn) def put(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('PUT', parsed.path, '', new_headers) return check_response(conn) def delete(url, token, parsed, conn, headers): new_headers = dict({'X-Auth-Token': token}, **headers) conn.request('DELETE', parsed.path, '', new_headers) return check_response(conn) def head(url, token, parsed, conn): conn.request('HEAD', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) def get(url, token, parsed, conn): conn.request('GET', parsed.path, '', {'X-Auth-Token': token}) return check_response(conn) try: # User1 can POST to their own account (and reset the ACLs) resp = retry(post, headers={'X-Account-Access-Control': '{}'}, use_account=1) resp.read() self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('X-Account-Access-Control'), None) # User1 can GET their own empty account resp = retry(get, use_account=1) resp.read() self.assertEqual(resp.status // 100, 2) self.assertEqual(resp.getheader('X-Account-Access-Control'), None) # User2 can't GET User1's account resp = retry(get, use_account=2, url_account=1) resp.read() self.assertEqual(resp.status, 403) # User1 is swift_owner of their own account, so they can POST an # ACL -- let's do this and make User2 (test_user[1]) an admin acl_user = swift_testing.swift_test_user[1] acl = {'admin': [acl_user]} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # User1 can see the new header resp = retry(get, use_account=1) resp.read() self.assertEqual(resp.status // 100, 2) data_from_headers = resp.getheader('x-account-access-control') expected = json.dumps(acl, separators=(',', ':')) self.assertEqual(data_from_headers, expected) # Now User2 should be able to GET the account and see the ACL resp = retry(head, use_account=2, url_account=1) resp.read() data_from_headers = resp.getheader('x-account-access-control') self.assertEqual(data_from_headers, expected) # Revoke User2's admin access, grant User2 read-write access acl = {'read-write': [acl_user]} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # User2 can still GET the account, but not see the ACL # (since it's privileged data) resp = retry(head, use_account=2, url_account=1) resp.read() self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('x-account-access-control'), None) # User2 can PUT and DELETE a container resp = retry(put, use_account=2, url_account=1, resource='%(storage_url)s/mycontainer', headers={}) resp.read() self.assertEqual(resp.status, 201) resp = retry(delete, use_account=2, url_account=1, resource='%(storage_url)s/mycontainer', headers={}) resp.read() self.assertEqual(resp.status, 204) # Revoke User2's read-write access, grant User2 read-only access acl = {'read-only': [acl_user]} headers = {'x-account-access-control': format_acl( version=2, acl_dict=acl)} resp = retry(post, headers=headers, use_account=1) resp.read() self.assertEqual(resp.status, 204) # User2 can still GET the account, but not see the ACL # (since it's privileged data) resp = retry(head, use_account=2, url_account=1) resp.read() self.assertEqual(resp.status, 204) self.assertEqual(resp.getheader('x-account-access-control'), None) # User2 can't PUT a container resp = retry(put, use_account=2, url_account=1, resource='%(storage_url)s/mycontainer', headers={}) resp.read() self.assertEqual(resp.status, 403) finally: # Make sure to clean up even if tests fail -- User2 should not # have access to User1's account in other functional tests! resp = retry(post, headers={'X-Account-Access-Control': '{}'}, use_account=1) resp.read()
def get_cluster_info(): conn = Connection(conf) conn.authenticate() global cluster_info cluster_info = conn.cluster_info()