def __init__(self, store_api): LOG.info( _LI("Initializing scrubber with configuration: %s"), six.text_type({ 'registry_host': CONF.registry_host, 'registry_port': CONF.registry_port })) self.store_api = store_api registry.configure_registry_client() registry.configure_registry_admin_creds() # Here we create a request context with credentials to support # delayed delete when using multi-tenant backend storage admin_user = CONF.admin_user admin_tenant = CONF.admin_tenant_name if CONF.send_identity_headers: # When registry is operating in trusted-auth mode roles = [CONF.admin_role] self.admin_context = context.RequestContext(user=admin_user, tenant=admin_tenant, auth_token=None, roles=roles) self.registry = registry.get_registry_client(self.admin_context) else: ctxt = context.RequestContext() self.registry = registry.get_registry_client(ctxt) auth_token = self.registry.auth_token self.admin_context = context.RequestContext(user=admin_user, tenant=admin_tenant, auth_token=auth_token) self.db_queue = get_scrub_queue() self.pool = eventlet.greenpool.GreenPool(CONF.scrub_pool_size)
def test_v1_process_response_download_permitted(self): """ Test process_response for v1 api where member role able to download the image with custom property. """ image_id = 'test1' def fake_fetch_request_info(*args, **kwargs): return ('test1', 'GET', 'v1') def fake_get_v1_image_metadata(*args, **kwargs): return { 'id': image_id, 'name': 'fake_image', 'status': 'active', 'created_at': '', 'min_disk': '10G', 'min_ram': '1024M', 'protected': False, 'locations': '', 'checksum': 'c1234', 'owner': '', 'disk_format': 'raw', 'container_format': 'bare', 'size': '123456789', 'virtual_size': '123456789', 'is_public': 'public', 'deleted': False, 'updated_at': '', 'x_test_key': 'test_1234' } cache_filter = ProcessRequestTestCacheFilter() cache_filter._fetch_request_info = fake_fetch_request_info cache_filter._get_v1_image_metadata = fake_get_v1_image_metadata rules = { "restricted": "not ('test_1234':%(x_test_key)s and role:_member_)", "download_image": "role:admin or rule:restricted" } self.set_policy_rules(rules) cache_filter.policy = glance.api.policy.Enforcer() request = webob.Request.blank('/v1/images/%s' % image_id) request.context = context.RequestContext(roles=['member']) resp = webob.Response(request=request) actual = cache_filter.process_response(resp) self.assertEqual(resp, actual)
def test_v1_process_request_image_fetch(self): def fake_get_image_metadata(context, image_id): return {'is_public': True, 'deleted': False, 'size': '20'} def dummy_img_iterator(): for i in range(3): yield i image_id = 'test1' request = webob.Request.blank('/v1/images/%s' % image_id) request.context = context.RequestContext() cache_filter = ProcessRequestTestCacheFilter() self.stubs.Set(registry, 'get_image_metadata', fake_get_image_metadata) actual = cache_filter._process_v1_request(request, image_id, dummy_img_iterator) self.assertTrue(actual)
def test_process_request_without_download_image_policy(self): """ Test for cache middleware skip processing when request context has not 'download_image' role. """ image_id = 'test1' request = webob.Request.blank('/v1/images/%s' % image_id) request.context = context.RequestContext() cache_filter = ProcessRequestTestCacheFilter() rules = {'download_image': '!'} self.set_policy_rules(rules) cache_filter.policy = glance.api.policy.Enforcer() self.assertEqual(None, cache_filter.process_request(request))
def verify_scheme_or_exit(self, scheme): """ Verifies availability of the storage backend for the given scheme or exits :param scheme: The backend store scheme """ try: get_store_from_scheme(context.RequestContext(), scheme) except exception.UnknownScheme: msg = _("Store for scheme %s not found") LOG.error(msg % scheme) # message on stderr will only be visible if started directly via # bin/glance-api, as opposed to being daemonized by glance-control sys.stderr.write(msg % scheme) sys.exit(255)
def test_get_ksa_client(self, mock_session, mock_token): # Make sure we can get a keystoneauth1 client from our context # with the token auth as expected. ctx = context.RequestContext(auth_token='token') # NOTE(danms): The auth config group and options are # dynamically registered. Tickling enough of the relevant # code to make that happen would significantly inflate the # amount of code here for no real gain, so we just mock the # CONF object. CONF.register_group(cfg.OptGroup('keystone_authtoken')) with mock.patch.object(CONF, 'keystone_authtoken') as ksat: ksat.identity_uri = 'http://keystone' client = context.get_ksa_client(ctx) self.assertEqual(mock_session.return_value, client) mock_session.assert_called_once_with(auth=mock_token.return_value) mock_token.assert_called_once_with('http://keystone', 'token')
def setUp(self): super(TestImportTask, self).setUp() glance_store.register_opts(CONF) self.config(default_store='file', stores=['file', 'http'], filesystem_store_datadir=self.test_dir, group="glance_store") glance_store.create_stores(CONF) self.work_dir = os.path.join(self.test_dir, 'work_dir') utils.safe_mkdirs(self.work_dir) self.config(work_dir=self.work_dir, group='task') self.context = context.RequestContext(user_id=TENANT1, project_id=TENANT1, overwrite=False) self.img_repo = mock.MagicMock() self.task_repo = mock.MagicMock() self.gateway = gateway.Gateway() self.task_factory = domain.TaskFactory() self.img_factory = self.gateway.get_image_factory(self.context) self.image = self.img_factory.new_image(image_id=UUID1, disk_format='qcow2', container_format='bare') task_input = { "import_from": "http://cloud.foo/image.qcow2", "import_from_format": "qcow2", "image_properties": { 'disk_format': 'qcow2', 'container_format': 'bare' } } task_ttl = CONF.task.task_time_to_live self.task_type = 'import' request_id = 'fake_request_id' user_id = 'fake_user' self.task = self.task_factory.new_task(self.task_type, TENANT1, UUID1, user_id, request_id, task_time_to_live=task_ttl, task_input=task_input)
def test_v2_process_request_response_headers(self): def dummy_img_iterator(): for i in range(3): yield i def fake_image_get(self, image_id): return { 'id': 'test1', 'name': 'fake_image', 'status': 'active', 'created_at': '', 'min_disk': '10G', 'min_ram': '1024M', 'protected': False, 'locations': '', 'checksum': 'c352f4e7121c6eae958bc1570324f17e', 'owner': '', 'disk_format': 'raw', 'container_format': 'bare', 'size': '123456789', 'virtual_size': '123456789', 'is_public': 'public', 'deleted': False, 'updated_at': '', 'properties': {}, } def fake_image_tag_get_all(context, image_id, session=None): return None image_id = 'test1' request = webob.Request.blank('/v2/images/test1/file') request.context = context.RequestContext() self.stubs.Set(db, 'image_get', fake_image_get) self.stubs.Set(db, 'image_tag_get_all', fake_image_tag_get_all) cache_filter = ProcessRequestTestCacheFilter() response = cache_filter._process_v2_request( request, image_id, dummy_img_iterator) self.assertEqual(response.headers['Content-Type'], 'application/octet-stream') self.assertEqual(response.headers['Content-MD5'], 'c352f4e7121c6eae958bc1570324f17e') self.assertEqual(response.headers['Content-Length'], '123456789')
def _delete(self, id, uri, now): file_path = os.path.join(self.datadir, str(id)) try: LOG.debug(_("Deleting %(uri)s") % {'uri': uri}) # Here we create a request context with credentials to support # delayed delete when using multi-tenant backend storage ctx = context.RequestContext(auth_tok=self.registry.auth_tok, user=self.admin_user, tenant=self.admin_tenant) store.delete_from_backend(ctx, uri) except store.UnsupportedBackend: msg = _("Failed to delete image from store (%(uri)s).") LOG.error(msg % {'uri': uri}) write_queue_file(file_path, uri, now) self.registry.update_image(id, {'status': 'deleted'}) utils.safe_remove(file_path)
def setUp(self): """Establish a clean test environment""" super(TestRegistryV1Client, self).setUp() db_api.get_engine() self.context = context.RequestContext(is_admin=True) self.FIXTURES = [ self.get_fixture( id=UUID1, name='fake image #1', is_public=False, disk_format='ami', container_format='ami', size=13, location="swift://*****:*****@acct/container/obj.tar.0", properties={'type': 'kernel'}), self.get_fixture(id=UUID2, name='fake image #2', properties={}, size=19, location="file:///tmp/glance-tests/2")] self.destroy_fixtures() self.create_fixtures() self.client = rclient.RegistryClient("0.0.0.0")
def test_process_response(self): def fake_fetch_request_info(*args, **kwargs): return ('test1', 'GET', 'v1') def fake_get_v1_image_metadata(*args, **kwargs): return {'properties': {}} cache_filter = ProcessRequestTestCacheFilter() cache_filter._fetch_request_info = fake_fetch_request_info cache_filter._get_v1_image_metadata = fake_get_v1_image_metadata image_id = 'test1' request = webob.Request.blank('/v1/images/%s' % image_id) request.context = context.RequestContext() headers = {"x-image-meta-deleted": True} resp = webob.Response(request=request, headers=headers) actual = cache_filter.process_response(resp) self.assertEqual(resp, actual)
def test_process_v1_request_for_deleted_but_cached_image(self): """ Test for determining image is deleted from cache when it is not found in Glance Registry. """ def fake_process_v1_request(request, image_id, image_iterator): raise exception.NotFound() image_id = 'test1' request = webob.Request.blank('/v1/images/%s' % image_id) request.context = context.RequestContext() cache_filter = ProcessRequestTestCacheFilter() self.stubs.Set(cache_filter, '_process_v1_request', fake_process_v1_request) cache_filter.process_request(request) self.assertTrue(image_id in cache_filter.cache.deleted_images)
def fetch_image_into_cache(self, image_id): ctx = context.RequestContext(is_admin=True, show_deleted=True) try: image_meta = registry.get_image_metadata(ctx, image_id) if image_meta['status'] != 'active': LOG.warn(_("Image '%s' is not active. Not caching."), image_id) return False except exception.NotFound: LOG.warn(_("No metadata found for image '%s'"), image_id) return False image_data, image_size = get_from_backend(ctx, image_meta['location']) LOG.debug(_("Caching image '%s'"), image_id) self.cache.cache_image_iter(image_id, image_data) return True
def test_v1_deleted_image_fetch(self): """ Test for determining that when an admin tries to download a deleted image it returns 404 Not Found error. """ def fake_get_image_metadata(context, image_id): return {'deleted': True} def dummy_img_iterator(): for i in range(3): yield i image_id = 'test1' request = webob.Request.blank('/v1/images/%s' % image_id) request.context = context.RequestContext() cache_filter = ProcessRequestTestCacheFilter() self.stubs.Set(registry, 'get_image_metadata', fake_get_image_metadata) self.assertRaises(exception.NotFound, cache_filter._process_v1_request, request, image_id, dummy_img_iterator)
def test_v2_process_request_response_headers(self): def dummy_img_iterator(): for i in range(3): yield i image_id = 'test1' request = webob.Request.blank('/v2/images/test1/file') request.context = context.RequestContext() image_meta = { 'id': image_id, 'name': 'fake_image', 'status': 'active', 'created_at': '', 'min_disk': '10G', 'min_ram': '1024M', 'protected': False, 'locations': '', 'checksum': 'c1234', 'owner': '', 'disk_format': 'raw', 'container_format': 'bare', 'size': '123456789', 'virtual_size': '123456789', 'is_public': 'public', 'deleted': False, 'updated_at': '', 'properties': {}, } image = ImageStub(image_id, request.context.project_id) request.environ['api.cache.image'] = image for k, v in image_meta.items(): setattr(image, k, v) cache_filter = ProcessRequestTestCacheFilter() response = cache_filter._process_v2_request(request, image_id, dummy_img_iterator, image_meta) self.assertEqual('application/octet-stream', response.headers['Content-Type']) self.assertEqual('c1234', response.headers['Content-MD5']) self.assertEqual('123456789', response.headers['Content-Length'])
def test_process_request_without_download_image_policy(self): """ Test for cache middleware skip processing when request context has not 'download_image' role. """ def fake_get_v2_image_metadata(*args, **kwargs): return {'status': 'active', 'properties': {}} image_id = 'test1' request = webob.Request.blank('/v2/images/%s/file' % image_id) request.context = context.RequestContext() cache_filter = ProcessRequestTestCacheFilter() cache_filter._get_v2_image_metadata = fake_get_v2_image_metadata enforcer = self._enforcer_from_rules({'download_image': '!'}) cache_filter.policy = enforcer self.assertRaises(webob.exc.HTTPForbidden, cache_filter.process_request, request)
def _test_prefetcher(self, mock_get_db): self.config(enabled_backends={'cheap': 'file'}) store.register_store_opts(CONF) self.config(filesystem_store_datadir='/tmp', group='cheap') store.create_multi_stores(CONF) tempf = tempfile.NamedTemporaryFile() tempf.write(b'foo') db = unit_test_utils.FakeDB(initialize=False) mock_get_db.return_value = db ctx = context.RequestContext(is_admin=True, roles=['admin']) gateway = glance_gateway.Gateway() image_factory = gateway.get_image_factory(ctx, authorization_layer=False) image_repo = gateway.get_repo(ctx, authorization_layer=False) fetcher = prefetcher.Prefetcher() # Create an image with no values set and queue it image = image_factory.new_image() image_repo.add(image) fetcher.cache.queue_image(image.image_id) # Image is not active, so it should fail to cache, but remain queued self.assertFalse(fetcher.run()) self.assertFalse(fetcher.cache.is_cached(image.image_id)) self.assertTrue(fetcher.cache.is_queued(image.image_id)) # Set the disk/container format and give it a location image.disk_format = 'raw' image.container_format = 'bare' image.status = 'active' loc = {'url': 'file://%s' % tempf.name, 'metadata': {'store': 'cheap'}} with mock.patch('glance.location._check_image_location'): # FIXME(danms): Why do I have to do this? image.locations = [loc] image_repo.save(image) # Image is now active and has a location, so it should cache self.assertTrue(fetcher.run()) self.assertTrue(fetcher.cache.is_cached(image.image_id)) self.assertFalse(fetcher.cache.is_queued(image.image_id))
def test_v1_process_request_download_restricted(self): """ Test process_request for v1 api where _member_ role not able to download the image with custom property. """ image_id = 'test1' def fake_get_v1_image_metadata(*args, **kwargs): return { 'id': image_id, 'name': 'fake_image', 'status': 'active', 'created_at': '', 'min_disk': '10G', 'min_ram': '1024M', 'protected': False, 'locations': '', 'checksum': 'c1234', 'owner': '', 'disk_format': 'raw', 'container_format': 'bare', 'size': '123456789', 'virtual_size': '123456789', 'is_public': 'public', 'deleted': False, 'updated_at': '', 'x_test_key': 'test_1234' } enforcer = self._enforcer_from_rules({ "restricted": "not ('test_1234':%(x_test_key)s and role:_member_)", "download_image": "role:admin or rule:restricted" }) request = webob.Request.blank('/v1/images/%s' % image_id) request.context = context.RequestContext(roles=['_member_']) cache_filter = ProcessRequestTestCacheFilter() cache_filter._get_v1_image_metadata = fake_get_v1_image_metadata cache_filter.policy = enforcer self.assertRaises(webob.exc.HTTPForbidden, cache_filter.process_request, request)
def setUp(self): """Establish a clean test environment""" super(TestRegistryV1Client, self).setUp() db_api.setup_db_env() db_api.get_engine() self.context = context.RequestContext(is_admin=True) self.FIXTURES = [{ 'id': UUID1, 'name': 'fake image #1', 'status': 'active', 'disk_format': 'ami', 'container_format': 'ami', 'is_public': False, 'created_at': timeutils.utcnow(), 'updated_at': timeutils.utcnow(), 'deleted_at': None, 'deleted': False, 'checksum': None, 'size': 13, 'location': "swift://*****:*****@acct/container/obj.tar.0", 'properties': { 'type': 'kernel' } }, { 'id': UUID2, 'name': 'fake image #2', 'status': 'active', 'disk_format': 'vhd', 'container_format': 'ovf', 'is_public': True, 'created_at': timeutils.utcnow(), 'updated_at': timeutils.utcnow(), 'deleted_at': None, 'deleted': False, 'checksum': None, 'size': 19, 'location': "file:///tmp/glance-tests/2", 'properties': {} }] self.destroy_fixtures() self.create_fixtures() self.client = rclient.RegistryClient("0.0.0.0")
def _delete_image_from_backend(self, image_id, uri): if CONF.metadata_encryption_key is not None: uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri) try: LOG.debug("Deleting URI from image %(image_id)s." % {'image_id': image_id}) # Here we create a request context with credentials to support # delayed delete when using multi-tenant backend storage admin_tenant = CONF.admin_tenant_name auth_token = self.registry.auth_tok admin_context = context.RequestContext(user=CONF.admin_user, tenant=admin_tenant, auth_tok=auth_token) self.store_api.delete_from_backend(admin_context, uri) except Exception: msg = _("Failed to delete URI from image %(image_id)s") LOG.error(msg % {'image_id': image_id})
def test_process_request_without_download_image_policy(self): """ Test for cache middleware skip processing when request context has not 'download_image' role. """ def fake_get_v1_image_metadata(*args, **kwargs): return {'properties': {}} image_id = 'test1' request = webob.Request.blank('/v1/images/%s' % image_id) request.context = context.RequestContext() cache_filter = ProcessRequestTestCacheFilter() cache_filter._get_v1_image_metadata = fake_get_v1_image_metadata rules = {'download_image': '!'} self.set_policy_rules(rules) cache_filter.policy = glance.api.policy.Enforcer() self.assertRaises(webob.exc.HTTPForbidden, cache_filter.process_request, request)
def process_request(self, req): auth_tok = req.headers.get('X-Auth-Token') user = None tenant = None roles = [] if auth_tok: user, tenant, role = auth_tok.split(':') roles = [role] req.headers['X-User-Id'] = user req.headers['X-Tenant-Id'] = tenant req.headers['X-Roles'] = role req.headers['X-Identity-Status'] = 'Confirmed' kwargs = { 'user': user, 'tenant': tenant, 'roles': roles, 'is_admin': self.is_admin, } req.context = context.RequestContext(**kwargs)
def __init__(self, store_api): LOG.info( _("Initializing scrubber with configuration: %s") % unicode({ 'scrubber_datadir': CONF.scrubber_datadir, 'cleanup': CONF.cleanup_scrubber, 'cleanup_time': CONF.cleanup_scrubber_time, 'registry_host': CONF.registry_host, 'registry_port': CONF.registry_port })) utils.safe_mkdirs(CONF.scrubber_datadir) self.store_api = store_api registry.configure_registry_client() registry.configure_registry_admin_creds() self.registry = registry.get_registry_client(context.RequestContext()) (self.file_queue, self.db_queue) = get_scrub_queues()
def do_sharable(self, exp_res, img_owner, membership=None, **kwargs): """ Perform a context sharability test. Creates a (fake) image with the specified owner and is_public attributes, then creates a context with the given keyword arguments and expects exp_res as the result of an is_image_sharable() call on the context. If membership is not None, its value will be passed in as the 'membership' keyword argument of is_image_sharable(). """ img = _fake_image(img_owner, True) ctx = context.RequestContext(**kwargs) sharable_args = {} if membership is not None: sharable_args['membership'] = membership output = self.db_api.is_image_sharable(ctx, img, **sharable_args) self.assertEqual(exp_res, output)
def test_v1_process_request_download_without_restricted_property(self): """ Test process_request for v1 api where _member_ role able to download the image if restricted property is not added to the image. """ image_id = 'test1' def fake_get_v1_image_metadata(*args, **kwargs): return { 'id': image_id, 'name': 'fake_image', 'status': 'active', 'created_at': '', 'min_disk': '10G', 'min_ram': '1024M', 'protected': False, 'locations': '', 'checksum': 'c1234', 'owner': '', 'disk_format': 'raw', 'container_format': 'bare', 'size': '123456789', 'virtual_size': '123456789', 'is_public': 'public', 'deleted': False, 'updated_at': '' } request = webob.Request.blank('/v1/images/%s' % image_id) request.context = context.RequestContext(roles=['_member_']) cache_filter = ProcessRequestTestCacheFilter() cache_filter._get_v1_image_metadata = fake_get_v1_image_metadata rules = { "restricted": "not ('test_1234':%(x_test_key)s and role:_member_)", "download_image": "role:admin or rule:restricted" } self.set_policy_rules(rules) cache_filter.policy = glance.api.policy.Enforcer() actual = cache_filter.process_request(request) self.assertTrue(actual)
def _delete(self, id, uri, now): file_path = os.path.join(self.datadir, str(id)) if CONF.metadata_encryption_key is not None: uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri) try: LOG.debug(_("Deleting %(id)s") % {'id': id}) # Here we create a request context with credentials to support # delayed delete when using multi-tenant backend storage ctx = context.RequestContext(auth_tok=self.registry.auth_tok, user=self.admin_user, tenant=self.admin_tenant) store.delete_from_backend(ctx, uri) except store.UnsupportedBackend: msg = _("Failed to delete image from store (%(id)s).") LOG.error(msg % {'id': id}) except exception.NotFound: msg = _("Image not found in store (%(id)s).") LOG.error(msg % {'id': id}) self.registry.update_image(id, {'status': 'deleted'}) utils.safe_remove(file_path)
def setUp(self): """Establish a clean test environment""" super(TestRegistryV2Client, self).setUp() db_api.get_engine() self.context = context.RequestContext(is_admin=True) uuid1_time = timeutils.utcnow() uuid2_time = uuid1_time + datetime.timedelta(seconds=5) self.FIXTURES = [ self.get_extra_fixture( id=UUID1, name='fake image #1', is_public=False, disk_format='ami', container_format='ami', size=13, virtual_size=26, properties={'type': 'kernel'}, location="swift://*****:*****@acct/container/obj.tar.0", created_at=uuid1_time), self.get_extra_fixture(id=UUID2, name='fake image #2', properties={}, size=19, virtual_size=38, location="file:///tmp/glance-tests/2", created_at=uuid2_time)] self.destroy_fixtures() self.create_fixtures() self.client = rclient.RegistryClient("0.0.0.0")
def test_process_response_without_download_image_policy(self): """ Test for cache middleware raise webob.exc.HTTPForbidden directly when request context has not 'download_image' role. """ def fake_fetch_request_info(*args, **kwargs): return ('test1', 'GET') cache_filter = ProcessRequestTestCacheFilter() cache_filter._fetch_request_info = fake_fetch_request_info rules = {'download_image': '!'} self.set_policy_rules(rules) cache_filter.policy = glance.api.policy.Enforcer() image_id = 'test1' request = webob.Request.blank('/v1/images/%s' % image_id) request.context = context.RequestContext() resp = webob.Response(request=request) self.assertRaises(webob.exc.HTTPForbidden, cache_filter.process_response, resp) self.assertEqual([''], resp.app_iter)
def fetch_image_into_cache(self, image_id): ctx = context.RequestContext(is_admin=True, show_deleted=True) try: image_meta = registry.get_image_metadata(ctx, image_id) if image_meta['status'] != 'active': LOG.warn(_("Image '%s' is not active. Not caching."), image_id) return False except exception.NotFound: LOG.warn(_("No metadata found for image '%s'"), image_id) return False location = image_meta['location'] image_data, image_size = glance.store.get_from_backend(ctx, location) LOG.debug(_("Caching image '%s'"), image_id) cache_tee_iter = self.cache.cache_tee_iter(image_id, image_data, image_meta['checksum']) # Image is tee'd into cache and checksum verified # as we iterate list(cache_tee_iter) return True
def test_v1_remove_location_image_fetch(self): class CheckNoLocationDataSerializer(object): def show(self, response, raw_response): return 'location_data' in raw_response['image_meta'] def dummy_img_iterator(): for i in range(3): yield i image_id = 'test1' image_meta = { 'id': image_id, 'name': 'fake_image', 'status': 'active', 'created_at': '', 'min_disk': '10G', 'min_ram': '1024M', 'protected': False, 'locations': '', 'checksum': 'c1234', 'owner': '', 'disk_format': 'raw', 'container_format': 'bare', 'size': '123456789', 'virtual_size': '123456789', 'is_public': 'public', 'deleted': False, 'updated_at': '', 'properties': {}, } request = webob.Request.blank('/v1/images/%s' % image_id) request.context = context.RequestContext() cache_filter = ProcessRequestTestCacheFilter() cache_filter.serializer = CheckNoLocationDataSerializer() actual = cache_filter._process_v1_request( request, image_id, dummy_img_iterator, image_meta) self.assertFalse(actual)