def __init__(self, store_api):
LOG.info(_LI("Initializing scrubber with configuration: %s"),
six.text_type({'registry_host': CONF.registry_host,
'registry_port': CONF.registry_port}))
self.store_api = store_api
registry.configure_registry_client()
registry.configure_registry_admin_creds()
# Here we create a request context with credentials to support
# delayed delete when using multi-tenant backend storage
admin_user = CONF.admin_user
admin_tenant = CONF.admin_tenant_name
if CONF.send_identity_headers:
# When registry is operating in trusted-auth mode
roles = [CONF.admin_role]
self.admin_context = context.RequestContext(user=admin_user,
tenant=admin_tenant,
auth_token=None,
roles=roles)
self.registry = registry.get_registry_client(self.admin_context)
else:
ctxt = context.RequestContext()
self.registry = registry.get_registry_client(ctxt)
auth_token = self.registry.auth_token
self.admin_context = context.RequestContext(user=admin_user,
tenant=admin_tenant,
auth_token=auth_token)
self.db_queue = get_scrub_queue()
self.pool = eventlet.greenpool.GreenPool(CONF.scrub_pool_size)
def test_artifact_create_same_version_different_users(self):
tenant1 = str(uuid.uuid4())
tenant2 = str(uuid.uuid4())
ctx1 = context.RequestContext(is_admin=False, tenant=tenant1)
ctx2 = context.RequestContext(is_admin=False, tenant=tenant2)
artifact1 = get_fixture(owner=tenant1)
artifact2 = get_fixture(owner=tenant2)
self.db_api.artifact_create(ctx1, artifact1, TYPE_NAME, TYPE_VERSION)
self.assertIsNotNone(
self.db_api.artifact_create(ctx2, artifact2, TYPE_NAME,
TYPE_VERSION))
def test_artifact_get_owned(self):
tenant1 = str(uuid.uuid4())
tenant2 = str(uuid.uuid4())
ctx1 = context.RequestContext(is_admin=False, tenant=tenant1)
ctx2 = context.RequestContext(is_admin=False, tenant=tenant2)
artifact = get_fixture(owner=tenant1)
created = self.db_api.artifact_create(ctx1, artifact, TYPE_NAME,
TYPE_VERSION)
self.assertIsNotNone(
self.db_api.artifact_get(ctx1, created['id'], TYPE_NAME,
TYPE_VERSION))
self.assertRaises(exc.ArtifactForbidden, self.db_api.artifact_get,
ctx2, created['id'], TYPE_NAME, TYPE_VERSION)
def test_artifact_get_public(self):
tenant1 = str(uuid.uuid4())
tenant2 = str(uuid.uuid4())
ctx1 = context.RequestContext(is_admin=False, tenant=tenant1)
ctx2 = context.RequestContext(is_admin=False, tenant=tenant2)
artifact = get_fixture(owner=tenant1, visibility='public')
created = self.db_api.artifact_create(ctx1, artifact, TYPE_NAME,
TYPE_VERSION)
self.assertIsNotNone(
self.db_api.artifact_get(ctx1, created['id'], TYPE_NAME,
TYPE_VERSION))
self.assertIsNotNone(
self.db_api.artifact_get(ctx2, created['id'], TYPE_NAME,
TYPE_VERSION))
def test_v1_deleted_image_fetch(self):
"""
Test for determining that when an admin tries to download a deleted
image it returns 404 Not Found error.
"""
def dummy_img_iterator():
for i in range(3):
yield i
image_id = 'test1'
image_meta = {
'id': image_id,
'name': 'fake_image',
'status': 'deleted',
'created_at': '',
'min_disk': '10G',
'min_ram': '1024M',
'protected': False,
'locations': '',
'checksum': 'c1234',
'owner': '',
'disk_format': 'raw',
'container_format': 'bare',
'size': '123456789',
'virtual_size': '123456789',
'is_public': 'public',
'deleted': True,
'updated_at': '',
'properties': {},
}
request = webob.Request.blank('/v1/images/%s' % image_id)
request.context = context.RequestContext()
cache_filter = ProcessRequestTestCacheFilter()
self.assertRaises(exception.NotFound, cache_filter._process_v1_request,
request, image_id, dummy_img_iterator, image_meta)
def test_v1_process_request_image_fetch(self):
def dummy_img_iterator():
for i in range(3):
yield i
image_id = 'test1'
image_meta = {
'id': image_id,
'name': 'fake_image',
'status': 'active',
'created_at': '',
'min_disk': '10G',
'min_ram': '1024M',
'protected': False,
'locations': '',
'checksum': 'c1234',
'owner': '',
'disk_format': 'raw',
'container_format': 'bare',
'size': '123456789',
'virtual_size': '123456789',
'is_public': 'public',
'deleted': False,
'updated_at': '',
'properties': {},
}
request = webob.Request.blank('/v1/images/%s' % image_id)
request.context = context.RequestContext()
cache_filter = ProcessRequestTestCacheFilter()
actual = cache_filter._process_v1_request(request, image_id,
dummy_img_iterator,
image_meta)
self.assertTrue(actual)
def test_v1_process_response_image_meta_not_found(self):
"""
Test process_response for v1 api where registry raises NotFound
exception as image metadata not found.
"""
image_id = 'test1'
def fake_fetch_request_info(*args, **kwargs):
return ('test1', 'GET', 'v1')
def fake_get_v1_image_metadata(*args, **kwargs):
raise exception.NotFound()
cache_filter = ProcessRequestTestCacheFilter()
cache_filter._fetch_request_info = fake_fetch_request_info
self.stubs.Set(registry, 'get_image_metadata',
fake_get_v1_image_metadata)
rules = {
"restricted": "not ('test_1234':%(x_test_key)s and role:_member_)",
"download_image": "role:admin or rule:restricted"
}
self.set_policy_rules(rules)
cache_filter.policy = xmonitor.api.policy.Enforcer()
request = webob.Request.blank('/v1/images/%s' % image_id)
request.context = context.RequestContext(roles=['_member_'])
resp = webob.Response(request=request)
self.assertRaises(webob.exc.HTTPNotFound,
cache_filter.process_response, resp)
def test_v2_process_response_download_permitted(self):
"""
Test process_response for v2 api where member role able to
download the image with custom property.
"""
image_id = 'test1'
extra_properties = {'x_test_key': 'test_1234'}
def fake_fetch_request_info(*args, **kwargs):
return ('test1', 'GET', 'v2')
def fake_get_v2_image_metadata(*args, **kwargs):
image = ImageStub(image_id, extra_properties=extra_properties)
request.environ['api.cache.image'] = image
return xmonitor.api.policy.ImageTarget(image)
cache_filter = ProcessRequestTestCacheFilter()
cache_filter._fetch_request_info = fake_fetch_request_info
cache_filter._get_v2_image_metadata = fake_get_v2_image_metadata
rules = {
"restricted": "not ('test_1234':%(x_test_key)s and role:_member_)",
"download_image": "role:admin or rule:restricted"
}
self.set_policy_rules(rules)
cache_filter.policy = xmonitor.api.policy.Enforcer()
request = webob.Request.blank('/v2/images/test1/file')
request.context = context.RequestContext(roles=['member'])
resp = webob.Response(request=request)
actual = cache_filter.process_response(resp)
self.assertEqual(resp, actual)
def setUp(self):
"""Establish a clean test environment"""
super(TestRegistryV2Client, self).setUp()
db_api.get_engine()
self.context = context.RequestContext(is_admin=True)
uuid1_time = timeutils.utcnow()
uuid2_time = uuid1_time + datetime.timedelta(seconds=5)
self.FIXTURES = [
self.get_extra_fixture(
id=UUID1,
name='fake image #1',
is_public=False,
disk_format='ami',
container_format='ami',
size=13,
virtual_size=26,
properties={'type': 'kernel'},
location="swift://*****:*****@acct/container/obj.tar.0",
created_at=uuid1_time),
self.get_extra_fixture(id=UUID2,
name='fake image #2',
properties={},
size=19,
virtual_size=38,
location="file:///tmp/xmonitor-tests/2",
created_at=uuid2_time)
]
self.destroy_fixtures()
self.create_fixtures()
self.client = rclient.RegistryClient("0.0.0.0")
def test_v2_process_request_download_restricted(self):
"""
Test process_request for v2 api where _member_ role not able to
download the image with custom property.
"""
image_id = 'test1'
extra_properties = {'x_test_key': 'test_1234'}
def fake_get_v2_image_metadata(*args, **kwargs):
image = ImageStub(image_id, extra_properties=extra_properties)
request.environ['api.cache.image'] = image
return xmonitor.api.policy.ImageTarget(image)
enforcer = self._enforcer_from_rules({
"restricted":
"not ('test_1234':%(x_test_key)s and role:_member_)",
"download_image":
"role:admin or rule:restricted"
})
request = webob.Request.blank('/v2/images/test1/file')
request.context = context.RequestContext(roles=['_member_'])
cache_filter = ProcessRequestTestCacheFilter()
cache_filter._get_v2_image_metadata = fake_get_v2_image_metadata
cache_filter.policy = enforcer
self.assertRaises(webob.exc.HTTPForbidden,
cache_filter.process_request, request)
def test_process_response_without_download_image_policy(self):
"""
Test for cache middleware raise webob.exc.HTTPForbidden directly
when request context has not 'download_image' role.
"""
def fake_fetch_request_info(*args, **kwargs):
return ('test1', 'GET', 'v1')
def fake_get_v1_image_metadata(*args, **kwargs):
return {'properties': {}}
cache_filter = ProcessRequestTestCacheFilter()
cache_filter._fetch_request_info = fake_fetch_request_info
cache_filter._get_v1_image_metadata = fake_get_v1_image_metadata
rules = {'download_image': '!'}
self.set_policy_rules(rules)
cache_filter.policy = xmonitor.api.policy.Enforcer()
image_id = 'test1'
request = webob.Request.blank('/v1/images/%s' % image_id)
request.context = context.RequestContext()
resp = webob.Response(request=request)
self.assertRaises(webob.exc.HTTPForbidden,
cache_filter.process_response, resp)
self.assertEqual([b''], resp.app_iter)
def setUp(self):
"""Establish a clean test environment"""
super(TestRegistryV1Client, self).setUp()
db_api.get_engine()
self.context = context.RequestContext(is_admin=True)
self.FIXTURES = [
self.get_fixture(
id=UUID1,
name='fake image #1',
is_public=False,
disk_format='ami',
container_format='ami',
size=13,
location="swift://*****:*****@acct/container/obj.tar.0",
properties={'type': 'kernel'}),
self.get_fixture(id=UUID2,
name='fake image #2',
properties={},
size=19,
location="file:///tmp/xmonitor-tests/2")
]
self.destroy_fixtures()
self.create_fixtures()
self.client = rclient.RegistryClient("0.0.0.0")
def test_user_identity(self):
ctx = context.RequestContext(user="******",
tenant="tenant",
domain="domain",
user_domain="user-domain",
project_domain="project-domain")
self.assertEqual('user tenant domain user-domain project-domain',
ctx.to_dict()["user_identity"])
def test_process_v1_DELETE_response(self):
image_id = 'test1'
request = webob.Request.blank('/v1/images/%s' % image_id)
request.context = context.RequestContext()
cache_filter = ProcessRequestTestCacheFilter()
headers = {"x-image-meta-deleted": True}
resp = webob.Response(request=request, headers=headers)
actual = cache_filter._process_DELETE_response(resp, image_id)
self.assertEqual(resp, actual)
def test_registry_invalid_token_exception_handling(self):
self.image_controller = acontroller()
request = webob.Request.blank('/images')
request.method = 'GET'
request.context = context.RequestContext()
with patch.object(rapi, 'get_images_detail') as mock_detail:
mock_detail.side_effect = exception.NotAuthenticated()
self.assertRaises(webob.exc.HTTPUnauthorized,
self.image_controller.detail, request)
def do_visible(self, exp_res, img_owner, img_public, **kwargs):
"""
Perform a context visibility test. Creates a (fake) image
with the specified owner and is_public attributes, then
creates a context with the given keyword arguments and expects
exp_res as the result of an is_image_visible() call on the
context.
"""
img = _fake_image(img_owner, img_public)
ctx = context.RequestContext(**kwargs)
self.assertEqual(exp_res, self.db_api.is_image_visible(ctx, img))
def __init__(self):
self.scrub_time = CONF.scrub_time
self.metadata_encryption_key = CONF.metadata_encryption_key
registry.configure_registry_client()
registry.configure_registry_admin_creds()
admin_user = CONF.admin_user
admin_tenant = CONF.admin_tenant_name
if CONF.send_identity_headers:
# When registry is operating in trusted-auth mode
roles = [CONF.admin_role]
self.admin_context = context.RequestContext(user=admin_user,
tenant=admin_tenant,
auth_token=None,
roles=roles)
self.registry = registry.get_registry_client(self.admin_context)
else:
ctxt = context.RequestContext()
self.registry = registry.get_registry_client(ctxt)
admin_token = self.registry.auth_token
self.admin_context = context.RequestContext(user=admin_user,
tenant=admin_tenant,
auth_token=admin_token)
def test_v1_process_response_download_permitted(self):
"""
Test process_response for v1 api where member role able to
download the image with custom property.
"""
image_id = 'test1'
def fake_fetch_request_info(*args, **kwargs):
return ('test1', 'GET', 'v1')
def fake_get_v1_image_metadata(*args, **kwargs):
return {
'id': image_id,
'name': 'fake_image',
'status': 'active',
'created_at': '',
'min_disk': '10G',
'min_ram': '1024M',
'protected': False,
'locations': '',
'checksum': 'c1234',
'owner': '',
'disk_format': 'raw',
'container_format': 'bare',
'size': '123456789',
'virtual_size': '123456789',
'is_public': 'public',
'deleted': False,
'updated_at': '',
'x_test_key': 'test_1234'
}
cache_filter = ProcessRequestTestCacheFilter()
cache_filter._fetch_request_info = fake_fetch_request_info
cache_filter._get_v1_image_metadata = fake_get_v1_image_metadata
rules = {
"restricted": "not ('test_1234':%(x_test_key)s and role:_member_)",
"download_image": "role:admin or rule:restricted"
}
self.set_policy_rules(rules)
cache_filter.policy = xmonitor.api.policy.Enforcer()
request = webob.Request.blank('/v1/images/%s' % image_id)
request.context = context.RequestContext(roles=['member'])
resp = webob.Response(request=request)
actual = cache_filter.process_response(resp)
self.assertEqual(resp, actual)
def test_process_response(self):
def fake_fetch_request_info(*args, **kwargs):
return ('test1', 'GET', 'v1')
def fake_get_v1_image_metadata(*args, **kwargs):
return {'properties': {}}
cache_filter = ProcessRequestTestCacheFilter()
cache_filter._fetch_request_info = fake_fetch_request_info
cache_filter._get_v1_image_metadata = fake_get_v1_image_metadata
image_id = 'test1'
request = webob.Request.blank('/v1/images/%s' % image_id)
request.context = context.RequestContext()
headers = {"x-image-meta-deleted": True}
resp = webob.Response(request=request, headers=headers)
actual = cache_filter.process_response(resp)
self.assertEqual(resp, actual)
def test_process_request_without_download_image_policy(self):
"""
Test for cache middleware skip processing when request
context has not 'download_image' role.
"""
def fake_get_v1_image_metadata(*args, **kwargs):
return {'status': 'active', 'properties': {}}
image_id = 'test1'
request = webob.Request.blank('/v1/images/%s' % image_id)
request.context = context.RequestContext()
cache_filter = ProcessRequestTestCacheFilter()
cache_filter._get_v1_image_metadata = fake_get_v1_image_metadata
enforcer = self._enforcer_from_rules({'download_image': '!'})
cache_filter.policy = enforcer
self.assertRaises(webob.exc.HTTPForbidden,
cache_filter.process_request, request)
def test_v1_process_request_download_restricted(self):
"""
Test process_request for v1 api where _member_ role not able to
download the image with custom property.
"""
image_id = 'test1'
def fake_get_v1_image_metadata(*args, **kwargs):
return {
'id': image_id,
'name': 'fake_image',
'status': 'active',
'created_at': '',
'min_disk': '10G',
'min_ram': '1024M',
'protected': False,
'locations': '',
'checksum': 'c1234',
'owner': '',
'disk_format': 'raw',
'container_format': 'bare',
'size': '123456789',
'virtual_size': '123456789',
'is_public': 'public',
'deleted': False,
'updated_at': '',
'x_test_key': 'test_1234'
}
enforcer = self._enforcer_from_rules({
"restricted":
"not ('test_1234':%(x_test_key)s and role:_member_)",
"download_image":
"role:admin or rule:restricted"
})
request = webob.Request.blank('/v1/images/%s' % image_id)
request.context = context.RequestContext(roles=['_member_'])
cache_filter = ProcessRequestTestCacheFilter()
cache_filter._get_v1_image_metadata = fake_get_v1_image_metadata
cache_filter.policy = enforcer
self.assertRaises(webob.exc.HTTPForbidden,
cache_filter.process_request, request)
def test_v2_process_request_response_headers(self):
def dummy_img_iterator():
for i in range(3):
yield i
image_id = 'test1'
request = webob.Request.blank('/v2/images/test1/file')
request.context = context.RequestContext()
request.environ['api.cache.image'] = ImageStub(image_id)
image_meta = {
'id': image_id,
'name': 'fake_image',
'status': 'active',
'created_at': '',
'min_disk': '10G',
'min_ram': '1024M',
'protected': False,
'locations': '',
'checksum': 'c1234',
'owner': '',
'disk_format': 'raw',
'container_format': 'bare',
'size': '123456789',
'virtual_size': '123456789',
'is_public': 'public',
'deleted': False,
'updated_at': '',
'properties': {},
}
cache_filter = ProcessRequestTestCacheFilter()
response = cache_filter._process_v2_request(request, image_id,
dummy_img_iterator,
image_meta)
self.assertEqual('application/octet-stream',
response.headers['Content-Type'])
self.assertEqual('c1234', response.headers['Content-MD5'])
self.assertEqual('123456789', response.headers['Content-Length'])
def process_request(self, req):
auth_token = req.headers.get('X-Auth-Token')
user = None
tenant = None
roles = []
if auth_token:
user, tenant, role = auth_token.split(':')
if tenant.lower() == 'none':
tenant = None
roles = [role]
req.headers['X-User-Id'] = user
req.headers['X-Tenant-Id'] = tenant
req.headers['X-Roles'] = role
req.headers['X-Identity-Status'] = 'Confirmed'
kwargs = {
'user': user,
'tenant': tenant,
'roles': roles,
'is_admin': self.is_admin,
'auth_token': auth_token,
}
req.context = context.RequestContext(**kwargs)
def test_process_v1_request_for_deleted_but_cached_image(self):
"""
Test for determining image is deleted from cache when it is not found
in Glance Registry.
"""
def fake_process_v1_request(request, image_id, image_iterator,
image_meta):
raise exception.ImageNotFound()
def fake_get_v1_image_metadata(request, image_id):
return {'status': 'active', 'properties': {}}
image_id = 'test1'
request = webob.Request.blank('/v1/images/%s' % image_id)
request.context = context.RequestContext()
cache_filter = ProcessRequestTestCacheFilter()
self.stubs.Set(cache_filter, '_get_v1_image_metadata',
fake_get_v1_image_metadata)
self.stubs.Set(cache_filter, '_process_v1_request',
fake_process_v1_request)
cache_filter.process_request(request)
self.assertIn(image_id, cache_filter.cache.deleted_images)
def fetch_image_into_cache(self, image_id):
ctx = context.RequestContext(is_admin=True, show_deleted=True)
try:
image_meta = registry.get_image_metadata(ctx, image_id)
if image_meta['status'] != 'active':
LOG.warn(_LW("Image '%s' is not active. Not caching.") %
image_id)
return False
except exception.NotFound:
LOG.warn(_LW("No metadata found for image '%s'") % image_id)
return False
location = image_meta['location']
image_data, image_size = glance_store.get_from_backend(location,
context=ctx)
LOG.debug("Caching image '%s'", image_id)
cache_tee_iter = self.cache.cache_tee_iter(image_id, image_data,
image_meta['checksum'])
# Image is tee'd into cache and checksum verified
# as we iterate
list(cache_tee_iter)
return True
def setUp(self):
"""Establish a clean test environment."""
super(TestRegistryV1ClientApi, self).setUp()
self.context = context.RequestContext()
reload_module(rapi)
def setUp(self):
super(TestCacheMiddlewareChecksumVerification, self).setUp()
self.context = context.RequestContext(is_admin=True)
self.request = webob.Request.blank('')
self.request.context = self.context
def setUp(self):
super(DBCommandsTestCase, self).setUp()
self.commands = manage.DbCommands()
self.context = context.RequestContext(
user=USER1, tenant=TENANT1)
def test_service_catalog(self):
ctx = context.RequestContext(service_catalog=['foo'])
self.assertEqual(['foo'], ctx.service_catalog)
def test_request_id(self):
contexts = [context.RequestContext().request_id for _ in range(5)]
# Check for uniqueness -- set() will normalize its argument
self.assertEqual(5, len(set(contexts)))
