def decrypt_metadata(self, image_metadata):
if self.metadata_encryption_key:
if image_metadata.get("location"):
location = crypt.urlsafe_decrypt(self.metadata_encryption_key, image_metadata["location"])
image_metadata["location"] = location
if image_metadata.get("location_data"):
ld = []
for loc in image_metadata["location_data"]:
url = crypt.urlsafe_decrypt(self.metadata_encryption_key, loc["url"])
ld.append({"id": loc["id"], "url": url, "metadata": loc["metadata"], "status": loc["status"]})
image_metadata["location_data"] = ld
return image_metadata
def _check_16_to_17(self, engine):
"""
Check that migrating swift location credentials to quoted form
and back works.
"""
migration_api.version_control(version=0)
migration_api.upgrade(16)
conn = engine.connect()
images_table = Table('images', MetaData(), autoload=True,
autoload_with=engine)
def get_locations():
conn = engine.connect()
locations = [x[0] for x in
conn.execute(
select(['location'], from_obj=[images_table]))]
conn.close()
return locations
unquoted = 'swift://*****:*****@example.com/container/obj-id'
encrypted_unquoted = crypt.urlsafe_encrypt(
self.metadata_encryption_key,
unquoted, 64)
quoted = 'swift://acct%3Ausr:[email protected]/container/obj-id'
# Insert image with an unquoted image location
now = datetime.datetime.now()
kwargs = dict(deleted=False,
created_at=now,
updated_at=now,
status='active',
is_public=True,
min_disk=0,
min_ram=0)
kwargs.update(location=encrypted_unquoted, id=1)
conn.execute(images_table.insert(), [kwargs])
conn.close()
migration_api.upgrade(17)
actual_location = crypt.urlsafe_decrypt(self.metadata_encryption_key,
get_locations()[0])
self.assertEqual(actual_location, quoted)
migration_api.downgrade(16)
actual_location = crypt.urlsafe_decrypt(self.metadata_encryption_key,
get_locations()[0])
self.assertEqual(actual_location, unquoted)
def decrypt_metadata(self, image_metadata):
if self.metadata_encryption_key is not None:
if image_metadata.get('location'):
location = crypt.urlsafe_decrypt(self.metadata_encryption_key,
image_metadata['location'])
image_metadata['location'] = location
if image_metadata.get('location_data'):
ld = []
for loc in image_metadata['location_data']:
url = crypt.urlsafe_decrypt(self.metadata_encryption_key,
loc['url'])
ld.append({'url': url, 'metadata': loc['metadata']})
image_metadata['location_data'] = ld
return image_metadata
def _format_image_from_db(self, db_image, db_tags):
visibility = 'public' if db_image['is_public'] else 'private'
properties = {}
for prop in db_image.pop('properties'):
# NOTE(markwash) db api requires us to filter deleted
if not prop['deleted']:
properties[prop['name']] = prop['value']
locations = db_image['locations']
if CONF.metadata_encryption_key:
key = CONF.metadata_encryption_key
ld = []
for l in locations:
url = crypt.urlsafe_decrypt(key, l['url'])
ld.append({'url': url, 'metadata': l['metadata']})
locations = ld
return glance.domain.Image(
image_id=db_image['id'],
name=db_image['name'],
status=db_image['status'],
created_at=db_image['created_at'],
updated_at=db_image['updated_at'],
visibility=visibility,
min_disk=db_image['min_disk'],
min_ram=db_image['min_ram'],
protected=db_image['protected'],
locations=location_strategy.get_ordered_locations(locations),
checksum=db_image['checksum'],
owner=db_image['owner'],
disk_format=db_image['disk_format'],
container_format=db_image['container_format'],
size=db_image['size'],
virtual_size=db_image['virtual_size'],
extra_properties=properties,
tags=db_tags
)
def _check_017(self, engine, data):
metadata_encryption_key = 'a' * 16
quoted = 'swift://acct%3Ausr:[email protected]/container/obj-id'
images = get_table(engine, 'images')
result = images.select().execute()
locations = map(lambda x: x['location'], result)
actual_location = []
for location in locations:
if location:
try:
temp_loc = crypt.urlsafe_decrypt(metadata_encryption_key,
location)
actual_location.append(temp_loc)
except TypeError:
actual_location.append(location)
except ValueError:
actual_location.append(location)
self.assertIn(quoted, actual_location)
loc_list = ['file://ab',
'file://abc',
'swift://acct3A%foobar:[email protected]/container/obj-id2']
for location in loc_list:
if not location in actual_location:
self.fail(_("location: %s data lost") % location)
def _format_image_from_db(self, db_image, db_tags):
properties = {}
for prop in db_image.pop('properties'):
# NOTE(markwash) db api requires us to filter deleted
if not prop['deleted']:
properties[prop['name']] = prop['value']
locations = [loc for loc in db_image['locations']
if loc['status'] == 'active']
if CONF.metadata_encryption_key:
key = CONF.metadata_encryption_key
for l in locations:
l['url'] = crypt.urlsafe_decrypt(key, l['url'])
return glance.domain.Image(
image_id=db_image['id'],
name=db_image['name'],
status=db_image['status'],
created_at=db_image['created_at'],
updated_at=db_image['updated_at'],
visibility=db_image['visibility'],
min_disk=db_image['min_disk'],
min_ram=db_image['min_ram'],
protected=db_image['protected'],
locations=location_strategy.get_ordered_locations(locations),
checksum=db_image['checksum'],
os_hash_algo=db_image['os_hash_algo'],
os_hash_value=db_image['os_hash_value'],
owner=db_image['owner'],
disk_format=db_image['disk_format'],
container_format=db_image['container_format'],
size=db_image['size'],
virtual_size=db_image['virtual_size'],
extra_properties=properties,
tags=db_tags,
os_hidden=db_image['os_hidden'],
)
def _format_image_from_db(self, db_image, db_tags):
visibility = "public" if db_image["is_public"] else "private"
properties = {}
for prop in db_image.pop("properties"):
# NOTE(markwash) db api requires us to filter deleted
if not prop["deleted"]:
properties[prop["name"]] = prop["value"]
locations = db_image["locations"]
if CONF.metadata_encryption_key:
key = CONF.metadata_encryption_key
locations = [crypt.urlsafe_decrypt(key, l) for l in locations]
return glance.domain.Image(
image_id=db_image["id"],
name=db_image["name"],
status=db_image["status"],
created_at=db_image["created_at"],
updated_at=db_image["updated_at"],
visibility=visibility,
min_disk=db_image["min_disk"],
min_ram=db_image["min_ram"],
protected=db_image["protected"],
locations=locations,
checksum=db_image["checksum"],
owner=db_image["owner"],
disk_format=db_image["disk_format"],
container_format=db_image["container_format"],
size=db_image["size"],
extra_properties=properties,
tags=db_tags,
)
def _check_017(self, engine, data):
metadata_encryption_key = 'a' * 16
quoted = 'swift://acct%3Ausr:[email protected]/container/obj-id'
images = get_table(engine, 'images')
result = images.select().execute()
locations = map(lambda x: x['location'], result)
actual_location = []
for location in locations:
if location:
try:
temp_loc = crypt.urlsafe_decrypt(metadata_encryption_key,
location)
actual_location.append(temp_loc)
except TypeError:
actual_location.append(location)
except ValueError:
actual_location.append(location)
self.assertIn(quoted, actual_location)
loc_list = ['file://ab',
'file://abc',
'swift://acct3A%foobar:[email protected]/container/obj-id2']
for location in loc_list:
if not location in actual_location:
self.fail(_("location: %s data lost") % location)
def _format_image_from_db(self, db_image, db_tags):
visibility = 'public' if db_image['is_public'] else 'private'
properties = {}
for prop in db_image.pop('properties'):
# NOTE(markwash) db api requires us to filter deleted
if not prop['deleted']:
properties[prop['name']] = prop['value']
locations = db_image['locations']
if CONF.metadata_encryption_key:
key = CONF.metadata_encryption_key
ld = []
for l in locations:
url = crypt.urlsafe_decrypt(key, l['url'])
ld.append({'url': url, 'metadata': l['metadata']})
locations = ld
return glance.domain.Image(
image_id=db_image['id'],
name=db_image['name'],
status=db_image['status'],
created_at=db_image['created_at'],
updated_at=db_image['updated_at'],
visibility=visibility,
min_disk=db_image['min_disk'],
min_ram=db_image['min_ram'],
protected=db_image['protected'],
locations=location_strategy.get_ordered_locations(locations),
checksum=db_image['checksum'],
owner=db_image['owner'],
disk_format=db_image['disk_format'],
container_format=db_image['container_format'],
size=db_image['size'],
virtual_size=db_image['virtual_size'],
extra_properties=properties,
tags=db_tags)
def test_encryption(self):
# Check that original plaintext and unencrypted ciphertext match
# Check keys of the three allowed lengths
key_list = ["1234567890abcdef",
"12345678901234567890abcd",
"1234567890abcdef1234567890ABCDEF"]
plaintext_list = ['']
blocksize = 64
for i in range(3 * blocksize):
text = os.urandom(i)
if six.PY3:
text = text.decode('latin1')
plaintext_list.append(text)
for key in key_list:
for plaintext in plaintext_list:
ciphertext = crypt.urlsafe_encrypt(key, plaintext, blocksize)
self.assertIsInstance(ciphertext, bytes)
if six.PY3:
self.assertNotEqual(ciphertext, plaintext.encode('utf-8'))
else:
self.assertNotEqual(ciphertext, plaintext)
text = crypt.urlsafe_decrypt(key, ciphertext)
self.assertIsInstance(text, str)
self.assertEqual(plaintext, text)
def _format_image_from_db(self, db_image, db_tags):
properties = {}
for prop in db_image.pop('properties'):
# NOTE(markwash) db api requires us to filter deleted
if not prop['deleted']:
properties[prop['name']] = prop['value']
locations = [
loc for loc in db_image['locations'] if loc['status'] == 'active'
]
if CONF.metadata_encryption_key:
key = CONF.metadata_encryption_key
for l in locations:
l['url'] = crypt.urlsafe_decrypt(key, l['url'])
#利用数据库中查询到的数据，填充Image对象
return glance.domain.Image(
image_id=db_image['id'],
name=db_image['name'],
status=db_image['status'],
created_at=db_image['created_at'],
updated_at=db_image['updated_at'],
visibility=db_image['visibility'],
min_disk=db_image['min_disk'],
min_ram=db_image['min_ram'],
protected=db_image['protected'],
locations=location_strategy.get_ordered_locations(locations),
checksum=db_image['checksum'],
owner=db_image['owner'],
disk_format=db_image['disk_format'],
container_format=db_image['container_format'],
size=db_image['size'],
virtual_size=db_image['virtual_size'],
extra_properties=properties,
tags=db_tags)
def test_scrubber_with_metadata_enc(self):
"""
test that files written to scrubber_data_dir use
metadata_encryption_key when available to encrypt the location
"""
config_path = os.environ.get("GLANCE_TEST_SWIFT_CONF")
if not config_path:
msg = "GLANCE_TEST_SWIFT_CONF environ not set."
self.skipTest(msg)
raw_config = read_config(config_path)
swift_config = parse_config(raw_config)
self.cleanup()
self.start_servers(delayed_delete=True, daemon=True, default_store="swift", **swift_config)
# add an image
headers = {
"x-image-meta-name": "test_image",
"x-image-meta-is_public": "true",
"x-image-meta-disk_format": "raw",
"x-image-meta-container_format": "ovf",
"content-type": "application/octet-stream",
}
path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
http = httplib2.Http()
response, content = http.request(path, "POST", body="XXX", headers=headers)
self.assertEqual(response.status, 201)
image = json.loads(content)["image"]
self.assertEqual("active", image["status"])
image_id = image["id"]
# delete the image
path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port, image_id)
http = httplib2.Http()
response, content = http.request(path, "DELETE")
self.assertEqual(response.status, 200)
response, content = http.request(path, "HEAD")
self.assertEqual(response.status, 200)
self.assertEqual("pending_delete", response["x-image-meta-status"])
# ensure the marker file has encrypted the image location by decrypting
# it and checking the image_id is intact
file_path = os.path.join(self.api_server.scrubber_datadir, str(image_id))
marker_uri = None
with open(file_path, "r") as f:
marker_uri = f.readline().strip()
self.assertTrue(marker_uri is not None)
decrypted_uri = crypt.urlsafe_decrypt(self.api_server.metadata_encryption_key, marker_uri)
loc = StoreLocation({})
loc.parse_uri(decrypted_uri)
self.assertIn(loc.scheme, ("swift+http", "swift+https"))
self.assertEqual(image["id"], loc.obj)
self.wait_for_scrub(path)
self.stop_servers()
def _delete_image_location_from_backend(self, image_id, loc_id, uri):
if CONF.metadata_encryption_key:
uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri)
try:
LOG.debug("Scrubbing image %s from a location.", image_id)
try:
self.store_api.delete_from_backend(uri, self.admin_context)
except store_exceptions.NotFound:
LOG.info(
_LI("Image location for image '%s' not found in "
"backend; Marking image location deleted in "
"db."), image_id)
if loc_id != '-':
db_api.get_api().image_location_delete(self.admin_context,
image_id, int(loc_id),
'deleted')
LOG.info(_LI("Image %s is scrubbed from a location."), image_id)
except Exception as e:
LOG.error(
_LE("Unable to scrub image %(id)s from a location. "
"Reason: %(exc)s ") % {
'id': image_id,
'exc': encodeutils.exception_to_unicode(e)
})
raise
def get_all_locations(self):
"""Returns a list of image id and location tuple from scrub queue.
:returns: a list of image id, location id and uri tuple from
scrub queue
"""
ret = []
for image in self._get_all_images():
deleted_at = image.get('deleted_at')
if not deleted_at:
continue
# NOTE: Strip off microseconds which may occur after the last '.,'
# Example: 2012-07-07T19:14:34.974216
deleted_at = timeutils.isotime(deleted_at)
date_str = deleted_at.rsplit('.', 1)[0].rsplit(',', 1)[0]
delete_time = calendar.timegm(
time.strptime(date_str, "%Y-%m-%dT%H:%M:%SZ"))
if delete_time + self.scrub_time > time.time():
continue
for loc in image['locations']:
if loc['status'] != 'pending_delete':
continue
if self.metadata_encryption_key:
uri = crypt.urlsafe_decrypt(self.metadata_encryption_key,
loc['url'])
else:
uri = loc['url']
ret.append((image['id'], loc['id'], uri))
return ret
def decrypt_metadata(self, image_metadata):
if (self.metadata_encryption_key is not None
and 'location' in image_metadata.keys()
and image_metadata['location'] is not None):
location = crypt.urlsafe_decrypt(self.metadata_encryption_key,
image_metadata['location'])
image_metadata['location'] = location
return image_metadata
def test_encrypt_locations_on_add(self):
image = self.image_factory.new_image(UUID1)
image.locations = self.foo_bar_location
self.image_repo.add(image)
db_data = self.db.image_get(self.context, UUID1)
self.assertNotEqual(db_data["locations"], ["foo", "bar"])
decrypted_locations = [crypt.urlsafe_decrypt(self.crypt_key, l["url"]) for l in db_data["locations"]]
self.assertEqual(decrypted_locations, [l["url"] for l in self.foo_bar_location])
def decrypt_metadata(self, image_metadata):
if (self.metadata_encryption_key is not None and
'location' in image_metadata and
image_metadata['location'] is not None):
location = crypt.urlsafe_decrypt(self.metadata_encryption_key,
image_metadata['location'])
image_metadata['location'] = location
return image_metadata
def test_encrypt_locations_on_add(self):
image = self.image_factory.new_image(UUID1)
image.locations = ['foo', 'bar']
self.image_repo.add(image)
db_data = self.db.image_get(self.context, UUID1)
self.assertNotEqual(db_data['locations'], ['foo', 'bar'])
decrypted_locations = [crypt.urlsafe_decrypt(self.crypt_key, l)
for l in db_data['locations']]
self.assertEqual(decrypted_locations, ['foo', 'bar'])
def decrypt_metadata(self, image_metadata):
if (
self.metadata_encryption_key is not None
and "location" in image_metadata
and image_metadata["location"] is not None
):
location = crypt.urlsafe_decrypt(self.metadata_encryption_key, image_metadata["location"])
image_metadata["location"] = location
return image_metadata
def test_encrypt_locations_on_add(self):
image = self.image_factory.new_image(UUID1)
image.locations = ['foo', 'bar']
self.image_repo.add(image)
db_data = self.db.image_get(self.context, UUID1)
self.assertNotEqual(db_data['locations'], ['foo', 'bar'])
decrypted_locations = [crypt.urlsafe_decrypt(self.crypt_key, l)
for l in db_data['locations']]
self.assertEqual(decrypted_locations, ['foo', 'bar'])
def test_encrypt_locations_on_save(self):
image = self.image_factory.new_image(UUID1)
self.image_repo.add(image)
image.locations = self.foo_bar_location
self.image_repo.save(image)
db_data = self.db.image_get(self.context, UUID1)
self.assertNotEqual(db_data['locations'], ['foo', 'bar'])
decrypted_locations = [crypt.urlsafe_decrypt(self.crypt_key, l['url'])
for l in db_data['locations']]
self.assertEqual([l['url'] for l in self.foo_bar_location],
decrypted_locations)
def test_encrypt_locations_on_save(self):
image = self.image_factory.new_image(UUID1)
self.image_repo.add(image)
image.locations = self.foo_bar_location
self.image_repo.save(image)
db_data = self.db.image_get(self.context, UUID1)
self.assertNotEqual(db_data['locations'], ['foo', 'bar'])
decrypted_locations = [crypt.urlsafe_decrypt(self.crypt_key, l['url'])
for l in db_data['locations']]
self.assertEqual([l['url'] for l in self.foo_bar_location],
decrypted_locations)
def _delete_image_location_from_backend(self, image_id, loc_id, uri):
if CONF.metadata_encryption_key:
uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri)
try:
LOG.debug("Deleting URI from image %s." % image_id)
self.store_api.delete_from_backend(self.admin_context, uri)
if loc_id != "-":
db_api.get_api().image_location_delete(self.admin_context, image_id, int(loc_id), "deleted")
LOG.info(_LI("Image %s has been deleted.") % image_id)
except Exception:
LOG.warn(_LW("Unable to delete URI from image %s.") % image_id)
def test_scrubber_with_metadata_enc(self):
"""
test that files written to scrubber_data_dir use
metadata_encryption_key when available to encrypt the location
"""
# FIXME(flaper87): It looks like an older commit
# may have broken this test. The file_queue `add_location`
# is not being called.
self.skipTest("Test broken. See bug #1366682")
self.cleanup()
self.start_servers(delayed_delete=True,
daemon=True,
default_store='file')
# add an image
path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
response, content = self._send_http_request(path, 'POST', body='XXX')
self.assertEqual(201, response.status)
image = jsonutils.loads(content)['image']
self.assertEqual('active', image['status'])
# delete the image
path = "http://%s:%d/v1/images/%s" % ("127.0.0.1",
self.api_port,
image['id'])
response, content = self._send_http_request(path, 'DELETE')
self.assertEqual(200, response.status)
response, content = self._send_http_request(path, 'HEAD')
self.assertEqual(200, response.status)
self.assertEqual('pending_delete', response['x-image-meta-status'])
# ensure the marker file has encrypted the image location by decrypting
# it and checking the image_id is intact
file_path = os.path.join(self.api_server.scrubber_datadir, image['id'])
marker_uri = None
with open(file_path, 'r') as f:
marker_uri = f.readline().strip()
self.assertTrue(marker_uri is not None)
decrypted_uri = crypt.urlsafe_decrypt(
self.api_server.metadata_encryption_key, marker_uri)
loc = glance_store.location.StoreLocation({})
loc.parse_uri(decrypted_uri)
self.assertEqual("file", loc.scheme)
self.assertEqual(image['id'], loc.obj)
self.wait_for_scrub(path)
self.stop_servers()
def _delete_image_location_from_backend(self, image_id, loc_id, uri):
if CONF.metadata_encryption_key:
uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri)
try:
LOG.debug("Deleting URI from image %s." % image_id)
self.store_api.delete_from_backend(uri, self.admin_context)
if loc_id != '-':
db_api.get_api().image_location_delete(self.admin_context,
image_id, int(loc_id),
'deleted')
LOG.info(_LI("Image %s has been deleted.") % image_id)
except Exception:
LOG.warn(_LW("Unable to delete URI from image %s.") % image_id)
def _format_image_from_db(self, db_image, db_tags):
properties = {}
for prop in db_image.pop('properties'):
# NOTE(markwash) db api requires us to filter deleted
if not prop['deleted']:
properties[prop['name']] = prop['value']
locations = [
loc for loc in db_image['locations'] if loc['status'] == 'active'
]
if CONF.metadata_encryption_key:
key = CONF.metadata_encryption_key
for l in locations:
l['url'] = crypt.urlsafe_decrypt(key, l['url'])
# NOTE(danms): If the image is shared and we are not the
# owner, we must have found it because we are a member. Set
# our tenant on the image as 'member' for policy checks in the
# upper layers. For any other image stage, we found the image
# some other way, so leave member=None.
if (db_image['visibility'] == 'shared'
and self.context.owner != db_image['owner']):
member = self.context.owner
else:
member = None
return glance.domain.Image(
image_id=db_image['id'],
name=db_image['name'],
status=db_image['status'],
created_at=db_image['created_at'],
updated_at=db_image['updated_at'],
visibility=db_image['visibility'],
min_disk=db_image['min_disk'],
min_ram=db_image['min_ram'],
protected=db_image['protected'],
locations=location_strategy.get_ordered_locations(locations),
checksum=db_image['checksum'],
os_hash_algo=db_image['os_hash_algo'],
os_hash_value=db_image['os_hash_value'],
owner=db_image['owner'],
disk_format=db_image['disk_format'],
container_format=db_image['container_format'],
size=db_image['size'],
virtual_size=db_image['virtual_size'],
extra_properties=properties,
tags=db_tags,
os_hidden=db_image['os_hidden'],
member=member,
)
def test_encryption(self):
# Check that original plaintext and unencrypted ciphertext match
# Check keys of the three allowed lengths
key_list = ["1234567890abcdef", "12345678901234567890abcd", "1234567890abcdef1234567890ABCDEF"]
plaintext_list = [""]
blocksize = 64
for i in range(3 * blocksize):
plaintext_list.append(os.urandom(i))
for key in key_list:
for plaintext in plaintext_list:
ciphertext = crypt.urlsafe_encrypt(key, plaintext, blocksize)
self.assertTrue(ciphertext != plaintext)
text = crypt.urlsafe_decrypt(key, ciphertext)
self.assertTrue(plaintext == text)
def test_encryption(self):
# Check that original plaintext and unencrypted ciphertext match
# Check keys of the three allowed lengths
key_list = ["1234567890abcdef",
"12345678901234567890abcd",
"1234567890abcdef1234567890ABCDEF"]
plaintext_list = ['']
blocksize = 64
for i in range(3 * blocksize):
plaintext_list.append(os.urandom(i))
for key in key_list:
for plaintext in plaintext_list:
ciphertext = crypt.urlsafe_encrypt(key, plaintext, blocksize)
self.assertTrue(ciphertext != plaintext)
text = crypt.urlsafe_decrypt(key, ciphertext)
self.assertTrue(plaintext == text)
def _delete_image_from_backend(self, image_id, uri):
if CONF.metadata_encryption_key is not None:
uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri)
try:
LOG.debug(_("Deleting URI from image %(image_id)s.") % {"image_id": image_id})
# Here we create a request context with credentials to support
# delayed delete when using multi-tenant backend storage
admin_tenant = CONF.admin_tenant_name
auth_token = self.registry.auth_tok
admin_context = context.RequestContext(user=CONF.admin_user, tenant=admin_tenant, auth_tok=auth_token)
self.store_api.delete_from_backend(admin_context, uri)
except Exception:
msg = _("Failed to delete URI from image %(image_id)s")
LOG.error(msg % {"image_id": image_id})
def _delete(self, id, uri, now):
file_path = os.path.join(self.datadir, str(id))
if CONF.metadata_encryption_key is not None:
uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri)
try:
LOG.debug(_("Deleting %(uri)s") % {'uri': uri})
# Here we create a request context with credentials to support
# delayed delete when using multi-tenant backend storage
ctx = context.RequestContext(auth_tok=self.registry.auth_tok,
user=self.admin_user,
tenant=self.admin_tenant)
store.delete_from_backend(ctx, uri)
except store.UnsupportedBackend:
msg = _("Failed to delete image from store (%(uri)s).")
LOG.error(msg % {'uri': uri})
write_queue_file(file_path, uri, now)
self.registry.update_image(id, {'status': 'deleted'})
utils.safe_remove(file_path)
def _delete_image_from_backend(self, image_id, uri):
if CONF.metadata_encryption_key is not None:
uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri)
try:
LOG.debug("Deleting URI from image %(image_id)s." %
{'image_id': image_id})
# Here we create a request context with credentials to support
# delayed delete when using multi-tenant backend storage
admin_tenant = CONF.admin_tenant_name
auth_token = self.registry.auth_tok
admin_context = context.RequestContext(user=CONF.admin_user,
tenant=admin_tenant,
auth_tok=auth_token)
self.store_api.delete_from_backend(admin_context, uri)
except Exception:
msg = _("Failed to delete URI from image %(image_id)s")
LOG.error(msg % {'image_id': image_id})
def get_all_locations(self):
"""Returns a list of image id and location tuple from scrub queue.
:returns: a list of image id, location id and uri tuple from
scrub queue
"""
ret = []
for image in self._get_all_images():
deleted_at = image.get('deleted_at')
if not deleted_at:
continue
# NOTE: Strip off microseconds which may occur after the last '.,'
# Example: 2012-07-07T19:14:34.974216
deleted_at = timeutils.isotime(deleted_at)
date_str = deleted_at.rsplit('.', 1)[0].rsplit(',', 1)[0]
delete_time = calendar.timegm(time.strptime(date_str,
"%Y-%m-%dT%H:%M:%SZ"))
if delete_time + self.scrub_time > time.time():
continue
for loc in image['locations']:
if loc['status'] != 'pending_delete':
continue
if self.metadata_encryption_key:
uri = crypt.urlsafe_decrypt(self.metadata_encryption_key,
loc['url'])
else:
uri = loc['url']
# if multi-store is enabled then we need to pass backend
# to delete the image.
backend = loc['metadata'].get('backend')
if CONF.enabled_backends:
ret.append((image['id'], loc['id'], uri, backend))
else:
ret.append((image['id'], loc['id'], uri))
return ret
def _delete(self, id, uri, now):
file_path = os.path.join(self.datadir, str(id))
if CONF.metadata_encryption_key is not None:
uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri)
try:
LOG.debug(_("Deleting %(id)s") % {'id': id})
# Here we create a request context with credentials to support
# delayed delete when using multi-tenant backend storage
ctx = context.RequestContext(auth_tok=self.registry.auth_tok,
user=self.admin_user,
tenant=self.admin_tenant)
store.delete_from_backend(ctx, uri)
except store.UnsupportedBackend:
msg = _("Failed to delete image from store (%(id)s).")
LOG.error(msg % {'id': id})
except exception.NotFound:
msg = _("Image not found in store (%(id)s).")
LOG.error(msg % {'id': id})
self.registry.update_image(id, {'status': 'deleted'})
utils.safe_remove(file_path)
def _delete_image_location_from_backend(self, image_id, loc_id, uri):
if CONF.metadata_encryption_key:
uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri)
try:
LOG.debug("Scrubbing image %s from a location." % image_id)
try:
self.store_api.delete_from_backend(uri, self.admin_context)
except store_exceptions.NotFound:
LOG.info(_LI("Image location for image '%s' not found in "
"backend; Marking image location deleted in "
"db."), image_id)
if loc_id != '-':
db_api.get_api().image_location_delete(self.admin_context,
image_id,
int(loc_id),
'deleted')
LOG.info(_LI("Image %s is scrubbed from a location."), image_id)
except Exception as e:
LOG.error(_LE("Unable to scrub image %(id)s from a location. "
"Reason: %(exc)s ") %
{'id': image_id,
'exc': encodeutils.exception_to_unicode(e)})
raise
def test_remote_image(self):
"""Verify an image added using a 'Location' header can be retrieved"""
self.cleanup()
self.start_servers(**self.__dict__.copy())
# 1. POST /images with public image named Image1
image_data = "*" * FIVE_KB
headers = minimal_headers('Image1')
path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
http = httplib2.Http()
response, content = http.request(path,
'POST',
headers=headers,
body=image_data)
self.assertEqual(response.status, 201)
data = json.loads(content)
self.assertEqual(data['image']['checksum'],
hashlib.md5(image_data).hexdigest())
self.assertEqual(data['image']['size'], FIVE_KB)
image_id1 = data['image']['id']
# 2. GET first image
# Verify all information on image we just added is correct
path = "http://%s:%d/v1/images/%s"
args = ("127.0.0.1", self.api_port, image_id1)
http = httplib2.Http()
response, content = http.request(path % args, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(response['content-length'], str(FIVE_KB))
self.assertEqual(content, "*" * FIVE_KB)
# 3. GET first image from registry in order to find S3 location
path = "http://%s:%d/images/%s"
args = ("127.0.0.1", self.registry_port, image_id1)
http = httplib2.Http()
response, content = http.request(path % args, 'GET')
if hasattr(self, 'metadata_encryption_key'):
key = self.metadata_encryption_key
else:
key = self.api_server.metadata_encryption_key
loc = json.loads(content)['image']['location']
s3_store_location = crypt.urlsafe_decrypt(key, loc)
# 4. POST /images using location generated by Image1
image_id2 = utils.generate_uuid()
image_data = "*" * FIVE_KB
headers = minimal_headers('Image2')
headers['X-Image-Meta-Id'] = image_id2
headers['X-Image-Meta-Location'] = s3_store_location
path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
# ensure data is refreshed, previously the size assertion
# applied to the metadata returned from the previous GET
data = json.loads(content)
self.assertEqual(data['image']['size'], FIVE_KB)
# checksum is not set for a remote image, as the image data
# is not yet retrieved
self.assertEqual(data['image']['checksum'], None)
# 5. GET second image and make sure it can stream the image
path = "http://%s:%d/v1/images/%s"
args = ("127.0.0.1", self.api_port, image_id2)
http = httplib2.Http()
response, content = http.request(path % args, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(response['content-length'], str(FIVE_KB))
self.assertEqual(content, "*" * FIVE_KB)
# 6. DELETE first and second images
path = "http://%s:%d/v1/images/%s"
args = ("127.0.0.1", self.api_port, image_id1)
http = httplib2.Http()
http.request(path % args, 'DELETE')
path = "http://%s:%d/v1/images/%s"
args = ("127.0.0.1", self.api_port, image_id2)
http = httplib2.Http()
http.request(path % args, 'DELETE')
self.stop_servers()
def test_remote_image(self):
"""
Ensure we can retrieve an image that was not stored by glance itself
"""
self.cleanup()
self.start_servers(**self.__dict__.copy())
api_port = self.api_port
registry_port = self.registry_port
# POST /images with public image named Image1
image_data = "*" * FIVE_KB
headers = {
'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1',
'X-Image-Meta-Is-Public': 'True'
}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path,
'POST',
headers=headers,
body=image_data)
self.assertEqual(response.status, 201, content)
data = json.loads(content)
self.assertEqual(data['image']['checksum'],
hashlib.md5(image_data).hexdigest())
self.assertEqual(data['image']['size'], FIVE_KB)
self.assertEqual(data['image']['name'], "Image1")
self.assertEqual(data['image']['is_public'], True)
image_id = data['image']['id']
# GET image and make sure data was uploaded
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(response['content-length'], str(FIVE_KB))
self.assertEqual(content, "*" * FIVE_KB)
self.assertEqual(
hashlib.md5(content).hexdigest(),
hashlib.md5("*" * FIVE_KB).hexdigest())
# Find the location that was just added and use it as
# the remote image location for the next image
path = "http://%s:%d/images/%s" % ("0.0.0.0", self.registry_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertTrue('location' in data['image'].keys())
loc = data['image']['location']
if hasattr(self, 'metadata_encryption_key'):
key = self.metadata_encryption_key
else:
key = self.api_server.metadata_encryption_key
swift_location = crypt.urlsafe_decrypt(key, loc)
# POST /images with public image named Image1 without uploading data
image_data = "*" * FIVE_KB
headers = {
'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1',
'X-Image-Meta-Is-Public': 'True',
'X-Image-Meta-Location': swift_location
}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201, content)
data = json.loads(content)
self.assertEqual(data['image']['checksum'], None)
self.assertEqual(data['image']['size'], 0)
self.assertEqual(data['image']['name'], "Image1")
self.assertEqual(data['image']['is_public'], True)
image_id2 = data['image']['id']
# GET /images/2 ensuring the data already in swift is accessible
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
image_id2)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(response['content-length'], str(FIVE_KB))
self.assertEqual(content, "*" * FIVE_KB)
self.assertEqual(
hashlib.md5(content).hexdigest(),
hashlib.md5("*" * FIVE_KB).hexdigest())
# DELETE boty images
# Verify image and all chunks are gone...
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'DELETE')
self.assertEqual(response.status, 200)
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
image_id2)
http = httplib2.Http()
response, content = http.request(path, 'DELETE')
self.assertEqual(response.status, 200)
self.stop_servers()
def test_large_objects(self):
"""
We test the large object manifest code path in the Swift driver.
In the case where an image file is bigger than the config variable
swift_store_large_object_size, then we chunk the image into
Swift, and add a manifest put_object at the end.
We test that the delete of the large object cleans up all the
chunks in Swift, in addition to the manifest file (LP Bug# 833285)
"""
self.cleanup()
self.swift_store_large_object_size = 2 # In MB
self.swift_store_large_object_chunk_size = 1 # In MB
self.start_servers(**self.__dict__.copy())
api_port = self.api_port
registry_port = self.registry_port
# GET /images
# Verify no public images
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(content, '{"images": []}')
# POST /images with public image named Image1
# attribute and no custom properties. Verify a 200 OK is returned
image_data = "*" * FIVE_MB
headers = {
'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1',
'X-Image-Meta-Is-Public': 'True'
}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path,
'POST',
headers=headers,
body=image_data)
self.assertEqual(response.status, 201, content)
data = json.loads(content)
self.assertEqual(data['image']['checksum'],
hashlib.md5(image_data).hexdigest())
self.assertEqual(data['image']['size'], FIVE_MB)
self.assertEqual(data['image']['name'], "Image1")
self.assertEqual(data['image']['is_public'], True)
image_id = data['image']['id']
# HEAD image
# Verify image found now
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'HEAD')
self.assertEqual(response.status, 200)
self.assertEqual(response['x-image-meta-name'], "Image1")
# GET image
# Verify all information on image we just added is correct
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
expected_image_headers = {
'x-image-meta-id': image_id,
'x-image-meta-name': 'Image1',
'x-image-meta-is_public': 'True',
'x-image-meta-status': 'active',
'x-image-meta-disk_format': '',
'x-image-meta-container_format': '',
'x-image-meta-size': str(FIVE_MB)
}
expected_std_headers = {
'content-length': str(FIVE_MB),
'content-type': 'application/octet-stream'
}
for expected_key, expected_value in expected_image_headers.items():
self.assertEqual(
response[expected_key], expected_value,
"For key '%s' expected header value '%s'. Got '%s'" %
(expected_key, expected_value, response[expected_key]))
for expected_key, expected_value in expected_std_headers.items():
self.assertEqual(
response[expected_key], expected_value,
"For key '%s' expected header value '%s'. Got '%s'" %
(expected_key, expected_value, response[expected_key]))
self.assertEqual(content, "*" * FIVE_MB)
self.assertEqual(
hashlib.md5(content).hexdigest(),
hashlib.md5("*" * FIVE_MB).hexdigest())
# We test that the delete of the large object cleans up all the
# chunks in Swift, in addition to the manifest file (LP Bug# 833285)
# Grab the actual Swift location and query the object manifest for
# the chunks/segments. We will check that the segments don't exist
# after we delete the object through Glance...
path = "http://%s:%d/images/%s" % ("0.0.0.0", self.registry_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
image_loc = data['image']['location']
if hasattr(self, 'metadata_encryption_key'):
key = self.metadata_encryption_key
else:
key = self.api_server.metadata_encryption_key
image_loc = crypt.urlsafe_decrypt(key, image_loc)
image_loc = get_location_from_uri(image_loc)
swift_loc = image_loc.store_location
from swift.common import client as swift_client
swift_conn = swift_client.Connection(authurl=swift_loc.swift_auth_url,
user=swift_loc.user,
key=swift_loc.key)
# Verify the object manifest exists
headers = swift_conn.head_object(swift_loc.container, swift_loc.obj)
manifest = headers.get('x-object-manifest')
self.assertTrue(manifest is not None, "Manifest could not be found!")
# Grab the segment identifiers
obj_container, obj_prefix = manifest.split('/', 1)
segments = [
segment['name']
for segment in swift_conn.get_container(obj_container,
prefix=obj_prefix)[1]
]
# Verify the segments exist
for segment in segments:
headers = swift_conn.head_object(obj_container, segment)
self.assertTrue(headers.get('content-length') is not None, headers)
# DELETE image
# Verify image and all chunks are gone...
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'DELETE')
self.assertEqual(response.status, 200)
# Verify the segments no longer exist
for segment in segments:
self.assertRaises(swift_client.ClientException,
swift_conn.head_object, obj_container, segment)
self.stop_servers()
def test_remote_image(self):
"""Verify an image added using a 'Location' header can be retrieved"""
self.cleanup()
self.start_servers(**self.__dict__.copy())
# 1. POST /images with public image named Image1
image_data = "*" * FIVE_KB
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Name': 'Image1',
'X-Image-Meta-Is-Public': 'True'}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers,
body=image_data)
self.assertEqual(response.status, 201)
data = json.loads(content)
self.assertEqual(data['image']['checksum'],
hashlib.md5(image_data).hexdigest())
self.assertEqual(data['image']['size'], FIVE_KB)
image_id1 = data['image']['id']
# 2. GET first image
# Verify all information on image we just added is correct
path = "http://%s:%d/v1/images/%s"
args = ("0.0.0.0", self.api_port, image_id1)
http = httplib2.Http()
response, content = http.request(path % args, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(response['content-length'], str(FIVE_KB))
self.assertEqual(content, "*" * FIVE_KB)
# 3. GET first image from registry in order to find S3 location
path = "http://%s:%d/images/%s"
args = ("0.0.0.0", self.registry_port, image_id1)
http = httplib2.Http()
response, content = http.request(path % args, 'GET')
if hasattr(self, 'metadata_encryption_key'):
key = self.metadata_encryption_key
else:
key = self.api_server.metadata_encryption_key
loc = json.loads(content)['image']['location']
s3_store_location = crypt.urlsafe_decrypt(key, loc)
# 4. POST /images using location generated by Image1
image_id2 = utils.generate_uuid()
image_data = "*" * FIVE_KB
headers = {'Content-Type': 'application/octet-stream',
'X-Image-Meta-Id': image_id2,
'X-Image-Meta-Name': 'Image2',
'X-Image-Meta-Is-Public': 'True',
'X-Image-Meta-Location': s3_store_location}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
self.assertEqual(data['image']['size'], FIVE_KB)
self.assertEqual(data['image']['checksum'],
hashlib.md5(image_data).hexdigest())
# 5. GET second image and make sure it can stream the image
path = "http://%s:%d/v1/images/%s"
args = ("0.0.0.0", self.api_port, image_id2)
http = httplib2.Http()
response, content = http.request(path % args, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(response['content-length'], str(FIVE_KB))
self.assertEqual(content, "*" * FIVE_KB)
# 6. DELETE first and second images
path = "http://%s:%d/v1/images/%s"
args = ("0.0.0.0", self.api_port, image_id1)
http = httplib2.Http()
http.request(path % args, 'DELETE')
path = "http://%s:%d/v1/images/%s"
args = ("0.0.0.0", self.api_port, image_id2)
http = httplib2.Http()
http.request(path % args, 'DELETE')
self.stop_servers()
def test_large_objects(self):
"""
We test the large object manifest code path in the Swift driver.
In the case where an image file is bigger than the config variable
swift_store_large_object_size, then we chunk the image into
Swift, and add a manifest put_object at the end.
We test that the delete of the large object cleans up all the
chunks in Swift, in addition to the manifest file (LP Bug# 833285)
"""
self.cleanup()
self.swift_store_large_object_size = 2 # In MB
self.swift_store_large_object_chunk_size = 1 # In MB
self.start_servers(**self.__dict__.copy())
api_port = self.api_port
registry_port = self.registry_port
# GET /images
# Verify no public images
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(content, '{"images": []}')
# POST /images with public image named Image1
# attribute and no custom properties. Verify a 200 OK is returned
image_data = "*" * FIVE_MB
headers = minimal_headers('Image1')
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers,
body=image_data)
self.assertEqual(response.status, 201, content)
data = json.loads(content)
self.assertEqual(data['image']['checksum'],
hashlib.md5(image_data).hexdigest())
self.assertEqual(data['image']['size'], FIVE_MB)
self.assertEqual(data['image']['name'], "Image1")
self.assertEqual(data['image']['is_public'], True)
image_id = data['image']['id']
# HEAD image
# Verify image found now
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'HEAD')
self.assertEqual(response.status, 200)
self.assertEqual(response['x-image-meta-name'], "Image1")
# GET image
# Verify all information on image we just added is correct
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
expected_image_headers = {
'x-image-meta-id': image_id,
'x-image-meta-name': 'Image1',
'x-image-meta-is_public': 'True',
'x-image-meta-status': 'active',
'x-image-meta-disk_format': 'raw',
'x-image-meta-container_format': 'ovf',
'x-image-meta-size': str(FIVE_MB)
}
expected_std_headers = {
'content-length': str(FIVE_MB),
'content-type': 'application/octet-stream'}
for expected_key, expected_value in expected_image_headers.items():
self.assertEqual(response[expected_key], expected_value,
"For key '%s' expected header value '%s'. Got '%s'"
% (expected_key, expected_value,
response[expected_key]))
for expected_key, expected_value in expected_std_headers.items():
self.assertEqual(response[expected_key], expected_value,
"For key '%s' expected header value '%s'. Got '%s'"
% (expected_key,
expected_value,
response[expected_key]))
self.assertEqual(content, "*" * FIVE_MB)
self.assertEqual(hashlib.md5(content).hexdigest(),
hashlib.md5("*" * FIVE_MB).hexdigest())
# We test that the delete of the large object cleans up all the
# chunks in Swift, in addition to the manifest file (LP Bug# 833285)
# Grab the actual Swift location and query the object manifest for
# the chunks/segments. We will check that the segments don't exist
# after we delete the object through Glance...
path = "http://%s:%d/images/%s" % ("0.0.0.0", self.registry_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
image_loc = data['image']['location']
if hasattr(self, 'metadata_encryption_key'):
key = self.metadata_encryption_key
else:
key = self.api_server.metadata_encryption_key
image_loc = crypt.urlsafe_decrypt(key, image_loc)
image_loc = get_location_from_uri(image_loc)
swift_loc = image_loc.store_location
from swift.common import client as swift_client
swift_conn = swift_client.Connection(
authurl=swift_loc.swift_auth_url,
user=swift_loc.user, key=swift_loc.key)
# Verify the object manifest exists
headers = swift_conn.head_object(swift_loc.container, swift_loc.obj)
manifest = headers.get('x-object-manifest')
self.assertTrue(manifest is not None, "Manifest could not be found!")
# Grab the segment identifiers
obj_container, obj_prefix = manifest.split('/', 1)
segments = [segment['name'] for segment in
swift_conn.get_container(obj_container,
prefix=obj_prefix)[1]]
# Verify the segments exist
for segment in segments:
headers = swift_conn.head_object(obj_container, segment)
self.assertTrue(headers.get('content-length') is not None,
headers)
# DELETE image
# Verify image and all chunks are gone...
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'DELETE')
self.assertEqual(response.status, 200)
# Verify the segments no longer exist
for segment in segments:
self.assertRaises(swift_client.ClientException,
swift_conn.head_object,
obj_container, segment)
self.stop_servers()
def test_remote_image(self):
"""
Ensure we can retrieve an image that was not stored by glance itself
"""
self.cleanup()
self.start_servers(**self.__dict__.copy())
api_port = self.api_port
registry_port = self.registry_port
# POST /images with public image named Image1
image_data = "*" * FIVE_KB
headers = minimal_headers('Image1')
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers,
body=image_data)
self.assertEqual(response.status, 201, content)
data = json.loads(content)
self.assertEqual(data['image']['checksum'],
hashlib.md5(image_data).hexdigest())
self.assertEqual(data['image']['size'], FIVE_KB)
self.assertEqual(data['image']['name'], "Image1")
self.assertEqual(data['image']['is_public'], True)
image_id = data['image']['id']
# GET image and make sure data was uploaded
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(response['content-length'], str(FIVE_KB))
self.assertEqual(content, "*" * FIVE_KB)
self.assertEqual(hashlib.md5(content).hexdigest(),
hashlib.md5("*" * FIVE_KB).hexdigest())
# Find the location that was just added and use it as
# the remote image location for the next image
path = "http://%s:%d/images/%s" % ("0.0.0.0", self.registry_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertTrue('location' in data['image'].keys())
loc = data['image']['location']
if hasattr(self, 'metadata_encryption_key'):
key = self.metadata_encryption_key
else:
key = self.api_server.metadata_encryption_key
swift_location = crypt.urlsafe_decrypt(key, loc)
# POST /images with public image named Image1 without uploading data
image_data = "*" * FIVE_KB
headers = minimal_headers('Image1')
headers['X-Image-Meta-Location'] = swift_location
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201, content)
data = json.loads(content)
self.assertEqual(data['image']['checksum'], None)
self.assertEqual(data['image']['size'], FIVE_KB)
self.assertEqual(data['image']['name'], "Image1")
self.assertEqual(data['image']['is_public'], True)
image_id2 = data['image']['id']
# GET /images/2 ensuring the data already in swift is accessible
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
image_id2)
http = httplib2.Http()
response, content = http.request(path, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(response['content-length'], str(FIVE_KB))
self.assertEqual(content, "*" * FIVE_KB)
self.assertEqual(hashlib.md5(content).hexdigest(),
hashlib.md5("*" * FIVE_KB).hexdigest())
# DELETE boty images
# Verify image and all chunks are gone...
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'DELETE')
self.assertEqual(response.status, 200)
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port,
image_id2)
http = httplib2.Http()
response, content = http.request(path, 'DELETE')
self.assertEqual(response.status, 200)
self.stop_servers()
def test_remote_image(self):
"""
Ensure we can retrieve an image that was not stored by glance itself
"""
self.cleanup()
self.start_servers(**self.__dict__.copy())
api_port = self.api_port
registry_port = self.registry_port
# POST /images with public image named Image1
image_data = "*" * FIVE_KB
headers = {
"Content-Type": "application/octet-stream",
"X-Image-Meta-Name": "Image1",
"X-Image-Meta-Is-Public": "True",
}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, "POST", headers=headers, body=image_data)
self.assertEqual(response.status, 201, content)
data = json.loads(content)
self.assertEqual(data["image"]["checksum"], hashlib.md5(image_data).hexdigest())
self.assertEqual(data["image"]["size"], FIVE_KB)
self.assertEqual(data["image"]["name"], "Image1")
self.assertEqual(data["image"]["is_public"], True)
image_id = data["image"]["id"]
# GET image and make sure data was uploaded
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id)
http = httplib2.Http()
response, content = http.request(path, "GET")
self.assertEqual(response.status, 200)
self.assertEqual(response["content-length"], str(FIVE_KB))
self.assertEqual(content, "*" * FIVE_KB)
self.assertEqual(hashlib.md5(content).hexdigest(), hashlib.md5("*" * FIVE_KB).hexdigest())
# Find the location that was just added and use it as
# the remote image location for the next image
path = "http://%s:%d/images/%s" % ("0.0.0.0", self.registry_port, image_id)
http = httplib2.Http()
response, content = http.request(path, "GET")
self.assertEqual(response.status, 200)
data = json.loads(content)
self.assertTrue("location" in data["image"].keys())
loc = data["image"]["location"]
if hasattr(self, "metadata_encryption_key"):
key = self.metadata_encryption_key
else:
key = self.api_server.metadata_encryption_key
swift_location = crypt.urlsafe_decrypt(key, loc)
# POST /images with public image named Image1 without uploading data
image_data = "*" * FIVE_KB
headers = {
"Content-Type": "application/octet-stream",
"X-Image-Meta-Name": "Image1",
"X-Image-Meta-Is-Public": "True",
"X-Image-Meta-Location": swift_location,
}
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, "POST", headers=headers)
self.assertEqual(response.status, 201, content)
data = json.loads(content)
self.assertEqual(data["image"]["checksum"], None)
self.assertEqual(data["image"]["size"], FIVE_KB)
self.assertEqual(data["image"]["name"], "Image1")
self.assertEqual(data["image"]["is_public"], True)
image_id2 = data["image"]["id"]
# GET /images/2 ensuring the data already in swift is accessible
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id2)
http = httplib2.Http()
response, content = http.request(path, "GET")
self.assertEqual(response.status, 200)
self.assertEqual(response["content-length"], str(FIVE_KB))
self.assertEqual(content, "*" * FIVE_KB)
self.assertEqual(hashlib.md5(content).hexdigest(), hashlib.md5("*" * FIVE_KB).hexdigest())
# DELETE boty images
# Verify image and all chunks are gone...
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id)
http = httplib2.Http()
response, content = http.request(path, "DELETE")
self.assertEqual(response.status, 200)
path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id2)
http = httplib2.Http()
response, content = http.request(path, "DELETE")
self.assertEqual(response.status, 200)
self.stop_servers()
def test_scrubber_with_metadata_enc(self):
"""
test that files written to scrubber_data_dir use
metadata_encryption_key when available to encrypt the location
"""
config_path = os.environ.get('GLANCE_TEST_SWIFT_CONF')
if not config_path:
msg = "GLANCE_TEST_SWIFT_CONF environ not set."
self.skipTest(msg)
raw_config = read_config(config_path)
swift_config = parse_config(raw_config)
self.cleanup()
self.start_servers(delayed_delete=True, daemon=True,
default_store='swift', **swift_config)
# add an image
headers = {
'x-image-meta-name': 'test_image',
'x-image-meta-is_public': 'true',
'x-image-meta-disk_format': 'raw',
'x-image-meta-container_format': 'ovf',
'content-type': 'application/octet-stream',
}
path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', body='XXX',
headers=headers)
self.assertEqual(response.status, 201)
image = json.loads(content)['image']
self.assertEqual('active', image['status'])
image_id = image['id']
# delete the image
path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'DELETE')
self.assertEqual(response.status, 200)
response, content = http.request(path, 'HEAD')
self.assertEqual(response.status, 200)
self.assertEqual('pending_delete', response['x-image-meta-status'])
# ensure the marker file has encrypted the image location by decrypting
# it and checking the image_id is intact
file_path = os.path.join(self.api_server.scrubber_datadir,
str(image_id))
marker_uri = ''
with open(file_path, 'r') as f:
marker_uri = f.readline().strip()
self.assertTrue(marker_uri is not None)
decrypted_uri = crypt.urlsafe_decrypt(
self.api_server.metadata_encryption_key, marker_uri)
loc = StoreLocation({})
loc.parse_uri(decrypted_uri)
self.assertEqual("swift+http", loc.scheme)
self.assertEqual(image['id'], loc.obj)
wait_for_scrub(path)
self.stop_servers()
def decrypt_location(uri):
return crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri)
def test_remote_image(self):
"""Verify an image added using a 'Location' header can be retrieved"""
self.cleanup()
self.start_servers(**self.__dict__.copy())
# 1. POST /images with public image named Image1
image_data = "*" * FIVE_KB
headers = minimal_headers('Image1')
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers,
body=image_data)
self.assertEqual(response.status, 201)
data = json.loads(content)
self.assertEqual(data['image']['checksum'],
hashlib.md5(image_data).hexdigest())
self.assertEqual(data['image']['size'], FIVE_KB)
image_id1 = data['image']['id']
# 2. GET first image
# Verify all information on image we just added is correct
path = "http://%s:%d/v1/images/%s"
args = ("0.0.0.0", self.api_port, image_id1)
http = httplib2.Http()
response, content = http.request(path % args, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(response['content-length'], str(FIVE_KB))
self.assertEqual(content, "*" * FIVE_KB)
# 3. GET first image from registry in order to find S3 location
path = "http://%s:%d/images/%s"
args = ("0.0.0.0", self.registry_port, image_id1)
http = httplib2.Http()
response, content = http.request(path % args, 'GET')
if hasattr(self, 'metadata_encryption_key'):
key = self.metadata_encryption_key
else:
key = self.api_server.metadata_encryption_key
loc = json.loads(content)['image']['location']
s3_store_location = crypt.urlsafe_decrypt(key, loc)
# 4. POST /images using location generated by Image1
image_id2 = utils.generate_uuid()
image_data = "*" * FIVE_KB
headers = minimal_headers('Image2')
headers['X-Image-Meta-Id'] = image_id2
headers['X-Image-Meta-Location'] = s3_store_location
path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', headers=headers)
self.assertEqual(response.status, 201)
# ensure data is refreshed, previously the size assertion
# applied to the metadata returned from the previous GET
data = json.loads(content)
self.assertEqual(data['image']['size'], FIVE_KB)
# checksum is not set for a remote image, as the image data
# is not yet retrieved
self.assertEqual(data['image']['checksum'], None)
# 5. GET second image and make sure it can stream the image
path = "http://%s:%d/v1/images/%s"
args = ("0.0.0.0", self.api_port, image_id2)
http = httplib2.Http()
response, content = http.request(path % args, 'GET')
self.assertEqual(response.status, 200)
self.assertEqual(response['content-length'], str(FIVE_KB))
self.assertEqual(content, "*" * FIVE_KB)
# 6. DELETE first and second images
path = "http://%s:%d/v1/images/%s"
args = ("0.0.0.0", self.api_port, image_id1)
http = httplib2.Http()
http.request(path % args, 'DELETE')
path = "http://%s:%d/v1/images/%s"
args = ("0.0.0.0", self.api_port, image_id2)
http = httplib2.Http()
http.request(path % args, 'DELETE')
self.stop_servers()
def test_scrubber_with_metadata_enc(self):
"""
test that files written to scrubber_data_dir use
metadata_encryption_key when available to encrypt the location
"""
# FIXME(flaper87): It looks like an older commit
# may have broken this test. The file_queue `add_location`
# is not being called.
self.skipTest("Test broken. See bug #1366682")
self.cleanup()
self.start_servers(delayed_delete=True,
daemon=True,
default_store='file')
# add an image
headers = {
'x-image-meta-name': 'test_image',
'x-image-meta-is_public': 'true',
'x-image-meta-disk_format': 'raw',
'x-image-meta-container_format': 'ovf',
'content-type': 'application/octet-stream',
}
path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
http = httplib2.Http()
response, content = http.request(path,
'POST',
body='XXX',
headers=headers)
self.assertEqual(201, response.status)
image = jsonutils.loads(content)['image']
self.assertEqual('active', image['status'])
image_id = image['id']
# delete the image
path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'DELETE')
self.assertEqual(200, response.status)
response, content = http.request(path, 'HEAD')
self.assertEqual(200, response.status)
self.assertEqual('pending_delete', response['x-image-meta-status'])
# ensure the marker file has encrypted the image location by decrypting
# it and checking the image_id is intact
file_path = os.path.join(self.api_server.scrubber_datadir,
str(image_id))
marker_uri = None
with open(file_path, 'r') as f:
marker_uri = f.readline().strip()
self.assertTrue(marker_uri is not None)
decrypted_uri = crypt.urlsafe_decrypt(
self.api_server.metadata_encryption_key, marker_uri)
loc = glance_store.location.StoreLocation({})
loc.parse_uri(decrypted_uri)
self.assertEqual("file", loc.scheme)
self.assertEqual(image['id'], loc.obj)
self.wait_for_scrub(path)
self.stop_servers()
def test_scrubber_with_metadata_enc(self):
"""
test that files written to scrubber_data_dir use
metadata_encryption_key when available to encrypt the location
"""
config_path = os.environ.get('GLANCE_TEST_SWIFT_CONF')
if not config_path:
msg = "GLANCE_TEST_SWIFT_CONF environ not set."
self.skipTest(msg)
raw_config = read_config(config_path)
swift_config = parse_config(raw_config)
self.cleanup()
self.start_servers(delayed_delete=True, daemon=True,
default_store='swift', **swift_config)
# add an image
headers = {
'x-image-meta-name': 'test_image',
'x-image-meta-is_public': 'true',
'x-image-meta-disk_format': 'raw',
'x-image-meta-container_format': 'ovf',
'content-type': 'application/octet-stream',
}
path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
http = httplib2.Http()
response, content = http.request(path, 'POST', body='XXX',
headers=headers)
self.assertEqual(response.status, 201)
image = json.loads(content)['image']
self.assertEqual('active', image['status'])
image_id = image['id']
# delete the image
path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port,
image_id)
http = httplib2.Http()
response, content = http.request(path, 'DELETE')
self.assertEqual(response.status, 200)
response, content = http.request(path, 'HEAD')
self.assertEqual(response.status, 200)
self.assertEqual('pending_delete', response['x-image-meta-status'])
# ensure the marker file has encrypted the image location by decrypting
# it and checking the image_id is intact
file_path = os.path.join(self.api_server.scrubber_datadir,
str(image_id))
marker_uri = ''
with open(file_path, 'r') as f:
marker_uri = f.readline().strip()
self.assertTrue(marker_uri is not None)
decrypted_uri = crypt.urlsafe_decrypt(
self.api_server.metadata_encryption_key, marker_uri)
loc = StoreLocation({})
loc.parse_uri(decrypted_uri)
self.assertIn(loc.scheme, ("swift+http", "swift+https"))
self.assertEqual(image['id'], loc.obj)
self.wait_for_scrub(path)
self.stop_servers()
def decrypt_location(uri):
return crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri)
def test_scrubber_with_metadata_enc(self):
"""
test that files written to scrubber_data_dir use
metadata_encryption_key when available to encrypt the location
"""
config_path = os.environ.get("GLANCE_TEST_SWIFT_CONF")
if not config_path:
msg = "GLANCE_TEST_SWIFT_CONF environ not set."
self.skipTest(msg)
raw_config = read_config(config_path)
swift_config = parse_config(raw_config)
self.cleanup()
self.start_servers(delayed_delete=True, daemon=True, default_store="swift", **swift_config)
# add an image
headers = {
"x-image-meta-name": "test_image",
"x-image-meta-is_public": "true",
"x-image-meta-disk_format": "raw",
"x-image-meta-container_format": "ovf",
"content-type": "application/octet-stream",
}
path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port)
http = httplib2.Http()
response, content = http.request(path, "POST", body="XXX", headers=headers)
self.assertEqual(response.status, 201)
image = json.loads(content)["image"]
self.assertEqual("active", image["status"])
image_id = image["id"]
# delete the image
path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port, image_id)
http = httplib2.Http()
response, content = http.request(path, "DELETE")
self.assertEqual(response.status, 200)
response, content = http.request(path, "HEAD")
self.assertEqual(response.status, 200)
self.assertEqual("pending_delete", response["x-image-meta-status"])
# ensure the marker file has encrypted the image location by decrypting
# it and checking the image_id is intact
file_path = os.path.join(self.api_server.scrubber_datadir, str(image_id))
marker_uri = ""
with open(file_path, "r") as f:
marker_uri = f.readline().strip()
self.assertTrue(marker_uri is not None)
decrypted_uri = crypt.urlsafe_decrypt(self.api_server.metadata_encryption_key, marker_uri)
loc = StoreLocation({})
loc.parse_uri(decrypted_uri)
self.assertEqual("swift+http", loc.scheme)
self.assertEqual(image["id"], loc.obj)
# NOTE(jkoelker) The build servers sometimes take longer than
# 15 seconds to scrub. Give it up to 5 min, checking
# checking every 15 seconds. When/if it flips to
# deleted, bail immediately.
for _ in xrange(3):
time.sleep(5)
response, content = http.request(path, "HEAD")
if response["x-image-meta-status"] == "deleted" and response["x-image-meta-deleted"] == "True":
break
else:
continue
else:
self.fail("image was never scrubbed")
self.stop_servers()
