def test_decrypt_locations_on_list(self):
url_loc = ['ping', 'pong']
orig_locations = [{
'url': l,
'metadata': {},
'status': 'active'
} for l in url_loc]
encrypted_locs = [
crypt.urlsafe_encrypt(self.crypt_key, l) for l in url_loc
]
encrypted_locations = [{
'url': l,
'metadata': {},
'status': 'active'
} for l in encrypted_locs]
self.assertNotEqual(encrypted_locations, orig_locations)
db_data = _db_fixture(UUID1,
owner=TENANT1,
locations=encrypted_locations)
self.db.image_create(None, db_data)
image = self.image_repo.list()[0]
self.assertIn('id', image.locations[0])
self.assertIn('id', image.locations[1])
image.locations[0].pop('id')
image.locations[1].pop('id')
self.assertEqual(orig_locations, image.locations)
def get_all_locations(self):
"""Returns a list of image id and location tuple from scrub queue.
:retval a list of image id, location id and uri tuple from scrub queue
"""
ret = []
for image in self._get_all_images():
deleted_at = image.get('deleted_at')
if not deleted_at:
continue
# NOTE: Strip off microseconds which may occur after the last '.,'
# Example: 2012-07-07T19:14:34.974216
date_str = deleted_at.rsplit('.', 1)[0].rsplit(',', 1)[0]
delete_time = calendar.timegm(time.strptime(date_str,
"%Y-%m-%dT%H:%M:%S"))
if delete_time + self.scrub_time > time.time():
continue
for loc in image['location_data']:
if loc['status'] != 'pending_delete':
continue
if self.metadata_encryption_key:
uri = crypt.urlsafe_encrypt(self.metadata_encryption_key,
loc['url'], 64)
else:
uri = loc['url']
ret.append((image['id'], loc['id'], uri))
return ret
def test_encryption(self):
# Check that original plaintext and unencrypted ciphertext match
# Check keys of the three allowed lengths
key_list = ["1234567890abcdef",
"12345678901234567890abcd",
"1234567890abcdef1234567890ABCDEF"]
plaintext_list = ['']
blocksize = 64
for i in range(3 * blocksize):
text = os.urandom(i)
if six.PY3:
text = text.decode('latin1')
plaintext_list.append(text)
for key in key_list:
for plaintext in plaintext_list:
ciphertext = crypt.urlsafe_encrypt(key, plaintext, blocksize)
self.assertIsInstance(ciphertext, bytes)
if six.PY3:
self.assertNotEqual(ciphertext, plaintext.encode('utf-8'))
else:
self.assertNotEqual(ciphertext, plaintext)
text = crypt.urlsafe_decrypt(key, ciphertext)
self.assertIsInstance(text, str)
self.assertEqual(plaintext, text)
def get_all_locations(self):
"""Returns a list of image id and location tuple from scrub queue.
:returns: a list of image id, location id and uri tuple from
scrub queue
"""
ret = []
for image in self._get_all_images():
deleted_at = image.get('deleted_at')
if not deleted_at:
continue
# NOTE: Strip off microseconds which may occur after the last '.,'
# Example: 2012-07-07T19:14:34.974216
date_str = deleted_at.rsplit('.', 1)[0].rsplit(',', 1)[0]
delete_time = calendar.timegm(
time.strptime(date_str, "%Y-%m-%dT%H:%M:%S"))
if delete_time + self.scrub_time > time.time():
continue
for loc in image['location_data']:
if loc['status'] != 'pending_delete':
continue
if self.metadata_encryption_key:
uri = crypt.urlsafe_encrypt(self.metadata_encryption_key,
loc['url'], 64)
else:
uri = loc['url']
ret.append((image['id'], loc['id'], uri))
return ret
def _format_image_to_db(self, image):
locations = image.locations
if CONF.metadata_encryption_key:
key = CONF.metadata_encryption_key
ld = []
for l in locations:
url = crypt.urlsafe_encrypt(key, l['url'])
ld.append({'url': url, 'metadata': l['metadata']})
locations = ld
return {
'id': image.image_id,
'name': image.name,
'status': image.status,
'created_at': image.created_at,
'min_disk': image.min_disk,
'min_ram': image.min_ram,
'protected': image.protected,
'locations': locations,
'checksum': image.checksum,
'owner': image.owner,
'disk_format': image.disk_format,
'container_format': image.container_format,
'size': image.size,
'virtual_size': image.virtual_size,
'is_public': image.visibility == 'public',
'properties': dict(image.extra_properties),
}
def _format_image_to_db(self, image):
locations = image.locations
if CONF.metadata_encryption_key:
key = CONF.metadata_encryption_key
ld = []
for l in locations:
url = crypt.urlsafe_encrypt(key, l['url'])
ld.append({'url': url, 'metadata': l['metadata']})
locations = ld
return {
'id': image.image_id,
'name': image.name,
'status': image.status,
'created_at': image.created_at,
'min_disk': image.min_disk,
'min_ram': image.min_ram,
'protected': image.protected,
'locations': locations,
'checksum': image.checksum,
'owner': image.owner,
'disk_format': image.disk_format,
'container_format': image.container_format,
'size': image.size,
'virtual_size': image.virtual_size,
'is_public': image.visibility == 'public',
'properties': dict(image.extra_properties),
}
def encrypt_metadata(self, image_metadata):
if self.metadata_encryption_key is not None:
location_url = image_metadata.get("location")
if location_url:
location = crypt.urlsafe_encrypt(self.metadata_encryption_key, location_url, 64)
image_metadata["location"] = location
if image_metadata.get("location_data"):
ld = []
for loc in image_metadata["location_data"]:
if loc["url"] == location_url:
url = location
else:
url = crypt.urlsafe_encrypt(self.metadata_encryption_key, loc["url"], 64)
ld.append({"url": url, "metadata": loc["metadata"]})
image_metadata["location_data"] = ld
return image_metadata
def _format_image_to_db(self, image):
locations = image.locations
if CONF.metadata_encryption_key:
key = CONF.metadata_encryption_key
ld = []
for loc in locations:
url = crypt.urlsafe_encrypt(key, loc['url'])
ld.append({'url': url, 'metadata': loc['metadata'],
'status': loc['status'],
# NOTE(zhiyan): New location has no ID field.
'id': loc.get('id')})
locations = ld
return {
'id': image.image_id,
'name': image.name,
'status': image.status,
'created_at': image.created_at,
'min_disk': image.min_disk,
'min_ram': image.min_ram,
'protected': image.protected,
'locations': locations,
'checksum': image.checksum,
'os_hash_algo': image.os_hash_algo,
'os_hash_value': image.os_hash_value,
'owner': image.owner,
'disk_format': image.disk_format,
'container_format': image.container_format,
'size': image.size,
'virtual_size': image.virtual_size,
'visibility': image.visibility,
'properties': dict(image.extra_properties),
'os_hidden': image.os_hidden
}
def _format_image_to_db(self, image):
locations = image.locations
if CONF.metadata_encryption_key:
key = CONF.metadata_encryption_key
ld = []
for loc in locations:
url = crypt.urlsafe_encrypt(key, loc["url"])
ld.append(
{
"url": url,
"metadata": loc["metadata"],
"status": loc["status"],
# NOTE(zhiyan): New location has no ID field.
"id": loc.get("id"),
}
)
locations = ld
return {
"id": image.image_id,
"name": image.name,
"status": image.status,
"created_at": image.created_at,
"min_disk": image.min_disk,
"min_ram": image.min_ram,
"protected": image.protected,
"locations": locations,
"checksum": image.checksum,
"owner": image.owner,
"disk_format": image.disk_format,
"container_format": image.container_format,
"size": image.size,
"virtual_size": image.virtual_size,
"is_public": image.visibility == "public",
"properties": dict(image.extra_properties),
}
def _format_image_to_db(self, image):
locations = image.locations
if CONF.metadata_encryption_key:
key = CONF.metadata_encryption_key
ld = []
for loc in locations:
url = crypt.urlsafe_encrypt(key, loc['url'])
ld.append({
'url': url,
'metadata': loc['metadata'],
'status': loc['status'],
# NOTE(zhiyan): New location has no ID field.
'id': loc.get('id')
})
locations = ld
return {
'id': image.image_id,
'name': image.name,
'status': image.status,
'created_at': image.created_at,
'min_disk': image.min_disk,
'min_ram': image.min_ram,
'protected': image.protected,
'locations': locations,
'checksum': image.checksum,
'owner': image.owner,
'disk_format': image.disk_format,
'container_format': image.container_format,
'size': image.size,
'virtual_size': image.virtual_size,
'visibility': image.visibility,
'properties': dict(image.extra_properties),
}
def _prerun_017(self, engine):
metadata_encryption_key = "a" * 16
self.config(metadata_encryption_key=metadata_encryption_key)
images = get_table(engine, "images")
unquoted = "swift://*****:*****@example.com/container/obj-id"
encrypted_unquoted = crypt.urlsafe_encrypt(metadata_encryption_key, unquoted, 64)
data = []
now = datetime.datetime.now()
temp = dict(
deleted=False,
created_at=now,
updated_at=now,
status="active",
is_public=True,
min_disk=0,
min_ram=0,
location=encrypted_unquoted,
id="fakeid1",
)
images.insert().values(temp).execute()
locations = ["file://ab", "file://abc", "swift://acct3A%foobar:[email protected]/container/obj-id2"]
now = datetime.datetime.now()
temp = dict(
deleted=False, created_at=now, updated_at=now, status="active", is_public=True, min_disk=0, min_ram=0
)
for i, location in enumerate(locations):
temp.update(location=location, id=uuidutils.generate_uuid())
data.append(temp)
images.insert().values(temp).execute()
return data
def encrypt_metadata(self, image_metadata):
if (self.metadata_encryption_key is not None and
'location' in image_metadata and
image_metadata['location'] is not None):
location = crypt.urlsafe_encrypt(self.metadata_encryption_key,
image_metadata['location'], 64)
image_metadata['location'] = location
return image_metadata
def encrypt_metadata(self, image_metadata):
if (self.metadata_encryption_key is not None
and 'location' in image_metadata.keys()
and image_metadata['location'] is not None):
location = crypt.urlsafe_encrypt(self.metadata_encryption_key,
image_metadata['location'], 64)
image_metadata['location'] = location
return image_metadata
def encrypt_metadata(self, image_metadata):
if (
self.metadata_encryption_key is not None
and "location" in image_metadata
and image_metadata["location"] is not None
):
location = crypt.urlsafe_encrypt(self.metadata_encryption_key, image_metadata["location"], 64)
image_metadata["location"] = location
return image_metadata
def test_decrypt_locations_on_list(self):
encrypted_locations = [crypt.urlsafe_encrypt(self.crypt_key, l)
for l in ['ping', 'pong']]
self.assertNotEqual(encrypted_locations, ['ping', 'pong'])
db_data = _db_fixture(UUID1, owner=TENANT1,
locations=encrypted_locations)
self.db.image_create(None, db_data)
image = self.image_repo.list()[0]
self.assertEqual(image.locations, ['ping', 'pong'])
def encrypt_metadata(self, image_metadata):
if self.metadata_encryption_key is not None:
location_url = image_metadata.get('location')
if location_url:
location = crypt.urlsafe_encrypt(self.metadata_encryption_key,
location_url, 64)
image_metadata['location'] = location
if image_metadata.get('location_data'):
ld = []
for loc in image_metadata['location_data']:
if loc['url'] == location_url:
url = location
else:
url = crypt.urlsafe_encrypt(
self.metadata_encryption_key, loc['url'], 64)
ld.append({'url': url, 'metadata': loc['metadata']})
image_metadata['location_data'] = ld
return image_metadata
def test_decrypt_locations_on_list(self):
encrypted_locations = [crypt.urlsafe_encrypt(self.crypt_key, l)
for l in ['ping', 'pong']]
self.assertNotEqual(encrypted_locations, ['ping', 'pong'])
db_data = _db_fixture(UUID1, owner=TENANT1,
locations=encrypted_locations)
self.db.image_create(None, db_data)
image = self.image_repo.list()[0]
self.assertEqual(image.locations, ['ping', 'pong'])
def add_location(self, image_id, location, user_context=None):
"""Adding image location to scrub queue.
:param image_id: The opaque image identifier
:param location: The opaque image location
:param user_context: The user's request context
:retval A boolean value to indicate success or not
"""
if user_context is not None:
registry_client = registry.get_registry_client(user_context)
else:
registry_client = self.registry
with lockutils.lock("scrubber-%s" % image_id,
lock_file_prefix='glance-',
external=True):
# NOTE(zhiyan): make sure scrubber does not cleanup
# 'pending_delete' images concurrently before the code
# get lock and reach here.
try:
image = registry_client.get_image(image_id)
if image['status'] == 'deleted':
return True
except exception.NotFound as e:
LOG.warn(_LW("Failed to find image to delete: %s"),
utils.exception_to_str(e))
return False
loc_id = location.get('id', '-')
if self.metadata_encryption_key:
uri = crypt.urlsafe_encrypt(self.metadata_encryption_key,
location['url'], 64)
else:
uri = location['url']
delete_time = time.time() + self.scrub_time
file_path = os.path.join(self.scrubber_datadir, str(image_id))
if os.path.exists(file_path):
# Append the uri of location to the queue file
with open(file_path, 'a') as f:
f.write('\n')
f.write('\n'.join(
[str(loc_id), uri,
str(int(delete_time))]))
else:
# NOTE(zhiyan): Protect the file before we write any data.
open(file_path, 'w').close()
os.chmod(file_path, 0o600)
with open(file_path, 'w') as f:
f.write('\n'.join(
[str(loc_id), uri,
str(int(delete_time))]))
os.utime(file_path, (delete_time, delete_time))
return True
def add_location(self, image_id, location, user_context=None):
"""Adding image location to scrub queue.
:param image_id: The opaque image identifier
:param location: The opaque image location
:param user_context: The user's request context
:retval A boolean value to indicate success or not
"""
if user_context is not None:
registry_client = registry.get_registry_client(user_context)
else:
registry_client = self.registry
with lockutils.lock("scrubber-%s" % image_id,
lock_file_prefix='glance-', external=True):
# NOTE(zhiyan): make sure scrubber does not cleanup
# 'pending_delete' images concurrently before the code
# get lock and reach here.
try:
image = registry_client.get_image(image_id)
if image['status'] == 'deleted':
return True
except exception.NotFound as e:
LOG.warn(_LW("Failed to find image to delete: %s"),
utils.exception_to_str(e))
return False
loc_id = location.get('id', '-')
if self.metadata_encryption_key:
uri = crypt.urlsafe_encrypt(self.metadata_encryption_key,
location['url'], 64)
else:
uri = location['url']
delete_time = time.time() + self.scrub_time
file_path = os.path.join(self.scrubber_datadir, str(image_id))
if os.path.exists(file_path):
# Append the uri of location to the queue file
with open(file_path, 'a') as f:
f.write('\n')
f.write('\n'.join([str(loc_id),
uri,
str(int(delete_time))]))
else:
# NOTE(zhiyan): Protect the file before we write any data.
open(file_path, 'w').close()
os.chmod(file_path, 0o600)
with open(file_path, 'w') as f:
f.write('\n'.join([str(loc_id),
uri,
str(int(delete_time))]))
os.utime(file_path, (delete_time, delete_time))
return True
def test_decrypt_locations_on_list(self):
url_loc = ["ping", "pong"]
orig_locations = [{"url": l, "metadata": {}} for l in url_loc]
encrypted_locs = [crypt.urlsafe_encrypt(self.crypt_key, l) for l in url_loc]
encrypted_locations = [{"url": l, "metadata": {}} for l in encrypted_locs]
self.assertNotEqual(encrypted_locations, orig_locations)
db_data = _db_fixture(UUID1, owner=TENANT1, locations=encrypted_locations)
self.db.image_create(None, db_data)
image = self.image_repo.list()[0]
self.assertEqual(image.locations, orig_locations)
def encrypt_metadata(self, image_metadata):
if self.metadata_encryption_key is not None:
location_url = image_metadata.get('location')
if location_url:
location = crypt.urlsafe_encrypt(self.metadata_encryption_key,
location_url,
64)
image_metadata['location'] = location
if image_metadata.get('location_data'):
ld = []
for loc in image_metadata['location_data']:
if loc['url'] == location_url:
url = location
else:
url = crypt.urlsafe_encrypt(
self.metadata_encryption_key, loc['url'], 64)
ld.append({'url': url, 'metadata': loc['metadata']})
image_metadata['location_data'] = ld
return image_metadata
def _check_16_to_17(self, engine):
"""
Check that migrating swift location credentials to quoted form
and back works.
"""
migration_api.version_control(version=0)
migration_api.upgrade(16)
conn = engine.connect()
images_table = Table('images', MetaData(), autoload=True,
autoload_with=engine)
def get_locations():
conn = engine.connect()
locations = [x[0] for x in
conn.execute(
select(['location'], from_obj=[images_table]))]
conn.close()
return locations
unquoted = 'swift://*****:*****@example.com/container/obj-id'
encrypted_unquoted = crypt.urlsafe_encrypt(
self.metadata_encryption_key,
unquoted, 64)
quoted = 'swift://acct%3Ausr:[email protected]/container/obj-id'
# Insert image with an unquoted image location
now = datetime.datetime.now()
kwargs = dict(deleted=False,
created_at=now,
updated_at=now,
status='active',
is_public=True,
min_disk=0,
min_ram=0)
kwargs.update(location=encrypted_unquoted, id=1)
conn.execute(images_table.insert(), [kwargs])
conn.close()
migration_api.upgrade(17)
actual_location = crypt.urlsafe_decrypt(self.metadata_encryption_key,
get_locations()[0])
self.assertEqual(actual_location, quoted)
migration_api.downgrade(16)
actual_location = crypt.urlsafe_decrypt(self.metadata_encryption_key,
get_locations()[0])
self.assertEqual(actual_location, unquoted)
def encrypt_metadata(self, image_metadata):
if self.metadata_encryption_key:
location_url = image_metadata.get('location')
if location_url:
location = crypt.urlsafe_encrypt(self.metadata_encryption_key,
location_url,
64)
image_metadata['location'] = location
if image_metadata.get('location_data'):
ld = []
for loc in image_metadata['location_data']:
if loc['url'] == location_url:
url = location
else:
url = crypt.urlsafe_encrypt(
self.metadata_encryption_key, loc['url'], 64)
ld.append({'url': url, 'metadata': loc['metadata'],
'status': loc['status'],
# NOTE(zhiyan): New location has no ID field.
'id': loc.get('id')})
image_metadata['location_data'] = ld
return image_metadata
def test_decrypt_locations_on_get(self):
url_loc = ['ping', 'pong']
orig_locations = [{'url': l, 'metadata': {}} for l in url_loc]
encrypted_locs = [crypt.urlsafe_encrypt(self.crypt_key, l)
for l in url_loc]
encrypted_locations = [{'url': l, 'metadata': {}}
for l in encrypted_locs]
self.assertNotEqual(encrypted_locations, orig_locations)
db_data = _db_fixture(UUID1, owner=TENANT1,
locations=encrypted_locations)
self.db.image_create(None, db_data)
image = self.image_repo.get(UUID1)
self.assertEqual(image.locations, orig_locations)
def encrypt_metadata(self, image_metadata):
if self.metadata_encryption_key is not None:
location_url = image_metadata.get('location')
if location_url:
location = crypt.urlsafe_encrypt(self.metadata_encryption_key,
location_url,
64)
image_metadata['location'] = location
if image_metadata.get('location_data'):
ld = []
for loc in image_metadata['location_data']:
if loc['url'] == location_url:
url = location
else:
url = crypt.urlsafe_encrypt(
self.metadata_encryption_key, loc['url'], 64)
ld.append({'url': url, 'metadata': loc['metadata'],
'status': loc['status'],
# NOTE(zhiyan): New location has no ID field.
'id': loc.get('id')})
image_metadata['location_data'] = ld
return image_metadata
def test_decrypt_locations_on_get(self):
url_loc = ["ping", "pong"]
orig_locations = [{"url": l, "metadata": {}, "status": "active"} for l in url_loc]
encrypted_locs = [crypt.urlsafe_encrypt(self.crypt_key, l) for l in url_loc]
encrypted_locations = [{"url": l, "metadata": {}, "status": "active"} for l in encrypted_locs]
self.assertNotEqual(encrypted_locations, orig_locations)
db_data = _db_fixture(UUID1, owner=TENANT1, locations=encrypted_locations)
self.db.image_create(None, db_data)
image = self.image_repo.get(UUID1)
self.assertIn("id", image.locations[0])
self.assertIn("id", image.locations[1])
image.locations[0].pop("id")
image.locations[1].pop("id")
self.assertEqual(orig_locations, image.locations)
def test_encryption(self):
# Check that original plaintext and unencrypted ciphertext match
# Check keys of the three allowed lengths
key_list = ["1234567890abcdef", "12345678901234567890abcd", "1234567890abcdef1234567890ABCDEF"]
plaintext_list = [""]
blocksize = 64
for i in range(3 * blocksize):
plaintext_list.append(os.urandom(i))
for key in key_list:
for plaintext in plaintext_list:
ciphertext = crypt.urlsafe_encrypt(key, plaintext, blocksize)
self.assertTrue(ciphertext != plaintext)
text = crypt.urlsafe_decrypt(key, ciphertext)
self.assertTrue(plaintext == text)
def encrypt_metadata(self, image_metadata):
if self.metadata_encryption_key:
location_url = image_metadata.get("location")
if location_url:
location = crypt.urlsafe_encrypt(self.metadata_encryption_key, location_url, 64)
image_metadata["location"] = location
if image_metadata.get("location_data"):
ld = []
for loc in image_metadata["location_data"]:
if loc["url"] == location_url:
url = location
else:
url = crypt.urlsafe_encrypt(self.metadata_encryption_key, loc["url"], 64)
ld.append(
{
"url": url,
"metadata": loc["metadata"],
"status": loc["status"],
# NOTE(zhiyan): New location has no ID field.
"id": loc.get("id"),
}
)
image_metadata["location_data"] = ld
return image_metadata
def schedule_delayed_delete_from_backend(uri, image_id, **kwargs):
"""Given a uri, schedule the deletion of an image."""
datadir = CONF.scrubber_datadir
delete_time = time.time() + CONF.scrub_time
file_path = os.path.join(datadir, str(image_id))
utils.safe_mkdirs(datadir)
if os.path.exists(file_path):
msg = _("Image id %(image_id)s already queued for delete") % {"image_id": image_id}
raise exception.Duplicate(msg)
if CONF.metadata_encryption_key is not None:
uri = crypt.urlsafe_encrypt(CONF.metadata_encryption_key, uri, 64)
with open(file_path, "w") as f:
f.write("\n".join([uri, str(int(delete_time))]))
os.chmod(file_path, 0o600)
os.utime(file_path, (delete_time, delete_time))
def test_decrypt_locations_on_get(self):
url_loc = ['ping', 'pong']
orig_locations = [{'url': l, 'metadata': {}} for l in url_loc]
encrypted_locs = [
crypt.urlsafe_encrypt(self.crypt_key, l) for l in url_loc
]
encrypted_locations = [{
'url': l,
'metadata': {}
} for l in encrypted_locs]
self.assertNotEqual(encrypted_locations, orig_locations)
db_data = _db_fixture(UUID1,
owner=TENANT1,
locations=encrypted_locations)
self.db.image_create(None, db_data)
image = self.image_repo.get(UUID1)
self.assertEqual(image.locations, orig_locations)
def test_encryption(self):
# Check that original plaintext and unencrypted ciphertext match
# Check keys of the three allowed lengths
key_list = ["1234567890abcdef",
"12345678901234567890abcd",
"1234567890abcdef1234567890ABCDEF"]
plaintext_list = ['']
blocksize = 64
for i in range(3 * blocksize):
plaintext_list.append(os.urandom(i))
for key in key_list:
for plaintext in plaintext_list:
ciphertext = crypt.urlsafe_encrypt(key, plaintext, blocksize)
self.assertTrue(ciphertext != plaintext)
text = crypt.urlsafe_decrypt(key, ciphertext)
self.assertTrue(plaintext == text)
def _walk_all_locations(self, remove=False):
"""Returns a list of image id and location tuple from scrub queue.
:param remove: Whether remove location from queue or not after walk
:retval a list of image id, location id and uri tuple from scrub queue
"""
filters = {
'deleted': True,
'is_public': 'none',
'status': 'pending_delete'
}
ret = []
for image in self.registry.get_images_detailed(filters=filters):
deleted_at = image.get('deleted_at')
if not deleted_at:
continue
# NOTE: Strip off microseconds which may occur after the last '.,'
# Example: 2012-07-07T19:14:34.974216
date_str = deleted_at.rsplit('.', 1)[0].rsplit(',', 1)[0]
delete_time = calendar.timegm(
time.strptime(date_str, "%Y-%m-%dT%H:%M:%S"))
if delete_time + self.scrub_time > time.time():
continue
for loc in image['location_data']:
if loc['status'] != 'pending_delete':
continue
if self.metadata_encryption_key:
uri = crypt.urlsafe_encrypt(self.metadata_encryption_key,
loc['url'], 64)
else:
uri = loc['url']
ret.append((image['id'], loc['id'], uri))
if remove:
db_api.get_api().image_location_delete(
self.admin_context, image['id'], loc['id'], 'deleted')
self.registry.update_image(image['id'],
{'status': 'deleted'})
return ret
def _walk_all_locations(self, remove=False):
"""Returns a list of image id and location tuple from scrub queue.
:param remove: Whether remove location from queue or not after walk
:retval a list of image id, location id and uri tuple from scrub queue
"""
filters = {'deleted': True,
'is_public': 'none',
'status': 'pending_delete'}
ret = []
for image in self.registry.get_images_detailed(filters=filters):
deleted_at = image.get('deleted_at')
if not deleted_at:
continue
# NOTE: Strip off microseconds which may occur after the last '.,'
# Example: 2012-07-07T19:14:34.974216
date_str = deleted_at.rsplit('.', 1)[0].rsplit(',', 1)[0]
delete_time = calendar.timegm(time.strptime(date_str,
"%Y-%m-%dT%H:%M:%S"))
if delete_time + self.scrub_time > time.time():
continue
for loc in image['location_data']:
if loc['status'] != 'pending_delete':
continue
if self.metadata_encryption_key:
uri = crypt.urlsafe_encrypt(self.metadata_encryption_key,
loc['url'], 64)
else:
uri = loc['url']
ret.append((image['id'], loc['id'], uri))
if remove:
db_api.get_api().image_location_delete(self.admin_context,
image['id'],
loc['id'],
'deleted')
self.registry.update_image(image['id'],
{'status': 'deleted'})
return ret
def schedule_delayed_delete_from_backend(uri, image_id, **kwargs):
"""Given a uri, schedule the deletion of an image."""
datadir = CONF.scrubber_datadir
delete_time = time.time() + CONF.scrub_time
file_path = os.path.join(datadir, str(image_id))
utils.safe_mkdirs(datadir)
if os.path.exists(file_path):
msg = _("Image id %(image_id)s already queued for delete") % {
'image_id': image_id}
raise exception.Duplicate(msg)
if CONF.metadata_encryption_key is not None:
uri = crypt.urlsafe_encrypt(CONF.metadata_encryption_key, uri, 64)
with open(file_path, 'w') as f:
f.write('\n'.join([uri, str(int(delete_time))]))
os.chmod(file_path, 0600)
os.utime(file_path, (delete_time, delete_time))
def test_decrypt_locations_on_list(self):
url_loc = ['ping', 'pong']
orig_locations = [{'url': l, 'metadata': {}, 'status': 'active'}
for l in url_loc]
encrypted_locs = [crypt.urlsafe_encrypt(self.crypt_key, l)
for l in url_loc]
encrypted_locations = [{'url': l, 'metadata': {}, 'status': 'active'}
for l in encrypted_locs]
self.assertNotEqual(encrypted_locations, orig_locations)
db_data = _db_fixture(UUID1, owner=TENANT1,
locations=encrypted_locations)
self.db.image_create(None, db_data)
image = self.image_repo.list()[0]
self.assertIn('id', image.locations[0])
self.assertIn('id', image.locations[1])
image.locations[0].pop('id')
image.locations[1].pop('id')
self.assertEqual(orig_locations, image.locations)
def add_location(self, image_id, uri):
"""Adding image location to scrub queue.
:param image_id: The opaque image identifier
:param uri: The opaque image location uri
"""
with lockutils.lock("scrubber-%s" % image_id,
lock_file_prefix='glance-',
external=True):
# NOTE(zhiyan): make sure scrubber does not cleanup
# 'pending_delete' images concurrently before the code
# get lock and reach here.
try:
image = self.registry.get_image(image_id)
if image['status'] == 'deleted':
return
except exception.NotFound as e:
LOG.error(
_("Failed to find image to delete: "
"%(e)s") % locals())
return
delete_time = time.time() + self.scrub_time
file_path = os.path.join(self.scrubber_datadir, str(image_id))
if self.metadata_encryption_key is not None:
uri = crypt.urlsafe_encrypt(self.metadata_encryption_key, uri,
64)
if os.path.exists(file_path):
# Append the uri of location to the queue file
with open(file_path, 'a') as f:
f.write('\n')
f.write('\n'.join([uri, str(int(delete_time))]))
else:
# NOTE(zhiyan): Protect the file before we write any data.
open(file_path, 'w').close()
os.chmod(file_path, 0o600)
with open(file_path, 'w') as f:
f.write('\n'.join([uri, str(int(delete_time))]))
os.utime(file_path, (delete_time, delete_time))
def _pre_upgrade_017(self, engine):
metadata_encryption_key = 'a' * 16
CONF.set_override('metadata_encryption_key', metadata_encryption_key)
self.addCleanup(CONF.reset)
images = db_utils.get_table(engine, 'images')
unquoted = 'swift://*****:*****@example.com/container/obj-id'
encrypted_unquoted = crypt.urlsafe_encrypt(
metadata_encryption_key,
unquoted, 64)
data = []
now = datetime.datetime.now()
temp = dict(deleted=False,
created_at=now,
updated_at=now,
status='active',
is_public=True,
min_disk=0,
min_ram=0,
location=encrypted_unquoted,
id='fakeid1')
images.insert().values(temp).execute()
locations = [
'file://ab',
'file://abc',
'swift://acct3A%foobar:[email protected]/container/obj-id2'
]
now = datetime.datetime.now()
temp = dict(deleted=False,
created_at=now,
updated_at=now,
status='active',
is_public=True,
min_disk=0,
min_ram=0)
for i, location in enumerate(locations):
temp.update(location=location, id=str(uuid.uuid4()))
data.append(temp)
images.insert().values(temp).execute()
return data
def _pre_upgrade_017(self, engine):
metadata_encryption_key = 'a' * 16
CONF.set_override('metadata_encryption_key', metadata_encryption_key)
self.addCleanup(CONF.reset)
images = db_utils.get_table(engine, 'images')
unquoted = 'swift://*****:*****@example.com/container/obj-id'
encrypted_unquoted = crypt.urlsafe_encrypt(
metadata_encryption_key,
unquoted, 64)
data = []
now = datetime.datetime.now()
temp = dict(deleted=False,
created_at=now,
updated_at=now,
status='active',
is_public=True,
min_disk=0,
min_ram=0,
location=encrypted_unquoted,
id='fakeid1')
images.insert().values(temp).execute()
locations = [
'file://ab',
'file://abc',
'swift://acct3A%foobar:[email protected]/container/obj-id2'
]
now = datetime.datetime.now()
temp = dict(deleted=False,
created_at=now,
updated_at=now,
status='active',
is_public=True,
min_disk=0,
min_ram=0)
for i, location in enumerate(locations):
temp.update(location=location, id=str(uuid.uuid4()))
data.append(temp)
images.insert().values(temp).execute()
return data
def add_location(self, image_id, uri):
"""Adding image location to scrub queue.
:param image_id: The opaque image identifier
:param uri: The opaque image location uri
"""
with lockutils.lock("scrubber-%s" % image_id,
lock_file_prefix='glance-', external=True):
# NOTE(zhiyan): make sure scrubber does not cleanup
# 'pending_delete' images concurrently before the code
# get lock and reach here.
try:
image = self.registry.get_image(image_id)
if image['status'] == 'deleted':
return
except exception.NotFound as e:
LOG.error(_("Failed to find image to delete: "
"%(e)s") % locals())
return
delete_time = time.time() + self.scrub_time
file_path = os.path.join(self.scrubber_datadir, str(image_id))
if self.metadata_encryption_key is not None:
uri = crypt.urlsafe_encrypt(self.metadata_encryption_key,
uri, 64)
if os.path.exists(file_path):
# Append the uri of location to the queue file
with open(file_path, 'a') as f:
f.write('\n')
f.write('\n'.join([uri, str(int(delete_time))]))
else:
# NOTE(zhiyan): Protect the file before we write any data.
open(file_path, 'w').close()
os.chmod(file_path, 0o600)
with open(file_path, 'w') as f:
f.write('\n'.join([uri, str(int(delete_time))]))
os.utime(file_path, (delete_time, delete_time))
def _format_image_to_db(self, image):
locations = image.locations
if CONF.metadata_encryption_key:
key = CONF.metadata_encryption_key
locations = [crypt.urlsafe_encrypt(key, l) for l in locations]
return {
"id": image.image_id,
"name": image.name,
"status": image.status,
"created_at": image.created_at,
"min_disk": image.min_disk,
"min_ram": image.min_ram,
"protected": image.protected,
"locations": locations,
"checksum": image.checksum,
"owner": image.owner,
"disk_format": image.disk_format,
"container_format": image.container_format,
"size": image.size,
"is_public": image.visibility == "public",
"properties": dict(image.extra_properties),
}
def _walk_all_locations(self, remove=False):
"""Returns a list of image id and location tuple from scrub queue.
:param remove: Whether remove location from queue or not after walk
:retval a list of image id, location id and uri tuple from scrub queue
"""
filters = {"deleted": True, "is_public": "none", "status": "pending_delete"}
ret = []
for image in self.registry.get_images_detailed(filters=filters):
deleted_at = image.get("deleted_at")
if not deleted_at:
continue
# NOTE: Strip off microseconds which may occur after the last '.,'
# Example: 2012-07-07T19:14:34.974216
date_str = deleted_at.rsplit(".", 1)[0].rsplit(",", 1)[0]
delete_time = calendar.timegm(time.strptime(date_str, "%Y-%m-%dT%H:%M:%S"))
if delete_time + self.scrub_time > time.time():
continue
for loc in image["location_data"]:
if loc["status"] != "pending_delete":
continue
if self.metadata_encryption_key:
uri = crypt.urlsafe_encrypt(self.metadata_encryption_key, loc["url"], 64)
else:
uri = loc["url"]
ret.append((image["id"], loc["id"], uri))
if remove:
db_api.get_api().image_location_delete(self.admin_context, image["id"], loc["id"], "deleted")
self.registry.update_image(image["id"], {"status": "deleted"})
return ret
def encrypt_location(uri):
return crypt.urlsafe_encrypt(CONF.metadata_encryption_key, uri, 64)
def encrypt_location(uri):
return crypt.urlsafe_encrypt(CONF.metadata_encryption_key, uri, 64)
