def test_decrypt_locations_on_list(self): url_loc = ['ping', 'pong'] orig_locations = [{ 'url': l, 'metadata': {}, 'status': 'active' } for l in url_loc] encrypted_locs = [ crypt.urlsafe_encrypt(self.crypt_key, l) for l in url_loc ] encrypted_locations = [{ 'url': l, 'metadata': {}, 'status': 'active' } for l in encrypted_locs] self.assertNotEqual(encrypted_locations, orig_locations) db_data = _db_fixture(UUID1, owner=TENANT1, locations=encrypted_locations) self.db.image_create(None, db_data) image = self.image_repo.list()[0] self.assertIn('id', image.locations[0]) self.assertIn('id', image.locations[1]) image.locations[0].pop('id') image.locations[1].pop('id') self.assertEqual(orig_locations, image.locations)
def get_all_locations(self): """Returns a list of image id and location tuple from scrub queue. :retval a list of image id, location id and uri tuple from scrub queue """ ret = [] for image in self._get_all_images(): deleted_at = image.get('deleted_at') if not deleted_at: continue # NOTE: Strip off microseconds which may occur after the last '.,' # Example: 2012-07-07T19:14:34.974216 date_str = deleted_at.rsplit('.', 1)[0].rsplit(',', 1)[0] delete_time = calendar.timegm(time.strptime(date_str, "%Y-%m-%dT%H:%M:%S")) if delete_time + self.scrub_time > time.time(): continue for loc in image['location_data']: if loc['status'] != 'pending_delete': continue if self.metadata_encryption_key: uri = crypt.urlsafe_encrypt(self.metadata_encryption_key, loc['url'], 64) else: uri = loc['url'] ret.append((image['id'], loc['id'], uri)) return ret
def test_encryption(self): # Check that original plaintext and unencrypted ciphertext match # Check keys of the three allowed lengths key_list = ["1234567890abcdef", "12345678901234567890abcd", "1234567890abcdef1234567890ABCDEF"] plaintext_list = [''] blocksize = 64 for i in range(3 * blocksize): text = os.urandom(i) if six.PY3: text = text.decode('latin1') plaintext_list.append(text) for key in key_list: for plaintext in plaintext_list: ciphertext = crypt.urlsafe_encrypt(key, plaintext, blocksize) self.assertIsInstance(ciphertext, bytes) if six.PY3: self.assertNotEqual(ciphertext, plaintext.encode('utf-8')) else: self.assertNotEqual(ciphertext, plaintext) text = crypt.urlsafe_decrypt(key, ciphertext) self.assertIsInstance(text, str) self.assertEqual(plaintext, text)
def get_all_locations(self): """Returns a list of image id and location tuple from scrub queue. :returns: a list of image id, location id and uri tuple from scrub queue """ ret = [] for image in self._get_all_images(): deleted_at = image.get('deleted_at') if not deleted_at: continue # NOTE: Strip off microseconds which may occur after the last '.,' # Example: 2012-07-07T19:14:34.974216 date_str = deleted_at.rsplit('.', 1)[0].rsplit(',', 1)[0] delete_time = calendar.timegm( time.strptime(date_str, "%Y-%m-%dT%H:%M:%S")) if delete_time + self.scrub_time > time.time(): continue for loc in image['location_data']: if loc['status'] != 'pending_delete': continue if self.metadata_encryption_key: uri = crypt.urlsafe_encrypt(self.metadata_encryption_key, loc['url'], 64) else: uri = loc['url'] ret.append((image['id'], loc['id'], uri)) return ret
def _format_image_to_db(self, image): locations = image.locations if CONF.metadata_encryption_key: key = CONF.metadata_encryption_key ld = [] for l in locations: url = crypt.urlsafe_encrypt(key, l['url']) ld.append({'url': url, 'metadata': l['metadata']}) locations = ld return { 'id': image.image_id, 'name': image.name, 'status': image.status, 'created_at': image.created_at, 'min_disk': image.min_disk, 'min_ram': image.min_ram, 'protected': image.protected, 'locations': locations, 'checksum': image.checksum, 'owner': image.owner, 'disk_format': image.disk_format, 'container_format': image.container_format, 'size': image.size, 'virtual_size': image.virtual_size, 'is_public': image.visibility == 'public', 'properties': dict(image.extra_properties), }
def encrypt_metadata(self, image_metadata): if self.metadata_encryption_key is not None: location_url = image_metadata.get("location") if location_url: location = crypt.urlsafe_encrypt(self.metadata_encryption_key, location_url, 64) image_metadata["location"] = location if image_metadata.get("location_data"): ld = [] for loc in image_metadata["location_data"]: if loc["url"] == location_url: url = location else: url = crypt.urlsafe_encrypt(self.metadata_encryption_key, loc["url"], 64) ld.append({"url": url, "metadata": loc["metadata"]}) image_metadata["location_data"] = ld return image_metadata
def _format_image_to_db(self, image): locations = image.locations if CONF.metadata_encryption_key: key = CONF.metadata_encryption_key ld = [] for loc in locations: url = crypt.urlsafe_encrypt(key, loc['url']) ld.append({'url': url, 'metadata': loc['metadata'], 'status': loc['status'], # NOTE(zhiyan): New location has no ID field. 'id': loc.get('id')}) locations = ld return { 'id': image.image_id, 'name': image.name, 'status': image.status, 'created_at': image.created_at, 'min_disk': image.min_disk, 'min_ram': image.min_ram, 'protected': image.protected, 'locations': locations, 'checksum': image.checksum, 'os_hash_algo': image.os_hash_algo, 'os_hash_value': image.os_hash_value, 'owner': image.owner, 'disk_format': image.disk_format, 'container_format': image.container_format, 'size': image.size, 'virtual_size': image.virtual_size, 'visibility': image.visibility, 'properties': dict(image.extra_properties), 'os_hidden': image.os_hidden }
def _format_image_to_db(self, image): locations = image.locations if CONF.metadata_encryption_key: key = CONF.metadata_encryption_key ld = [] for loc in locations: url = crypt.urlsafe_encrypt(key, loc["url"]) ld.append( { "url": url, "metadata": loc["metadata"], "status": loc["status"], # NOTE(zhiyan): New location has no ID field. "id": loc.get("id"), } ) locations = ld return { "id": image.image_id, "name": image.name, "status": image.status, "created_at": image.created_at, "min_disk": image.min_disk, "min_ram": image.min_ram, "protected": image.protected, "locations": locations, "checksum": image.checksum, "owner": image.owner, "disk_format": image.disk_format, "container_format": image.container_format, "size": image.size, "virtual_size": image.virtual_size, "is_public": image.visibility == "public", "properties": dict(image.extra_properties), }
def _format_image_to_db(self, image): locations = image.locations if CONF.metadata_encryption_key: key = CONF.metadata_encryption_key ld = [] for loc in locations: url = crypt.urlsafe_encrypt(key, loc['url']) ld.append({ 'url': url, 'metadata': loc['metadata'], 'status': loc['status'], # NOTE(zhiyan): New location has no ID field. 'id': loc.get('id') }) locations = ld return { 'id': image.image_id, 'name': image.name, 'status': image.status, 'created_at': image.created_at, 'min_disk': image.min_disk, 'min_ram': image.min_ram, 'protected': image.protected, 'locations': locations, 'checksum': image.checksum, 'owner': image.owner, 'disk_format': image.disk_format, 'container_format': image.container_format, 'size': image.size, 'virtual_size': image.virtual_size, 'visibility': image.visibility, 'properties': dict(image.extra_properties), }
def _prerun_017(self, engine): metadata_encryption_key = "a" * 16 self.config(metadata_encryption_key=metadata_encryption_key) images = get_table(engine, "images") unquoted = "swift://*****:*****@example.com/container/obj-id" encrypted_unquoted = crypt.urlsafe_encrypt(metadata_encryption_key, unquoted, 64) data = [] now = datetime.datetime.now() temp = dict( deleted=False, created_at=now, updated_at=now, status="active", is_public=True, min_disk=0, min_ram=0, location=encrypted_unquoted, id="fakeid1", ) images.insert().values(temp).execute() locations = ["file://ab", "file://abc", "swift://acct3A%foobar:[email protected]/container/obj-id2"] now = datetime.datetime.now() temp = dict( deleted=False, created_at=now, updated_at=now, status="active", is_public=True, min_disk=0, min_ram=0 ) for i, location in enumerate(locations): temp.update(location=location, id=uuidutils.generate_uuid()) data.append(temp) images.insert().values(temp).execute() return data
def encrypt_metadata(self, image_metadata): if (self.metadata_encryption_key is not None and 'location' in image_metadata and image_metadata['location'] is not None): location = crypt.urlsafe_encrypt(self.metadata_encryption_key, image_metadata['location'], 64) image_metadata['location'] = location return image_metadata
def encrypt_metadata(self, image_metadata): if (self.metadata_encryption_key is not None and 'location' in image_metadata.keys() and image_metadata['location'] is not None): location = crypt.urlsafe_encrypt(self.metadata_encryption_key, image_metadata['location'], 64) image_metadata['location'] = location return image_metadata
def encrypt_metadata(self, image_metadata): if ( self.metadata_encryption_key is not None and "location" in image_metadata and image_metadata["location"] is not None ): location = crypt.urlsafe_encrypt(self.metadata_encryption_key, image_metadata["location"], 64) image_metadata["location"] = location return image_metadata
def test_decrypt_locations_on_list(self): encrypted_locations = [crypt.urlsafe_encrypt(self.crypt_key, l) for l in ['ping', 'pong']] self.assertNotEqual(encrypted_locations, ['ping', 'pong']) db_data = _db_fixture(UUID1, owner=TENANT1, locations=encrypted_locations) self.db.image_create(None, db_data) image = self.image_repo.list()[0] self.assertEqual(image.locations, ['ping', 'pong'])
def encrypt_metadata(self, image_metadata): if self.metadata_encryption_key is not None: location_url = image_metadata.get('location') if location_url: location = crypt.urlsafe_encrypt(self.metadata_encryption_key, location_url, 64) image_metadata['location'] = location if image_metadata.get('location_data'): ld = [] for loc in image_metadata['location_data']: if loc['url'] == location_url: url = location else: url = crypt.urlsafe_encrypt( self.metadata_encryption_key, loc['url'], 64) ld.append({'url': url, 'metadata': loc['metadata']}) image_metadata['location_data'] = ld return image_metadata
def add_location(self, image_id, location, user_context=None): """Adding image location to scrub queue. :param image_id: The opaque image identifier :param location: The opaque image location :param user_context: The user's request context :retval A boolean value to indicate success or not """ if user_context is not None: registry_client = registry.get_registry_client(user_context) else: registry_client = self.registry with lockutils.lock("scrubber-%s" % image_id, lock_file_prefix='glance-', external=True): # NOTE(zhiyan): make sure scrubber does not cleanup # 'pending_delete' images concurrently before the code # get lock and reach here. try: image = registry_client.get_image(image_id) if image['status'] == 'deleted': return True except exception.NotFound as e: LOG.warn(_LW("Failed to find image to delete: %s"), utils.exception_to_str(e)) return False loc_id = location.get('id', '-') if self.metadata_encryption_key: uri = crypt.urlsafe_encrypt(self.metadata_encryption_key, location['url'], 64) else: uri = location['url'] delete_time = time.time() + self.scrub_time file_path = os.path.join(self.scrubber_datadir, str(image_id)) if os.path.exists(file_path): # Append the uri of location to the queue file with open(file_path, 'a') as f: f.write('\n') f.write('\n'.join( [str(loc_id), uri, str(int(delete_time))])) else: # NOTE(zhiyan): Protect the file before we write any data. open(file_path, 'w').close() os.chmod(file_path, 0o600) with open(file_path, 'w') as f: f.write('\n'.join( [str(loc_id), uri, str(int(delete_time))])) os.utime(file_path, (delete_time, delete_time)) return True
def add_location(self, image_id, location, user_context=None): """Adding image location to scrub queue. :param image_id: The opaque image identifier :param location: The opaque image location :param user_context: The user's request context :retval A boolean value to indicate success or not """ if user_context is not None: registry_client = registry.get_registry_client(user_context) else: registry_client = self.registry with lockutils.lock("scrubber-%s" % image_id, lock_file_prefix='glance-', external=True): # NOTE(zhiyan): make sure scrubber does not cleanup # 'pending_delete' images concurrently before the code # get lock and reach here. try: image = registry_client.get_image(image_id) if image['status'] == 'deleted': return True except exception.NotFound as e: LOG.warn(_LW("Failed to find image to delete: %s"), utils.exception_to_str(e)) return False loc_id = location.get('id', '-') if self.metadata_encryption_key: uri = crypt.urlsafe_encrypt(self.metadata_encryption_key, location['url'], 64) else: uri = location['url'] delete_time = time.time() + self.scrub_time file_path = os.path.join(self.scrubber_datadir, str(image_id)) if os.path.exists(file_path): # Append the uri of location to the queue file with open(file_path, 'a') as f: f.write('\n') f.write('\n'.join([str(loc_id), uri, str(int(delete_time))])) else: # NOTE(zhiyan): Protect the file before we write any data. open(file_path, 'w').close() os.chmod(file_path, 0o600) with open(file_path, 'w') as f: f.write('\n'.join([str(loc_id), uri, str(int(delete_time))])) os.utime(file_path, (delete_time, delete_time)) return True
def test_decrypt_locations_on_list(self): url_loc = ["ping", "pong"] orig_locations = [{"url": l, "metadata": {}} for l in url_loc] encrypted_locs = [crypt.urlsafe_encrypt(self.crypt_key, l) for l in url_loc] encrypted_locations = [{"url": l, "metadata": {}} for l in encrypted_locs] self.assertNotEqual(encrypted_locations, orig_locations) db_data = _db_fixture(UUID1, owner=TENANT1, locations=encrypted_locations) self.db.image_create(None, db_data) image = self.image_repo.list()[0] self.assertEqual(image.locations, orig_locations)
def _check_16_to_17(self, engine): """ Check that migrating swift location credentials to quoted form and back works. """ migration_api.version_control(version=0) migration_api.upgrade(16) conn = engine.connect() images_table = Table('images', MetaData(), autoload=True, autoload_with=engine) def get_locations(): conn = engine.connect() locations = [x[0] for x in conn.execute( select(['location'], from_obj=[images_table]))] conn.close() return locations unquoted = 'swift://*****:*****@example.com/container/obj-id' encrypted_unquoted = crypt.urlsafe_encrypt( self.metadata_encryption_key, unquoted, 64) quoted = 'swift://acct%3Ausr:[email protected]/container/obj-id' # Insert image with an unquoted image location now = datetime.datetime.now() kwargs = dict(deleted=False, created_at=now, updated_at=now, status='active', is_public=True, min_disk=0, min_ram=0) kwargs.update(location=encrypted_unquoted, id=1) conn.execute(images_table.insert(), [kwargs]) conn.close() migration_api.upgrade(17) actual_location = crypt.urlsafe_decrypt(self.metadata_encryption_key, get_locations()[0]) self.assertEqual(actual_location, quoted) migration_api.downgrade(16) actual_location = crypt.urlsafe_decrypt(self.metadata_encryption_key, get_locations()[0]) self.assertEqual(actual_location, unquoted)
def encrypt_metadata(self, image_metadata): if self.metadata_encryption_key: location_url = image_metadata.get('location') if location_url: location = crypt.urlsafe_encrypt(self.metadata_encryption_key, location_url, 64) image_metadata['location'] = location if image_metadata.get('location_data'): ld = [] for loc in image_metadata['location_data']: if loc['url'] == location_url: url = location else: url = crypt.urlsafe_encrypt( self.metadata_encryption_key, loc['url'], 64) ld.append({'url': url, 'metadata': loc['metadata'], 'status': loc['status'], # NOTE(zhiyan): New location has no ID field. 'id': loc.get('id')}) image_metadata['location_data'] = ld return image_metadata
def test_decrypt_locations_on_get(self): url_loc = ['ping', 'pong'] orig_locations = [{'url': l, 'metadata': {}} for l in url_loc] encrypted_locs = [crypt.urlsafe_encrypt(self.crypt_key, l) for l in url_loc] encrypted_locations = [{'url': l, 'metadata': {}} for l in encrypted_locs] self.assertNotEqual(encrypted_locations, orig_locations) db_data = _db_fixture(UUID1, owner=TENANT1, locations=encrypted_locations) self.db.image_create(None, db_data) image = self.image_repo.get(UUID1) self.assertEqual(image.locations, orig_locations)
def encrypt_metadata(self, image_metadata): if self.metadata_encryption_key is not None: location_url = image_metadata.get('location') if location_url: location = crypt.urlsafe_encrypt(self.metadata_encryption_key, location_url, 64) image_metadata['location'] = location if image_metadata.get('location_data'): ld = [] for loc in image_metadata['location_data']: if loc['url'] == location_url: url = location else: url = crypt.urlsafe_encrypt( self.metadata_encryption_key, loc['url'], 64) ld.append({'url': url, 'metadata': loc['metadata'], 'status': loc['status'], # NOTE(zhiyan): New location has no ID field. 'id': loc.get('id')}) image_metadata['location_data'] = ld return image_metadata
def test_decrypt_locations_on_get(self): url_loc = ["ping", "pong"] orig_locations = [{"url": l, "metadata": {}, "status": "active"} for l in url_loc] encrypted_locs = [crypt.urlsafe_encrypt(self.crypt_key, l) for l in url_loc] encrypted_locations = [{"url": l, "metadata": {}, "status": "active"} for l in encrypted_locs] self.assertNotEqual(encrypted_locations, orig_locations) db_data = _db_fixture(UUID1, owner=TENANT1, locations=encrypted_locations) self.db.image_create(None, db_data) image = self.image_repo.get(UUID1) self.assertIn("id", image.locations[0]) self.assertIn("id", image.locations[1]) image.locations[0].pop("id") image.locations[1].pop("id") self.assertEqual(orig_locations, image.locations)
def test_encryption(self): # Check that original plaintext and unencrypted ciphertext match # Check keys of the three allowed lengths key_list = ["1234567890abcdef", "12345678901234567890abcd", "1234567890abcdef1234567890ABCDEF"] plaintext_list = [""] blocksize = 64 for i in range(3 * blocksize): plaintext_list.append(os.urandom(i)) for key in key_list: for plaintext in plaintext_list: ciphertext = crypt.urlsafe_encrypt(key, plaintext, blocksize) self.assertTrue(ciphertext != plaintext) text = crypt.urlsafe_decrypt(key, ciphertext) self.assertTrue(plaintext == text)
def encrypt_metadata(self, image_metadata): if self.metadata_encryption_key: location_url = image_metadata.get("location") if location_url: location = crypt.urlsafe_encrypt(self.metadata_encryption_key, location_url, 64) image_metadata["location"] = location if image_metadata.get("location_data"): ld = [] for loc in image_metadata["location_data"]: if loc["url"] == location_url: url = location else: url = crypt.urlsafe_encrypt(self.metadata_encryption_key, loc["url"], 64) ld.append( { "url": url, "metadata": loc["metadata"], "status": loc["status"], # NOTE(zhiyan): New location has no ID field. "id": loc.get("id"), } ) image_metadata["location_data"] = ld return image_metadata
def schedule_delayed_delete_from_backend(uri, image_id, **kwargs): """Given a uri, schedule the deletion of an image.""" datadir = CONF.scrubber_datadir delete_time = time.time() + CONF.scrub_time file_path = os.path.join(datadir, str(image_id)) utils.safe_mkdirs(datadir) if os.path.exists(file_path): msg = _("Image id %(image_id)s already queued for delete") % {"image_id": image_id} raise exception.Duplicate(msg) if CONF.metadata_encryption_key is not None: uri = crypt.urlsafe_encrypt(CONF.metadata_encryption_key, uri, 64) with open(file_path, "w") as f: f.write("\n".join([uri, str(int(delete_time))])) os.chmod(file_path, 0o600) os.utime(file_path, (delete_time, delete_time))
def test_decrypt_locations_on_get(self): url_loc = ['ping', 'pong'] orig_locations = [{'url': l, 'metadata': {}} for l in url_loc] encrypted_locs = [ crypt.urlsafe_encrypt(self.crypt_key, l) for l in url_loc ] encrypted_locations = [{ 'url': l, 'metadata': {} } for l in encrypted_locs] self.assertNotEqual(encrypted_locations, orig_locations) db_data = _db_fixture(UUID1, owner=TENANT1, locations=encrypted_locations) self.db.image_create(None, db_data) image = self.image_repo.get(UUID1) self.assertEqual(image.locations, orig_locations)
def test_encryption(self): # Check that original plaintext and unencrypted ciphertext match # Check keys of the three allowed lengths key_list = ["1234567890abcdef", "12345678901234567890abcd", "1234567890abcdef1234567890ABCDEF"] plaintext_list = [''] blocksize = 64 for i in range(3 * blocksize): plaintext_list.append(os.urandom(i)) for key in key_list: for plaintext in plaintext_list: ciphertext = crypt.urlsafe_encrypt(key, plaintext, blocksize) self.assertTrue(ciphertext != plaintext) text = crypt.urlsafe_decrypt(key, ciphertext) self.assertTrue(plaintext == text)
def _walk_all_locations(self, remove=False): """Returns a list of image id and location tuple from scrub queue. :param remove: Whether remove location from queue or not after walk :retval a list of image id, location id and uri tuple from scrub queue """ filters = { 'deleted': True, 'is_public': 'none', 'status': 'pending_delete' } ret = [] for image in self.registry.get_images_detailed(filters=filters): deleted_at = image.get('deleted_at') if not deleted_at: continue # NOTE: Strip off microseconds which may occur after the last '.,' # Example: 2012-07-07T19:14:34.974216 date_str = deleted_at.rsplit('.', 1)[0].rsplit(',', 1)[0] delete_time = calendar.timegm( time.strptime(date_str, "%Y-%m-%dT%H:%M:%S")) if delete_time + self.scrub_time > time.time(): continue for loc in image['location_data']: if loc['status'] != 'pending_delete': continue if self.metadata_encryption_key: uri = crypt.urlsafe_encrypt(self.metadata_encryption_key, loc['url'], 64) else: uri = loc['url'] ret.append((image['id'], loc['id'], uri)) if remove: db_api.get_api().image_location_delete( self.admin_context, image['id'], loc['id'], 'deleted') self.registry.update_image(image['id'], {'status': 'deleted'}) return ret
def _walk_all_locations(self, remove=False): """Returns a list of image id and location tuple from scrub queue. :param remove: Whether remove location from queue or not after walk :retval a list of image id, location id and uri tuple from scrub queue """ filters = {'deleted': True, 'is_public': 'none', 'status': 'pending_delete'} ret = [] for image in self.registry.get_images_detailed(filters=filters): deleted_at = image.get('deleted_at') if not deleted_at: continue # NOTE: Strip off microseconds which may occur after the last '.,' # Example: 2012-07-07T19:14:34.974216 date_str = deleted_at.rsplit('.', 1)[0].rsplit(',', 1)[0] delete_time = calendar.timegm(time.strptime(date_str, "%Y-%m-%dT%H:%M:%S")) if delete_time + self.scrub_time > time.time(): continue for loc in image['location_data']: if loc['status'] != 'pending_delete': continue if self.metadata_encryption_key: uri = crypt.urlsafe_encrypt(self.metadata_encryption_key, loc['url'], 64) else: uri = loc['url'] ret.append((image['id'], loc['id'], uri)) if remove: db_api.get_api().image_location_delete(self.admin_context, image['id'], loc['id'], 'deleted') self.registry.update_image(image['id'], {'status': 'deleted'}) return ret
def schedule_delayed_delete_from_backend(uri, image_id, **kwargs): """Given a uri, schedule the deletion of an image.""" datadir = CONF.scrubber_datadir delete_time = time.time() + CONF.scrub_time file_path = os.path.join(datadir, str(image_id)) utils.safe_mkdirs(datadir) if os.path.exists(file_path): msg = _("Image id %(image_id)s already queued for delete") % { 'image_id': image_id} raise exception.Duplicate(msg) if CONF.metadata_encryption_key is not None: uri = crypt.urlsafe_encrypt(CONF.metadata_encryption_key, uri, 64) with open(file_path, 'w') as f: f.write('\n'.join([uri, str(int(delete_time))])) os.chmod(file_path, 0600) os.utime(file_path, (delete_time, delete_time))
def test_decrypt_locations_on_list(self): url_loc = ['ping', 'pong'] orig_locations = [{'url': l, 'metadata': {}, 'status': 'active'} for l in url_loc] encrypted_locs = [crypt.urlsafe_encrypt(self.crypt_key, l) for l in url_loc] encrypted_locations = [{'url': l, 'metadata': {}, 'status': 'active'} for l in encrypted_locs] self.assertNotEqual(encrypted_locations, orig_locations) db_data = _db_fixture(UUID1, owner=TENANT1, locations=encrypted_locations) self.db.image_create(None, db_data) image = self.image_repo.list()[0] self.assertIn('id', image.locations[0]) self.assertIn('id', image.locations[1]) image.locations[0].pop('id') image.locations[1].pop('id') self.assertEqual(orig_locations, image.locations)
def add_location(self, image_id, uri): """Adding image location to scrub queue. :param image_id: The opaque image identifier :param uri: The opaque image location uri """ with lockutils.lock("scrubber-%s" % image_id, lock_file_prefix='glance-', external=True): # NOTE(zhiyan): make sure scrubber does not cleanup # 'pending_delete' images concurrently before the code # get lock and reach here. try: image = self.registry.get_image(image_id) if image['status'] == 'deleted': return except exception.NotFound as e: LOG.error( _("Failed to find image to delete: " "%(e)s") % locals()) return delete_time = time.time() + self.scrub_time file_path = os.path.join(self.scrubber_datadir, str(image_id)) if self.metadata_encryption_key is not None: uri = crypt.urlsafe_encrypt(self.metadata_encryption_key, uri, 64) if os.path.exists(file_path): # Append the uri of location to the queue file with open(file_path, 'a') as f: f.write('\n') f.write('\n'.join([uri, str(int(delete_time))])) else: # NOTE(zhiyan): Protect the file before we write any data. open(file_path, 'w').close() os.chmod(file_path, 0o600) with open(file_path, 'w') as f: f.write('\n'.join([uri, str(int(delete_time))])) os.utime(file_path, (delete_time, delete_time))
def _pre_upgrade_017(self, engine): metadata_encryption_key = 'a' * 16 CONF.set_override('metadata_encryption_key', metadata_encryption_key) self.addCleanup(CONF.reset) images = db_utils.get_table(engine, 'images') unquoted = 'swift://*****:*****@example.com/container/obj-id' encrypted_unquoted = crypt.urlsafe_encrypt( metadata_encryption_key, unquoted, 64) data = [] now = datetime.datetime.now() temp = dict(deleted=False, created_at=now, updated_at=now, status='active', is_public=True, min_disk=0, min_ram=0, location=encrypted_unquoted, id='fakeid1') images.insert().values(temp).execute() locations = [ 'file://ab', 'file://abc', 'swift://acct3A%foobar:[email protected]/container/obj-id2' ] now = datetime.datetime.now() temp = dict(deleted=False, created_at=now, updated_at=now, status='active', is_public=True, min_disk=0, min_ram=0) for i, location in enumerate(locations): temp.update(location=location, id=str(uuid.uuid4())) data.append(temp) images.insert().values(temp).execute() return data
def add_location(self, image_id, uri): """Adding image location to scrub queue. :param image_id: The opaque image identifier :param uri: The opaque image location uri """ with lockutils.lock("scrubber-%s" % image_id, lock_file_prefix='glance-', external=True): # NOTE(zhiyan): make sure scrubber does not cleanup # 'pending_delete' images concurrently before the code # get lock and reach here. try: image = self.registry.get_image(image_id) if image['status'] == 'deleted': return except exception.NotFound as e: LOG.error(_("Failed to find image to delete: " "%(e)s") % locals()) return delete_time = time.time() + self.scrub_time file_path = os.path.join(self.scrubber_datadir, str(image_id)) if self.metadata_encryption_key is not None: uri = crypt.urlsafe_encrypt(self.metadata_encryption_key, uri, 64) if os.path.exists(file_path): # Append the uri of location to the queue file with open(file_path, 'a') as f: f.write('\n') f.write('\n'.join([uri, str(int(delete_time))])) else: # NOTE(zhiyan): Protect the file before we write any data. open(file_path, 'w').close() os.chmod(file_path, 0o600) with open(file_path, 'w') as f: f.write('\n'.join([uri, str(int(delete_time))])) os.utime(file_path, (delete_time, delete_time))
def _format_image_to_db(self, image): locations = image.locations if CONF.metadata_encryption_key: key = CONF.metadata_encryption_key locations = [crypt.urlsafe_encrypt(key, l) for l in locations] return { "id": image.image_id, "name": image.name, "status": image.status, "created_at": image.created_at, "min_disk": image.min_disk, "min_ram": image.min_ram, "protected": image.protected, "locations": locations, "checksum": image.checksum, "owner": image.owner, "disk_format": image.disk_format, "container_format": image.container_format, "size": image.size, "is_public": image.visibility == "public", "properties": dict(image.extra_properties), }
def _walk_all_locations(self, remove=False): """Returns a list of image id and location tuple from scrub queue. :param remove: Whether remove location from queue or not after walk :retval a list of image id, location id and uri tuple from scrub queue """ filters = {"deleted": True, "is_public": "none", "status": "pending_delete"} ret = [] for image in self.registry.get_images_detailed(filters=filters): deleted_at = image.get("deleted_at") if not deleted_at: continue # NOTE: Strip off microseconds which may occur after the last '.,' # Example: 2012-07-07T19:14:34.974216 date_str = deleted_at.rsplit(".", 1)[0].rsplit(",", 1)[0] delete_time = calendar.timegm(time.strptime(date_str, "%Y-%m-%dT%H:%M:%S")) if delete_time + self.scrub_time > time.time(): continue for loc in image["location_data"]: if loc["status"] != "pending_delete": continue if self.metadata_encryption_key: uri = crypt.urlsafe_encrypt(self.metadata_encryption_key, loc["url"], 64) else: uri = loc["url"] ret.append((image["id"], loc["id"], uri)) if remove: db_api.get_api().image_location_delete(self.admin_context, image["id"], loc["id"], "deleted") self.registry.update_image(image["id"], {"status": "deleted"}) return ret
def encrypt_location(uri): return crypt.urlsafe_encrypt(CONF.metadata_encryption_key, uri, 64)