def decrypt_metadata(self, image_metadata): if self.metadata_encryption_key: if image_metadata.get("location"): location = crypt.urlsafe_decrypt(self.metadata_encryption_key, image_metadata["location"]) image_metadata["location"] = location if image_metadata.get("location_data"): ld = [] for loc in image_metadata["location_data"]: url = crypt.urlsafe_decrypt(self.metadata_encryption_key, loc["url"]) ld.append({"id": loc["id"], "url": url, "metadata": loc["metadata"], "status": loc["status"]}) image_metadata["location_data"] = ld return image_metadata
def _check_16_to_17(self, engine): """ Check that migrating swift location credentials to quoted form and back works. """ migration_api.version_control(version=0) migration_api.upgrade(16) conn = engine.connect() images_table = Table('images', MetaData(), autoload=True, autoload_with=engine) def get_locations(): conn = engine.connect() locations = [x[0] for x in conn.execute( select(['location'], from_obj=[images_table]))] conn.close() return locations unquoted = 'swift://*****:*****@example.com/container/obj-id' encrypted_unquoted = crypt.urlsafe_encrypt( self.metadata_encryption_key, unquoted, 64) quoted = 'swift://acct%3Ausr:[email protected]/container/obj-id' # Insert image with an unquoted image location now = datetime.datetime.now() kwargs = dict(deleted=False, created_at=now, updated_at=now, status='active', is_public=True, min_disk=0, min_ram=0) kwargs.update(location=encrypted_unquoted, id=1) conn.execute(images_table.insert(), [kwargs]) conn.close() migration_api.upgrade(17) actual_location = crypt.urlsafe_decrypt(self.metadata_encryption_key, get_locations()[0]) self.assertEqual(actual_location, quoted) migration_api.downgrade(16) actual_location = crypt.urlsafe_decrypt(self.metadata_encryption_key, get_locations()[0]) self.assertEqual(actual_location, unquoted)
def decrypt_metadata(self, image_metadata): if self.metadata_encryption_key is not None: if image_metadata.get('location'): location = crypt.urlsafe_decrypt(self.metadata_encryption_key, image_metadata['location']) image_metadata['location'] = location if image_metadata.get('location_data'): ld = [] for loc in image_metadata['location_data']: url = crypt.urlsafe_decrypt(self.metadata_encryption_key, loc['url']) ld.append({'url': url, 'metadata': loc['metadata']}) image_metadata['location_data'] = ld return image_metadata
def _format_image_from_db(self, db_image, db_tags): visibility = 'public' if db_image['is_public'] else 'private' properties = {} for prop in db_image.pop('properties'): # NOTE(markwash) db api requires us to filter deleted if not prop['deleted']: properties[prop['name']] = prop['value'] locations = db_image['locations'] if CONF.metadata_encryption_key: key = CONF.metadata_encryption_key ld = [] for l in locations: url = crypt.urlsafe_decrypt(key, l['url']) ld.append({'url': url, 'metadata': l['metadata']}) locations = ld return glance.domain.Image( image_id=db_image['id'], name=db_image['name'], status=db_image['status'], created_at=db_image['created_at'], updated_at=db_image['updated_at'], visibility=visibility, min_disk=db_image['min_disk'], min_ram=db_image['min_ram'], protected=db_image['protected'], locations=location_strategy.get_ordered_locations(locations), checksum=db_image['checksum'], owner=db_image['owner'], disk_format=db_image['disk_format'], container_format=db_image['container_format'], size=db_image['size'], virtual_size=db_image['virtual_size'], extra_properties=properties, tags=db_tags )
def _check_017(self, engine, data): metadata_encryption_key = 'a' * 16 quoted = 'swift://acct%3Ausr:[email protected]/container/obj-id' images = get_table(engine, 'images') result = images.select().execute() locations = map(lambda x: x['location'], result) actual_location = [] for location in locations: if location: try: temp_loc = crypt.urlsafe_decrypt(metadata_encryption_key, location) actual_location.append(temp_loc) except TypeError: actual_location.append(location) except ValueError: actual_location.append(location) self.assertIn(quoted, actual_location) loc_list = ['file://ab', 'file://abc', 'swift://acct3A%foobar:[email protected]/container/obj-id2'] for location in loc_list: if not location in actual_location: self.fail(_("location: %s data lost") % location)
def _format_image_from_db(self, db_image, db_tags): properties = {} for prop in db_image.pop('properties'): # NOTE(markwash) db api requires us to filter deleted if not prop['deleted']: properties[prop['name']] = prop['value'] locations = [loc for loc in db_image['locations'] if loc['status'] == 'active'] if CONF.metadata_encryption_key: key = CONF.metadata_encryption_key for l in locations: l['url'] = crypt.urlsafe_decrypt(key, l['url']) return glance.domain.Image( image_id=db_image['id'], name=db_image['name'], status=db_image['status'], created_at=db_image['created_at'], updated_at=db_image['updated_at'], visibility=db_image['visibility'], min_disk=db_image['min_disk'], min_ram=db_image['min_ram'], protected=db_image['protected'], locations=location_strategy.get_ordered_locations(locations), checksum=db_image['checksum'], os_hash_algo=db_image['os_hash_algo'], os_hash_value=db_image['os_hash_value'], owner=db_image['owner'], disk_format=db_image['disk_format'], container_format=db_image['container_format'], size=db_image['size'], virtual_size=db_image['virtual_size'], extra_properties=properties, tags=db_tags, os_hidden=db_image['os_hidden'], )
def _format_image_from_db(self, db_image, db_tags): visibility = "public" if db_image["is_public"] else "private" properties = {} for prop in db_image.pop("properties"): # NOTE(markwash) db api requires us to filter deleted if not prop["deleted"]: properties[prop["name"]] = prop["value"] locations = db_image["locations"] if CONF.metadata_encryption_key: key = CONF.metadata_encryption_key locations = [crypt.urlsafe_decrypt(key, l) for l in locations] return glance.domain.Image( image_id=db_image["id"], name=db_image["name"], status=db_image["status"], created_at=db_image["created_at"], updated_at=db_image["updated_at"], visibility=visibility, min_disk=db_image["min_disk"], min_ram=db_image["min_ram"], protected=db_image["protected"], locations=locations, checksum=db_image["checksum"], owner=db_image["owner"], disk_format=db_image["disk_format"], container_format=db_image["container_format"], size=db_image["size"], extra_properties=properties, tags=db_tags, )
def _format_image_from_db(self, db_image, db_tags): visibility = 'public' if db_image['is_public'] else 'private' properties = {} for prop in db_image.pop('properties'): # NOTE(markwash) db api requires us to filter deleted if not prop['deleted']: properties[prop['name']] = prop['value'] locations = db_image['locations'] if CONF.metadata_encryption_key: key = CONF.metadata_encryption_key ld = [] for l in locations: url = crypt.urlsafe_decrypt(key, l['url']) ld.append({'url': url, 'metadata': l['metadata']}) locations = ld return glance.domain.Image( image_id=db_image['id'], name=db_image['name'], status=db_image['status'], created_at=db_image['created_at'], updated_at=db_image['updated_at'], visibility=visibility, min_disk=db_image['min_disk'], min_ram=db_image['min_ram'], protected=db_image['protected'], locations=location_strategy.get_ordered_locations(locations), checksum=db_image['checksum'], owner=db_image['owner'], disk_format=db_image['disk_format'], container_format=db_image['container_format'], size=db_image['size'], virtual_size=db_image['virtual_size'], extra_properties=properties, tags=db_tags)
def test_encryption(self): # Check that original plaintext and unencrypted ciphertext match # Check keys of the three allowed lengths key_list = ["1234567890abcdef", "12345678901234567890abcd", "1234567890abcdef1234567890ABCDEF"] plaintext_list = [''] blocksize = 64 for i in range(3 * blocksize): text = os.urandom(i) if six.PY3: text = text.decode('latin1') plaintext_list.append(text) for key in key_list: for plaintext in plaintext_list: ciphertext = crypt.urlsafe_encrypt(key, plaintext, blocksize) self.assertIsInstance(ciphertext, bytes) if six.PY3: self.assertNotEqual(ciphertext, plaintext.encode('utf-8')) else: self.assertNotEqual(ciphertext, plaintext) text = crypt.urlsafe_decrypt(key, ciphertext) self.assertIsInstance(text, str) self.assertEqual(plaintext, text)
def _format_image_from_db(self, db_image, db_tags): properties = {} for prop in db_image.pop('properties'): # NOTE(markwash) db api requires us to filter deleted if not prop['deleted']: properties[prop['name']] = prop['value'] locations = [ loc for loc in db_image['locations'] if loc['status'] == 'active' ] if CONF.metadata_encryption_key: key = CONF.metadata_encryption_key for l in locations: l['url'] = crypt.urlsafe_decrypt(key, l['url']) #利用数据库中查询到的数据,填充Image对象 return glance.domain.Image( image_id=db_image['id'], name=db_image['name'], status=db_image['status'], created_at=db_image['created_at'], updated_at=db_image['updated_at'], visibility=db_image['visibility'], min_disk=db_image['min_disk'], min_ram=db_image['min_ram'], protected=db_image['protected'], locations=location_strategy.get_ordered_locations(locations), checksum=db_image['checksum'], owner=db_image['owner'], disk_format=db_image['disk_format'], container_format=db_image['container_format'], size=db_image['size'], virtual_size=db_image['virtual_size'], extra_properties=properties, tags=db_tags)
def test_scrubber_with_metadata_enc(self): """ test that files written to scrubber_data_dir use metadata_encryption_key when available to encrypt the location """ config_path = os.environ.get("GLANCE_TEST_SWIFT_CONF") if not config_path: msg = "GLANCE_TEST_SWIFT_CONF environ not set." self.skipTest(msg) raw_config = read_config(config_path) swift_config = parse_config(raw_config) self.cleanup() self.start_servers(delayed_delete=True, daemon=True, default_store="swift", **swift_config) # add an image headers = { "x-image-meta-name": "test_image", "x-image-meta-is_public": "true", "x-image-meta-disk_format": "raw", "x-image-meta-container_format": "ovf", "content-type": "application/octet-stream", } path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port) http = httplib2.Http() response, content = http.request(path, "POST", body="XXX", headers=headers) self.assertEqual(response.status, 201) image = json.loads(content)["image"] self.assertEqual("active", image["status"]) image_id = image["id"] # delete the image path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, "DELETE") self.assertEqual(response.status, 200) response, content = http.request(path, "HEAD") self.assertEqual(response.status, 200) self.assertEqual("pending_delete", response["x-image-meta-status"]) # ensure the marker file has encrypted the image location by decrypting # it and checking the image_id is intact file_path = os.path.join(self.api_server.scrubber_datadir, str(image_id)) marker_uri = None with open(file_path, "r") as f: marker_uri = f.readline().strip() self.assertTrue(marker_uri is not None) decrypted_uri = crypt.urlsafe_decrypt(self.api_server.metadata_encryption_key, marker_uri) loc = StoreLocation({}) loc.parse_uri(decrypted_uri) self.assertIn(loc.scheme, ("swift+http", "swift+https")) self.assertEqual(image["id"], loc.obj) self.wait_for_scrub(path) self.stop_servers()
def _delete_image_location_from_backend(self, image_id, loc_id, uri): if CONF.metadata_encryption_key: uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri) try: LOG.debug("Scrubbing image %s from a location.", image_id) try: self.store_api.delete_from_backend(uri, self.admin_context) except store_exceptions.NotFound: LOG.info( _LI("Image location for image '%s' not found in " "backend; Marking image location deleted in " "db."), image_id) if loc_id != '-': db_api.get_api().image_location_delete(self.admin_context, image_id, int(loc_id), 'deleted') LOG.info(_LI("Image %s is scrubbed from a location."), image_id) except Exception as e: LOG.error( _LE("Unable to scrub image %(id)s from a location. " "Reason: %(exc)s ") % { 'id': image_id, 'exc': encodeutils.exception_to_unicode(e) }) raise
def get_all_locations(self): """Returns a list of image id and location tuple from scrub queue. :returns: a list of image id, location id and uri tuple from scrub queue """ ret = [] for image in self._get_all_images(): deleted_at = image.get('deleted_at') if not deleted_at: continue # NOTE: Strip off microseconds which may occur after the last '.,' # Example: 2012-07-07T19:14:34.974216 deleted_at = timeutils.isotime(deleted_at) date_str = deleted_at.rsplit('.', 1)[0].rsplit(',', 1)[0] delete_time = calendar.timegm( time.strptime(date_str, "%Y-%m-%dT%H:%M:%SZ")) if delete_time + self.scrub_time > time.time(): continue for loc in image['locations']: if loc['status'] != 'pending_delete': continue if self.metadata_encryption_key: uri = crypt.urlsafe_decrypt(self.metadata_encryption_key, loc['url']) else: uri = loc['url'] ret.append((image['id'], loc['id'], uri)) return ret
def decrypt_metadata(self, image_metadata): if (self.metadata_encryption_key is not None and 'location' in image_metadata.keys() and image_metadata['location'] is not None): location = crypt.urlsafe_decrypt(self.metadata_encryption_key, image_metadata['location']) image_metadata['location'] = location return image_metadata
def test_encrypt_locations_on_add(self): image = self.image_factory.new_image(UUID1) image.locations = self.foo_bar_location self.image_repo.add(image) db_data = self.db.image_get(self.context, UUID1) self.assertNotEqual(db_data["locations"], ["foo", "bar"]) decrypted_locations = [crypt.urlsafe_decrypt(self.crypt_key, l["url"]) for l in db_data["locations"]] self.assertEqual(decrypted_locations, [l["url"] for l in self.foo_bar_location])
def decrypt_metadata(self, image_metadata): if (self.metadata_encryption_key is not None and 'location' in image_metadata and image_metadata['location'] is not None): location = crypt.urlsafe_decrypt(self.metadata_encryption_key, image_metadata['location']) image_metadata['location'] = location return image_metadata
def test_encrypt_locations_on_add(self): image = self.image_factory.new_image(UUID1) image.locations = ['foo', 'bar'] self.image_repo.add(image) db_data = self.db.image_get(self.context, UUID1) self.assertNotEqual(db_data['locations'], ['foo', 'bar']) decrypted_locations = [crypt.urlsafe_decrypt(self.crypt_key, l) for l in db_data['locations']] self.assertEqual(decrypted_locations, ['foo', 'bar'])
def decrypt_metadata(self, image_metadata): if ( self.metadata_encryption_key is not None and "location" in image_metadata and image_metadata["location"] is not None ): location = crypt.urlsafe_decrypt(self.metadata_encryption_key, image_metadata["location"]) image_metadata["location"] = location return image_metadata
def test_encrypt_locations_on_save(self): image = self.image_factory.new_image(UUID1) self.image_repo.add(image) image.locations = self.foo_bar_location self.image_repo.save(image) db_data = self.db.image_get(self.context, UUID1) self.assertNotEqual(db_data['locations'], ['foo', 'bar']) decrypted_locations = [crypt.urlsafe_decrypt(self.crypt_key, l['url']) for l in db_data['locations']] self.assertEqual([l['url'] for l in self.foo_bar_location], decrypted_locations)
def _delete_image_location_from_backend(self, image_id, loc_id, uri): if CONF.metadata_encryption_key: uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri) try: LOG.debug("Deleting URI from image %s." % image_id) self.store_api.delete_from_backend(self.admin_context, uri) if loc_id != "-": db_api.get_api().image_location_delete(self.admin_context, image_id, int(loc_id), "deleted") LOG.info(_LI("Image %s has been deleted.") % image_id) except Exception: LOG.warn(_LW("Unable to delete URI from image %s.") % image_id)
def test_scrubber_with_metadata_enc(self): """ test that files written to scrubber_data_dir use metadata_encryption_key when available to encrypt the location """ # FIXME(flaper87): It looks like an older commit # may have broken this test. The file_queue `add_location` # is not being called. self.skipTest("Test broken. See bug #1366682") self.cleanup() self.start_servers(delayed_delete=True, daemon=True, default_store='file') # add an image path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port) response, content = self._send_http_request(path, 'POST', body='XXX') self.assertEqual(201, response.status) image = jsonutils.loads(content)['image'] self.assertEqual('active', image['status']) # delete the image path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port, image['id']) response, content = self._send_http_request(path, 'DELETE') self.assertEqual(200, response.status) response, content = self._send_http_request(path, 'HEAD') self.assertEqual(200, response.status) self.assertEqual('pending_delete', response['x-image-meta-status']) # ensure the marker file has encrypted the image location by decrypting # it and checking the image_id is intact file_path = os.path.join(self.api_server.scrubber_datadir, image['id']) marker_uri = None with open(file_path, 'r') as f: marker_uri = f.readline().strip() self.assertTrue(marker_uri is not None) decrypted_uri = crypt.urlsafe_decrypt( self.api_server.metadata_encryption_key, marker_uri) loc = glance_store.location.StoreLocation({}) loc.parse_uri(decrypted_uri) self.assertEqual("file", loc.scheme) self.assertEqual(image['id'], loc.obj) self.wait_for_scrub(path) self.stop_servers()
def _delete_image_location_from_backend(self, image_id, loc_id, uri): if CONF.metadata_encryption_key: uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri) try: LOG.debug("Deleting URI from image %s." % image_id) self.store_api.delete_from_backend(uri, self.admin_context) if loc_id != '-': db_api.get_api().image_location_delete(self.admin_context, image_id, int(loc_id), 'deleted') LOG.info(_LI("Image %s has been deleted.") % image_id) except Exception: LOG.warn(_LW("Unable to delete URI from image %s.") % image_id)
def _format_image_from_db(self, db_image, db_tags): properties = {} for prop in db_image.pop('properties'): # NOTE(markwash) db api requires us to filter deleted if not prop['deleted']: properties[prop['name']] = prop['value'] locations = [ loc for loc in db_image['locations'] if loc['status'] == 'active' ] if CONF.metadata_encryption_key: key = CONF.metadata_encryption_key for l in locations: l['url'] = crypt.urlsafe_decrypt(key, l['url']) # NOTE(danms): If the image is shared and we are not the # owner, we must have found it because we are a member. Set # our tenant on the image as 'member' for policy checks in the # upper layers. For any other image stage, we found the image # some other way, so leave member=None. if (db_image['visibility'] == 'shared' and self.context.owner != db_image['owner']): member = self.context.owner else: member = None return glance.domain.Image( image_id=db_image['id'], name=db_image['name'], status=db_image['status'], created_at=db_image['created_at'], updated_at=db_image['updated_at'], visibility=db_image['visibility'], min_disk=db_image['min_disk'], min_ram=db_image['min_ram'], protected=db_image['protected'], locations=location_strategy.get_ordered_locations(locations), checksum=db_image['checksum'], os_hash_algo=db_image['os_hash_algo'], os_hash_value=db_image['os_hash_value'], owner=db_image['owner'], disk_format=db_image['disk_format'], container_format=db_image['container_format'], size=db_image['size'], virtual_size=db_image['virtual_size'], extra_properties=properties, tags=db_tags, os_hidden=db_image['os_hidden'], member=member, )
def test_encryption(self): # Check that original plaintext and unencrypted ciphertext match # Check keys of the three allowed lengths key_list = ["1234567890abcdef", "12345678901234567890abcd", "1234567890abcdef1234567890ABCDEF"] plaintext_list = [""] blocksize = 64 for i in range(3 * blocksize): plaintext_list.append(os.urandom(i)) for key in key_list: for plaintext in plaintext_list: ciphertext = crypt.urlsafe_encrypt(key, plaintext, blocksize) self.assertTrue(ciphertext != plaintext) text = crypt.urlsafe_decrypt(key, ciphertext) self.assertTrue(plaintext == text)
def test_encryption(self): # Check that original plaintext and unencrypted ciphertext match # Check keys of the three allowed lengths key_list = ["1234567890abcdef", "12345678901234567890abcd", "1234567890abcdef1234567890ABCDEF"] plaintext_list = [''] blocksize = 64 for i in range(3 * blocksize): plaintext_list.append(os.urandom(i)) for key in key_list: for plaintext in plaintext_list: ciphertext = crypt.urlsafe_encrypt(key, plaintext, blocksize) self.assertTrue(ciphertext != plaintext) text = crypt.urlsafe_decrypt(key, ciphertext) self.assertTrue(plaintext == text)
def _delete_image_from_backend(self, image_id, uri): if CONF.metadata_encryption_key is not None: uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri) try: LOG.debug(_("Deleting URI from image %(image_id)s.") % {"image_id": image_id}) # Here we create a request context with credentials to support # delayed delete when using multi-tenant backend storage admin_tenant = CONF.admin_tenant_name auth_token = self.registry.auth_tok admin_context = context.RequestContext(user=CONF.admin_user, tenant=admin_tenant, auth_tok=auth_token) self.store_api.delete_from_backend(admin_context, uri) except Exception: msg = _("Failed to delete URI from image %(image_id)s") LOG.error(msg % {"image_id": image_id})
def _delete(self, id, uri, now): file_path = os.path.join(self.datadir, str(id)) if CONF.metadata_encryption_key is not None: uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri) try: LOG.debug(_("Deleting %(uri)s") % {'uri': uri}) # Here we create a request context with credentials to support # delayed delete when using multi-tenant backend storage ctx = context.RequestContext(auth_tok=self.registry.auth_tok, user=self.admin_user, tenant=self.admin_tenant) store.delete_from_backend(ctx, uri) except store.UnsupportedBackend: msg = _("Failed to delete image from store (%(uri)s).") LOG.error(msg % {'uri': uri}) write_queue_file(file_path, uri, now) self.registry.update_image(id, {'status': 'deleted'}) utils.safe_remove(file_path)
def _delete_image_from_backend(self, image_id, uri): if CONF.metadata_encryption_key is not None: uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri) try: LOG.debug("Deleting URI from image %(image_id)s." % {'image_id': image_id}) # Here we create a request context with credentials to support # delayed delete when using multi-tenant backend storage admin_tenant = CONF.admin_tenant_name auth_token = self.registry.auth_tok admin_context = context.RequestContext(user=CONF.admin_user, tenant=admin_tenant, auth_tok=auth_token) self.store_api.delete_from_backend(admin_context, uri) except Exception: msg = _("Failed to delete URI from image %(image_id)s") LOG.error(msg % {'image_id': image_id})
def get_all_locations(self): """Returns a list of image id and location tuple from scrub queue. :returns: a list of image id, location id and uri tuple from scrub queue """ ret = [] for image in self._get_all_images(): deleted_at = image.get('deleted_at') if not deleted_at: continue # NOTE: Strip off microseconds which may occur after the last '.,' # Example: 2012-07-07T19:14:34.974216 deleted_at = timeutils.isotime(deleted_at) date_str = deleted_at.rsplit('.', 1)[0].rsplit(',', 1)[0] delete_time = calendar.timegm(time.strptime(date_str, "%Y-%m-%dT%H:%M:%SZ")) if delete_time + self.scrub_time > time.time(): continue for loc in image['locations']: if loc['status'] != 'pending_delete': continue if self.metadata_encryption_key: uri = crypt.urlsafe_decrypt(self.metadata_encryption_key, loc['url']) else: uri = loc['url'] # if multi-store is enabled then we need to pass backend # to delete the image. backend = loc['metadata'].get('backend') if CONF.enabled_backends: ret.append((image['id'], loc['id'], uri, backend)) else: ret.append((image['id'], loc['id'], uri)) return ret
def _delete(self, id, uri, now): file_path = os.path.join(self.datadir, str(id)) if CONF.metadata_encryption_key is not None: uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri) try: LOG.debug(_("Deleting %(id)s") % {'id': id}) # Here we create a request context with credentials to support # delayed delete when using multi-tenant backend storage ctx = context.RequestContext(auth_tok=self.registry.auth_tok, user=self.admin_user, tenant=self.admin_tenant) store.delete_from_backend(ctx, uri) except store.UnsupportedBackend: msg = _("Failed to delete image from store (%(id)s).") LOG.error(msg % {'id': id}) except exception.NotFound: msg = _("Image not found in store (%(id)s).") LOG.error(msg % {'id': id}) self.registry.update_image(id, {'status': 'deleted'}) utils.safe_remove(file_path)
def _delete_image_location_from_backend(self, image_id, loc_id, uri): if CONF.metadata_encryption_key: uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri) try: LOG.debug("Scrubbing image %s from a location." % image_id) try: self.store_api.delete_from_backend(uri, self.admin_context) except store_exceptions.NotFound: LOG.info(_LI("Image location for image '%s' not found in " "backend; Marking image location deleted in " "db."), image_id) if loc_id != '-': db_api.get_api().image_location_delete(self.admin_context, image_id, int(loc_id), 'deleted') LOG.info(_LI("Image %s is scrubbed from a location."), image_id) except Exception as e: LOG.error(_LE("Unable to scrub image %(id)s from a location. " "Reason: %(exc)s ") % {'id': image_id, 'exc': encodeutils.exception_to_unicode(e)}) raise
def test_remote_image(self): """Verify an image added using a 'Location' header can be retrieved""" self.cleanup() self.start_servers(**self.__dict__.copy()) # 1. POST /images with public image named Image1 image_data = "*" * FIVE_KB headers = minimal_headers('Image1') path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, body=image_data) self.assertEqual(response.status, 201) data = json.loads(content) self.assertEqual(data['image']['checksum'], hashlib.md5(image_data).hexdigest()) self.assertEqual(data['image']['size'], FIVE_KB) image_id1 = data['image']['id'] # 2. GET first image # Verify all information on image we just added is correct path = "http://%s:%d/v1/images/%s" args = ("127.0.0.1", self.api_port, image_id1) http = httplib2.Http() response, content = http.request(path % args, 'GET') self.assertEqual(response.status, 200) self.assertEqual(response['content-length'], str(FIVE_KB)) self.assertEqual(content, "*" * FIVE_KB) # 3. GET first image from registry in order to find S3 location path = "http://%s:%d/images/%s" args = ("127.0.0.1", self.registry_port, image_id1) http = httplib2.Http() response, content = http.request(path % args, 'GET') if hasattr(self, 'metadata_encryption_key'): key = self.metadata_encryption_key else: key = self.api_server.metadata_encryption_key loc = json.loads(content)['image']['location'] s3_store_location = crypt.urlsafe_decrypt(key, loc) # 4. POST /images using location generated by Image1 image_id2 = utils.generate_uuid() image_data = "*" * FIVE_KB headers = minimal_headers('Image2') headers['X-Image-Meta-Id'] = image_id2 headers['X-Image-Meta-Location'] = s3_store_location path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers) self.assertEqual(response.status, 201) # ensure data is refreshed, previously the size assertion # applied to the metadata returned from the previous GET data = json.loads(content) self.assertEqual(data['image']['size'], FIVE_KB) # checksum is not set for a remote image, as the image data # is not yet retrieved self.assertEqual(data['image']['checksum'], None) # 5. GET second image and make sure it can stream the image path = "http://%s:%d/v1/images/%s" args = ("127.0.0.1", self.api_port, image_id2) http = httplib2.Http() response, content = http.request(path % args, 'GET') self.assertEqual(response.status, 200) self.assertEqual(response['content-length'], str(FIVE_KB)) self.assertEqual(content, "*" * FIVE_KB) # 6. DELETE first and second images path = "http://%s:%d/v1/images/%s" args = ("127.0.0.1", self.api_port, image_id1) http = httplib2.Http() http.request(path % args, 'DELETE') path = "http://%s:%d/v1/images/%s" args = ("127.0.0.1", self.api_port, image_id2) http = httplib2.Http() http.request(path % args, 'DELETE') self.stop_servers()
def test_remote_image(self): """ Ensure we can retrieve an image that was not stored by glance itself """ self.cleanup() self.start_servers(**self.__dict__.copy()) api_port = self.api_port registry_port = self.registry_port # POST /images with public image named Image1 image_data = "*" * FIVE_KB headers = { 'Content-Type': 'application/octet-stream', 'X-Image-Meta-Name': 'Image1', 'X-Image-Meta-Is-Public': 'True' } path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, body=image_data) self.assertEqual(response.status, 201, content) data = json.loads(content) self.assertEqual(data['image']['checksum'], hashlib.md5(image_data).hexdigest()) self.assertEqual(data['image']['size'], FIVE_KB) self.assertEqual(data['image']['name'], "Image1") self.assertEqual(data['image']['is_public'], True) image_id = data['image']['id'] # GET image and make sure data was uploaded path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'GET') self.assertEqual(response.status, 200) self.assertEqual(response['content-length'], str(FIVE_KB)) self.assertEqual(content, "*" * FIVE_KB) self.assertEqual( hashlib.md5(content).hexdigest(), hashlib.md5("*" * FIVE_KB).hexdigest()) # Find the location that was just added and use it as # the remote image location for the next image path = "http://%s:%d/images/%s" % ("0.0.0.0", self.registry_port, image_id) http = httplib2.Http() response, content = http.request(path, 'GET') self.assertEqual(response.status, 200) data = json.loads(content) self.assertTrue('location' in data['image'].keys()) loc = data['image']['location'] if hasattr(self, 'metadata_encryption_key'): key = self.metadata_encryption_key else: key = self.api_server.metadata_encryption_key swift_location = crypt.urlsafe_decrypt(key, loc) # POST /images with public image named Image1 without uploading data image_data = "*" * FIVE_KB headers = { 'Content-Type': 'application/octet-stream', 'X-Image-Meta-Name': 'Image1', 'X-Image-Meta-Is-Public': 'True', 'X-Image-Meta-Location': swift_location } path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers) self.assertEqual(response.status, 201, content) data = json.loads(content) self.assertEqual(data['image']['checksum'], None) self.assertEqual(data['image']['size'], 0) self.assertEqual(data['image']['name'], "Image1") self.assertEqual(data['image']['is_public'], True) image_id2 = data['image']['id'] # GET /images/2 ensuring the data already in swift is accessible path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id2) http = httplib2.Http() response, content = http.request(path, 'GET') self.assertEqual(response.status, 200) self.assertEqual(response['content-length'], str(FIVE_KB)) self.assertEqual(content, "*" * FIVE_KB) self.assertEqual( hashlib.md5(content).hexdigest(), hashlib.md5("*" * FIVE_KB).hexdigest()) # DELETE boty images # Verify image and all chunks are gone... path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'DELETE') self.assertEqual(response.status, 200) path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id2) http = httplib2.Http() response, content = http.request(path, 'DELETE') self.assertEqual(response.status, 200) self.stop_servers()
def test_large_objects(self): """ We test the large object manifest code path in the Swift driver. In the case where an image file is bigger than the config variable swift_store_large_object_size, then we chunk the image into Swift, and add a manifest put_object at the end. We test that the delete of the large object cleans up all the chunks in Swift, in addition to the manifest file (LP Bug# 833285) """ self.cleanup() self.swift_store_large_object_size = 2 # In MB self.swift_store_large_object_chunk_size = 1 # In MB self.start_servers(**self.__dict__.copy()) api_port = self.api_port registry_port = self.registry_port # GET /images # Verify no public images path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'GET') self.assertEqual(response.status, 200) self.assertEqual(content, '{"images": []}') # POST /images with public image named Image1 # attribute and no custom properties. Verify a 200 OK is returned image_data = "*" * FIVE_MB headers = { 'Content-Type': 'application/octet-stream', 'X-Image-Meta-Name': 'Image1', 'X-Image-Meta-Is-Public': 'True' } path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, body=image_data) self.assertEqual(response.status, 201, content) data = json.loads(content) self.assertEqual(data['image']['checksum'], hashlib.md5(image_data).hexdigest()) self.assertEqual(data['image']['size'], FIVE_MB) self.assertEqual(data['image']['name'], "Image1") self.assertEqual(data['image']['is_public'], True) image_id = data['image']['id'] # HEAD image # Verify image found now path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'HEAD') self.assertEqual(response.status, 200) self.assertEqual(response['x-image-meta-name'], "Image1") # GET image # Verify all information on image we just added is correct path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'GET') self.assertEqual(response.status, 200) expected_image_headers = { 'x-image-meta-id': image_id, 'x-image-meta-name': 'Image1', 'x-image-meta-is_public': 'True', 'x-image-meta-status': 'active', 'x-image-meta-disk_format': '', 'x-image-meta-container_format': '', 'x-image-meta-size': str(FIVE_MB) } expected_std_headers = { 'content-length': str(FIVE_MB), 'content-type': 'application/octet-stream' } for expected_key, expected_value in expected_image_headers.items(): self.assertEqual( response[expected_key], expected_value, "For key '%s' expected header value '%s'. Got '%s'" % (expected_key, expected_value, response[expected_key])) for expected_key, expected_value in expected_std_headers.items(): self.assertEqual( response[expected_key], expected_value, "For key '%s' expected header value '%s'. Got '%s'" % (expected_key, expected_value, response[expected_key])) self.assertEqual(content, "*" * FIVE_MB) self.assertEqual( hashlib.md5(content).hexdigest(), hashlib.md5("*" * FIVE_MB).hexdigest()) # We test that the delete of the large object cleans up all the # chunks in Swift, in addition to the manifest file (LP Bug# 833285) # Grab the actual Swift location and query the object manifest for # the chunks/segments. We will check that the segments don't exist # after we delete the object through Glance... path = "http://%s:%d/images/%s" % ("0.0.0.0", self.registry_port, image_id) http = httplib2.Http() response, content = http.request(path, 'GET') self.assertEqual(response.status, 200) data = json.loads(content) image_loc = data['image']['location'] if hasattr(self, 'metadata_encryption_key'): key = self.metadata_encryption_key else: key = self.api_server.metadata_encryption_key image_loc = crypt.urlsafe_decrypt(key, image_loc) image_loc = get_location_from_uri(image_loc) swift_loc = image_loc.store_location from swift.common import client as swift_client swift_conn = swift_client.Connection(authurl=swift_loc.swift_auth_url, user=swift_loc.user, key=swift_loc.key) # Verify the object manifest exists headers = swift_conn.head_object(swift_loc.container, swift_loc.obj) manifest = headers.get('x-object-manifest') self.assertTrue(manifest is not None, "Manifest could not be found!") # Grab the segment identifiers obj_container, obj_prefix = manifest.split('/', 1) segments = [ segment['name'] for segment in swift_conn.get_container(obj_container, prefix=obj_prefix)[1] ] # Verify the segments exist for segment in segments: headers = swift_conn.head_object(obj_container, segment) self.assertTrue(headers.get('content-length') is not None, headers) # DELETE image # Verify image and all chunks are gone... path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'DELETE') self.assertEqual(response.status, 200) # Verify the segments no longer exist for segment in segments: self.assertRaises(swift_client.ClientException, swift_conn.head_object, obj_container, segment) self.stop_servers()
def test_remote_image(self): """Verify an image added using a 'Location' header can be retrieved""" self.cleanup() self.start_servers(**self.__dict__.copy()) # 1. POST /images with public image named Image1 image_data = "*" * FIVE_KB headers = {'Content-Type': 'application/octet-stream', 'X-Image-Meta-Name': 'Image1', 'X-Image-Meta-Is-Public': 'True'} path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, body=image_data) self.assertEqual(response.status, 201) data = json.loads(content) self.assertEqual(data['image']['checksum'], hashlib.md5(image_data).hexdigest()) self.assertEqual(data['image']['size'], FIVE_KB) image_id1 = data['image']['id'] # 2. GET first image # Verify all information on image we just added is correct path = "http://%s:%d/v1/images/%s" args = ("0.0.0.0", self.api_port, image_id1) http = httplib2.Http() response, content = http.request(path % args, 'GET') self.assertEqual(response.status, 200) self.assertEqual(response['content-length'], str(FIVE_KB)) self.assertEqual(content, "*" * FIVE_KB) # 3. GET first image from registry in order to find S3 location path = "http://%s:%d/images/%s" args = ("0.0.0.0", self.registry_port, image_id1) http = httplib2.Http() response, content = http.request(path % args, 'GET') if hasattr(self, 'metadata_encryption_key'): key = self.metadata_encryption_key else: key = self.api_server.metadata_encryption_key loc = json.loads(content)['image']['location'] s3_store_location = crypt.urlsafe_decrypt(key, loc) # 4. POST /images using location generated by Image1 image_id2 = utils.generate_uuid() image_data = "*" * FIVE_KB headers = {'Content-Type': 'application/octet-stream', 'X-Image-Meta-Id': image_id2, 'X-Image-Meta-Name': 'Image2', 'X-Image-Meta-Is-Public': 'True', 'X-Image-Meta-Location': s3_store_location} path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers) self.assertEqual(response.status, 201) self.assertEqual(data['image']['size'], FIVE_KB) self.assertEqual(data['image']['checksum'], hashlib.md5(image_data).hexdigest()) # 5. GET second image and make sure it can stream the image path = "http://%s:%d/v1/images/%s" args = ("0.0.0.0", self.api_port, image_id2) http = httplib2.Http() response, content = http.request(path % args, 'GET') self.assertEqual(response.status, 200) self.assertEqual(response['content-length'], str(FIVE_KB)) self.assertEqual(content, "*" * FIVE_KB) # 6. DELETE first and second images path = "http://%s:%d/v1/images/%s" args = ("0.0.0.0", self.api_port, image_id1) http = httplib2.Http() http.request(path % args, 'DELETE') path = "http://%s:%d/v1/images/%s" args = ("0.0.0.0", self.api_port, image_id2) http = httplib2.Http() http.request(path % args, 'DELETE') self.stop_servers()
def test_large_objects(self): """ We test the large object manifest code path in the Swift driver. In the case where an image file is bigger than the config variable swift_store_large_object_size, then we chunk the image into Swift, and add a manifest put_object at the end. We test that the delete of the large object cleans up all the chunks in Swift, in addition to the manifest file (LP Bug# 833285) """ self.cleanup() self.swift_store_large_object_size = 2 # In MB self.swift_store_large_object_chunk_size = 1 # In MB self.start_servers(**self.__dict__.copy()) api_port = self.api_port registry_port = self.registry_port # GET /images # Verify no public images path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'GET') self.assertEqual(response.status, 200) self.assertEqual(content, '{"images": []}') # POST /images with public image named Image1 # attribute and no custom properties. Verify a 200 OK is returned image_data = "*" * FIVE_MB headers = minimal_headers('Image1') path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, body=image_data) self.assertEqual(response.status, 201, content) data = json.loads(content) self.assertEqual(data['image']['checksum'], hashlib.md5(image_data).hexdigest()) self.assertEqual(data['image']['size'], FIVE_MB) self.assertEqual(data['image']['name'], "Image1") self.assertEqual(data['image']['is_public'], True) image_id = data['image']['id'] # HEAD image # Verify image found now path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'HEAD') self.assertEqual(response.status, 200) self.assertEqual(response['x-image-meta-name'], "Image1") # GET image # Verify all information on image we just added is correct path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'GET') self.assertEqual(response.status, 200) expected_image_headers = { 'x-image-meta-id': image_id, 'x-image-meta-name': 'Image1', 'x-image-meta-is_public': 'True', 'x-image-meta-status': 'active', 'x-image-meta-disk_format': 'raw', 'x-image-meta-container_format': 'ovf', 'x-image-meta-size': str(FIVE_MB) } expected_std_headers = { 'content-length': str(FIVE_MB), 'content-type': 'application/octet-stream'} for expected_key, expected_value in expected_image_headers.items(): self.assertEqual(response[expected_key], expected_value, "For key '%s' expected header value '%s'. Got '%s'" % (expected_key, expected_value, response[expected_key])) for expected_key, expected_value in expected_std_headers.items(): self.assertEqual(response[expected_key], expected_value, "For key '%s' expected header value '%s'. Got '%s'" % (expected_key, expected_value, response[expected_key])) self.assertEqual(content, "*" * FIVE_MB) self.assertEqual(hashlib.md5(content).hexdigest(), hashlib.md5("*" * FIVE_MB).hexdigest()) # We test that the delete of the large object cleans up all the # chunks in Swift, in addition to the manifest file (LP Bug# 833285) # Grab the actual Swift location and query the object manifest for # the chunks/segments. We will check that the segments don't exist # after we delete the object through Glance... path = "http://%s:%d/images/%s" % ("0.0.0.0", self.registry_port, image_id) http = httplib2.Http() response, content = http.request(path, 'GET') self.assertEqual(response.status, 200) data = json.loads(content) image_loc = data['image']['location'] if hasattr(self, 'metadata_encryption_key'): key = self.metadata_encryption_key else: key = self.api_server.metadata_encryption_key image_loc = crypt.urlsafe_decrypt(key, image_loc) image_loc = get_location_from_uri(image_loc) swift_loc = image_loc.store_location from swift.common import client as swift_client swift_conn = swift_client.Connection( authurl=swift_loc.swift_auth_url, user=swift_loc.user, key=swift_loc.key) # Verify the object manifest exists headers = swift_conn.head_object(swift_loc.container, swift_loc.obj) manifest = headers.get('x-object-manifest') self.assertTrue(manifest is not None, "Manifest could not be found!") # Grab the segment identifiers obj_container, obj_prefix = manifest.split('/', 1) segments = [segment['name'] for segment in swift_conn.get_container(obj_container, prefix=obj_prefix)[1]] # Verify the segments exist for segment in segments: headers = swift_conn.head_object(obj_container, segment) self.assertTrue(headers.get('content-length') is not None, headers) # DELETE image # Verify image and all chunks are gone... path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'DELETE') self.assertEqual(response.status, 200) # Verify the segments no longer exist for segment in segments: self.assertRaises(swift_client.ClientException, swift_conn.head_object, obj_container, segment) self.stop_servers()
def test_remote_image(self): """ Ensure we can retrieve an image that was not stored by glance itself """ self.cleanup() self.start_servers(**self.__dict__.copy()) api_port = self.api_port registry_port = self.registry_port # POST /images with public image named Image1 image_data = "*" * FIVE_KB headers = minimal_headers('Image1') path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, body=image_data) self.assertEqual(response.status, 201, content) data = json.loads(content) self.assertEqual(data['image']['checksum'], hashlib.md5(image_data).hexdigest()) self.assertEqual(data['image']['size'], FIVE_KB) self.assertEqual(data['image']['name'], "Image1") self.assertEqual(data['image']['is_public'], True) image_id = data['image']['id'] # GET image and make sure data was uploaded path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'GET') self.assertEqual(response.status, 200) self.assertEqual(response['content-length'], str(FIVE_KB)) self.assertEqual(content, "*" * FIVE_KB) self.assertEqual(hashlib.md5(content).hexdigest(), hashlib.md5("*" * FIVE_KB).hexdigest()) # Find the location that was just added and use it as # the remote image location for the next image path = "http://%s:%d/images/%s" % ("0.0.0.0", self.registry_port, image_id) http = httplib2.Http() response, content = http.request(path, 'GET') self.assertEqual(response.status, 200) data = json.loads(content) self.assertTrue('location' in data['image'].keys()) loc = data['image']['location'] if hasattr(self, 'metadata_encryption_key'): key = self.metadata_encryption_key else: key = self.api_server.metadata_encryption_key swift_location = crypt.urlsafe_decrypt(key, loc) # POST /images with public image named Image1 without uploading data image_data = "*" * FIVE_KB headers = minimal_headers('Image1') headers['X-Image-Meta-Location'] = swift_location path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers) self.assertEqual(response.status, 201, content) data = json.loads(content) self.assertEqual(data['image']['checksum'], None) self.assertEqual(data['image']['size'], FIVE_KB) self.assertEqual(data['image']['name'], "Image1") self.assertEqual(data['image']['is_public'], True) image_id2 = data['image']['id'] # GET /images/2 ensuring the data already in swift is accessible path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id2) http = httplib2.Http() response, content = http.request(path, 'GET') self.assertEqual(response.status, 200) self.assertEqual(response['content-length'], str(FIVE_KB)) self.assertEqual(content, "*" * FIVE_KB) self.assertEqual(hashlib.md5(content).hexdigest(), hashlib.md5("*" * FIVE_KB).hexdigest()) # DELETE boty images # Verify image and all chunks are gone... path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'DELETE') self.assertEqual(response.status, 200) path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id2) http = httplib2.Http() response, content = http.request(path, 'DELETE') self.assertEqual(response.status, 200) self.stop_servers()
def test_remote_image(self): """ Ensure we can retrieve an image that was not stored by glance itself """ self.cleanup() self.start_servers(**self.__dict__.copy()) api_port = self.api_port registry_port = self.registry_port # POST /images with public image named Image1 image_data = "*" * FIVE_KB headers = { "Content-Type": "application/octet-stream", "X-Image-Meta-Name": "Image1", "X-Image-Meta-Is-Public": "True", } path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, "POST", headers=headers, body=image_data) self.assertEqual(response.status, 201, content) data = json.loads(content) self.assertEqual(data["image"]["checksum"], hashlib.md5(image_data).hexdigest()) self.assertEqual(data["image"]["size"], FIVE_KB) self.assertEqual(data["image"]["name"], "Image1") self.assertEqual(data["image"]["is_public"], True) image_id = data["image"]["id"] # GET image and make sure data was uploaded path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, "GET") self.assertEqual(response.status, 200) self.assertEqual(response["content-length"], str(FIVE_KB)) self.assertEqual(content, "*" * FIVE_KB) self.assertEqual(hashlib.md5(content).hexdigest(), hashlib.md5("*" * FIVE_KB).hexdigest()) # Find the location that was just added and use it as # the remote image location for the next image path = "http://%s:%d/images/%s" % ("0.0.0.0", self.registry_port, image_id) http = httplib2.Http() response, content = http.request(path, "GET") self.assertEqual(response.status, 200) data = json.loads(content) self.assertTrue("location" in data["image"].keys()) loc = data["image"]["location"] if hasattr(self, "metadata_encryption_key"): key = self.metadata_encryption_key else: key = self.api_server.metadata_encryption_key swift_location = crypt.urlsafe_decrypt(key, loc) # POST /images with public image named Image1 without uploading data image_data = "*" * FIVE_KB headers = { "Content-Type": "application/octet-stream", "X-Image-Meta-Name": "Image1", "X-Image-Meta-Is-Public": "True", "X-Image-Meta-Location": swift_location, } path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, "POST", headers=headers) self.assertEqual(response.status, 201, content) data = json.loads(content) self.assertEqual(data["image"]["checksum"], None) self.assertEqual(data["image"]["size"], FIVE_KB) self.assertEqual(data["image"]["name"], "Image1") self.assertEqual(data["image"]["is_public"], True) image_id2 = data["image"]["id"] # GET /images/2 ensuring the data already in swift is accessible path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id2) http = httplib2.Http() response, content = http.request(path, "GET") self.assertEqual(response.status, 200) self.assertEqual(response["content-length"], str(FIVE_KB)) self.assertEqual(content, "*" * FIVE_KB) self.assertEqual(hashlib.md5(content).hexdigest(), hashlib.md5("*" * FIVE_KB).hexdigest()) # DELETE boty images # Verify image and all chunks are gone... path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, "DELETE") self.assertEqual(response.status, 200) path = "http://%s:%d/v1/images/%s" % ("0.0.0.0", self.api_port, image_id2) http = httplib2.Http() response, content = http.request(path, "DELETE") self.assertEqual(response.status, 200) self.stop_servers()
def test_scrubber_with_metadata_enc(self): """ test that files written to scrubber_data_dir use metadata_encryption_key when available to encrypt the location """ config_path = os.environ.get('GLANCE_TEST_SWIFT_CONF') if not config_path: msg = "GLANCE_TEST_SWIFT_CONF environ not set." self.skipTest(msg) raw_config = read_config(config_path) swift_config = parse_config(raw_config) self.cleanup() self.start_servers(delayed_delete=True, daemon=True, default_store='swift', **swift_config) # add an image headers = { 'x-image-meta-name': 'test_image', 'x-image-meta-is_public': 'true', 'x-image-meta-disk_format': 'raw', 'x-image-meta-container_format': 'ovf', 'content-type': 'application/octet-stream', } path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', body='XXX', headers=headers) self.assertEqual(response.status, 201) image = json.loads(content)['image'] self.assertEqual('active', image['status']) image_id = image['id'] # delete the image path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'DELETE') self.assertEqual(response.status, 200) response, content = http.request(path, 'HEAD') self.assertEqual(response.status, 200) self.assertEqual('pending_delete', response['x-image-meta-status']) # ensure the marker file has encrypted the image location by decrypting # it and checking the image_id is intact file_path = os.path.join(self.api_server.scrubber_datadir, str(image_id)) marker_uri = '' with open(file_path, 'r') as f: marker_uri = f.readline().strip() self.assertTrue(marker_uri is not None) decrypted_uri = crypt.urlsafe_decrypt( self.api_server.metadata_encryption_key, marker_uri) loc = StoreLocation({}) loc.parse_uri(decrypted_uri) self.assertEqual("swift+http", loc.scheme) self.assertEqual(image['id'], loc.obj) wait_for_scrub(path) self.stop_servers()
def decrypt_location(uri): return crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri)
def test_remote_image(self): """Verify an image added using a 'Location' header can be retrieved""" self.cleanup() self.start_servers(**self.__dict__.copy()) # 1. POST /images with public image named Image1 image_data = "*" * FIVE_KB headers = minimal_headers('Image1') path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers, body=image_data) self.assertEqual(response.status, 201) data = json.loads(content) self.assertEqual(data['image']['checksum'], hashlib.md5(image_data).hexdigest()) self.assertEqual(data['image']['size'], FIVE_KB) image_id1 = data['image']['id'] # 2. GET first image # Verify all information on image we just added is correct path = "http://%s:%d/v1/images/%s" args = ("0.0.0.0", self.api_port, image_id1) http = httplib2.Http() response, content = http.request(path % args, 'GET') self.assertEqual(response.status, 200) self.assertEqual(response['content-length'], str(FIVE_KB)) self.assertEqual(content, "*" * FIVE_KB) # 3. GET first image from registry in order to find S3 location path = "http://%s:%d/images/%s" args = ("0.0.0.0", self.registry_port, image_id1) http = httplib2.Http() response, content = http.request(path % args, 'GET') if hasattr(self, 'metadata_encryption_key'): key = self.metadata_encryption_key else: key = self.api_server.metadata_encryption_key loc = json.loads(content)['image']['location'] s3_store_location = crypt.urlsafe_decrypt(key, loc) # 4. POST /images using location generated by Image1 image_id2 = utils.generate_uuid() image_data = "*" * FIVE_KB headers = minimal_headers('Image2') headers['X-Image-Meta-Id'] = image_id2 headers['X-Image-Meta-Location'] = s3_store_location path = "http://%s:%d/v1/images" % ("0.0.0.0", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', headers=headers) self.assertEqual(response.status, 201) # ensure data is refreshed, previously the size assertion # applied to the metadata returned from the previous GET data = json.loads(content) self.assertEqual(data['image']['size'], FIVE_KB) # checksum is not set for a remote image, as the image data # is not yet retrieved self.assertEqual(data['image']['checksum'], None) # 5. GET second image and make sure it can stream the image path = "http://%s:%d/v1/images/%s" args = ("0.0.0.0", self.api_port, image_id2) http = httplib2.Http() response, content = http.request(path % args, 'GET') self.assertEqual(response.status, 200) self.assertEqual(response['content-length'], str(FIVE_KB)) self.assertEqual(content, "*" * FIVE_KB) # 6. DELETE first and second images path = "http://%s:%d/v1/images/%s" args = ("0.0.0.0", self.api_port, image_id1) http = httplib2.Http() http.request(path % args, 'DELETE') path = "http://%s:%d/v1/images/%s" args = ("0.0.0.0", self.api_port, image_id2) http = httplib2.Http() http.request(path % args, 'DELETE') self.stop_servers()
def test_scrubber_with_metadata_enc(self): """ test that files written to scrubber_data_dir use metadata_encryption_key when available to encrypt the location """ # FIXME(flaper87): It looks like an older commit # may have broken this test. The file_queue `add_location` # is not being called. self.skipTest("Test broken. See bug #1366682") self.cleanup() self.start_servers(delayed_delete=True, daemon=True, default_store='file') # add an image headers = { 'x-image-meta-name': 'test_image', 'x-image-meta-is_public': 'true', 'x-image-meta-disk_format': 'raw', 'x-image-meta-container_format': 'ovf', 'content-type': 'application/octet-stream', } path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', body='XXX', headers=headers) self.assertEqual(201, response.status) image = jsonutils.loads(content)['image'] self.assertEqual('active', image['status']) image_id = image['id'] # delete the image path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'DELETE') self.assertEqual(200, response.status) response, content = http.request(path, 'HEAD') self.assertEqual(200, response.status) self.assertEqual('pending_delete', response['x-image-meta-status']) # ensure the marker file has encrypted the image location by decrypting # it and checking the image_id is intact file_path = os.path.join(self.api_server.scrubber_datadir, str(image_id)) marker_uri = None with open(file_path, 'r') as f: marker_uri = f.readline().strip() self.assertTrue(marker_uri is not None) decrypted_uri = crypt.urlsafe_decrypt( self.api_server.metadata_encryption_key, marker_uri) loc = glance_store.location.StoreLocation({}) loc.parse_uri(decrypted_uri) self.assertEqual("file", loc.scheme) self.assertEqual(image['id'], loc.obj) self.wait_for_scrub(path) self.stop_servers()
def test_scrubber_with_metadata_enc(self): """ test that files written to scrubber_data_dir use metadata_encryption_key when available to encrypt the location """ config_path = os.environ.get('GLANCE_TEST_SWIFT_CONF') if not config_path: msg = "GLANCE_TEST_SWIFT_CONF environ not set." self.skipTest(msg) raw_config = read_config(config_path) swift_config = parse_config(raw_config) self.cleanup() self.start_servers(delayed_delete=True, daemon=True, default_store='swift', **swift_config) # add an image headers = { 'x-image-meta-name': 'test_image', 'x-image-meta-is_public': 'true', 'x-image-meta-disk_format': 'raw', 'x-image-meta-container_format': 'ovf', 'content-type': 'application/octet-stream', } path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port) http = httplib2.Http() response, content = http.request(path, 'POST', body='XXX', headers=headers) self.assertEqual(response.status, 201) image = json.loads(content)['image'] self.assertEqual('active', image['status']) image_id = image['id'] # delete the image path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, 'DELETE') self.assertEqual(response.status, 200) response, content = http.request(path, 'HEAD') self.assertEqual(response.status, 200) self.assertEqual('pending_delete', response['x-image-meta-status']) # ensure the marker file has encrypted the image location by decrypting # it and checking the image_id is intact file_path = os.path.join(self.api_server.scrubber_datadir, str(image_id)) marker_uri = '' with open(file_path, 'r') as f: marker_uri = f.readline().strip() self.assertTrue(marker_uri is not None) decrypted_uri = crypt.urlsafe_decrypt( self.api_server.metadata_encryption_key, marker_uri) loc = StoreLocation({}) loc.parse_uri(decrypted_uri) self.assertIn(loc.scheme, ("swift+http", "swift+https")) self.assertEqual(image['id'], loc.obj) self.wait_for_scrub(path) self.stop_servers()
def test_scrubber_with_metadata_enc(self): """ test that files written to scrubber_data_dir use metadata_encryption_key when available to encrypt the location """ config_path = os.environ.get("GLANCE_TEST_SWIFT_CONF") if not config_path: msg = "GLANCE_TEST_SWIFT_CONF environ not set." self.skipTest(msg) raw_config = read_config(config_path) swift_config = parse_config(raw_config) self.cleanup() self.start_servers(delayed_delete=True, daemon=True, default_store="swift", **swift_config) # add an image headers = { "x-image-meta-name": "test_image", "x-image-meta-is_public": "true", "x-image-meta-disk_format": "raw", "x-image-meta-container_format": "ovf", "content-type": "application/octet-stream", } path = "http://%s:%d/v1/images" % ("127.0.0.1", self.api_port) http = httplib2.Http() response, content = http.request(path, "POST", body="XXX", headers=headers) self.assertEqual(response.status, 201) image = json.loads(content)["image"] self.assertEqual("active", image["status"]) image_id = image["id"] # delete the image path = "http://%s:%d/v1/images/%s" % ("127.0.0.1", self.api_port, image_id) http = httplib2.Http() response, content = http.request(path, "DELETE") self.assertEqual(response.status, 200) response, content = http.request(path, "HEAD") self.assertEqual(response.status, 200) self.assertEqual("pending_delete", response["x-image-meta-status"]) # ensure the marker file has encrypted the image location by decrypting # it and checking the image_id is intact file_path = os.path.join(self.api_server.scrubber_datadir, str(image_id)) marker_uri = "" with open(file_path, "r") as f: marker_uri = f.readline().strip() self.assertTrue(marker_uri is not None) decrypted_uri = crypt.urlsafe_decrypt(self.api_server.metadata_encryption_key, marker_uri) loc = StoreLocation({}) loc.parse_uri(decrypted_uri) self.assertEqual("swift+http", loc.scheme) self.assertEqual(image["id"], loc.obj) # NOTE(jkoelker) The build servers sometimes take longer than # 15 seconds to scrub. Give it up to 5 min, checking # checking every 15 seconds. When/if it flips to # deleted, bail immediately. for _ in xrange(3): time.sleep(5) response, content = http.request(path, "HEAD") if response["x-image-meta-status"] == "deleted" and response["x-image-meta-deleted"] == "True": break else: continue else: self.fail("image was never scrubbed") self.stop_servers()