def add_to_backend(self, conf, image_id, data, size, scheme=None, context=None, verifier=None): store_max_size = 7 current_store_size = 2 for location in self.data.keys(): if image_id in location: raise exception.Duplicate() if not size: # 'data' is a string wrapped in a LimitingReader|CooperativeReader # pipeline, so peek under the hood of those objects to get at the # string itself. size = len(data.data.fd) if (current_store_size + size) > store_max_size: raise exception.StorageFull() if context.user == USER2: raise exception.Forbidden() if context.user == USER3: raise exception.StorageWriteDenied() self.data[image_id] = (data, size) checksum = 'Z' return (image_id, size, checksum, self.store_metadata)
def test_catch_error_unhandled_debug_mode(self, mock_function): mock_function.side_effect = exception.Duplicate() my_mock = mock.Mock() my_mock.debug = True self.assertRaises(exception.Duplicate, cache_manage.list_cached, my_mock)
def add(self, image_id, image_file, image_size): """Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ checksum = hashlib.md5() image_file = _Reader(image_file, checksum) loc = StoreLocation({'scheme': self.scheme, 'server_host': self.server_host, 'folder_name': self.store_image_dir, 'datacenter_path': self.datacenter_path, 'datastore_name': self.datastore_name, 'image_id': image_id}) cookie = self._build_vim_cookie_header( self._session.vim.client.options.transport.cookiejar) headers = {'Cookie': cookie, 'Content-Length': image_size} conn = self._get_http_conn('PUT', loc, headers, content=image_file) res = conn.getresponse() if res.status == httplib.CONFLICT: raise exception.Duplicate(_("Image file %(image_id)s already " "exists!") % {'image_id': image_id}) return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, and checksum :raises `glance.common.exception.Duplicate` if the image already existed """ image = SheepdogImage(self.addr, self.port, image_id, self.chunk_size) if image.exist(): raise exception.Duplicate(_("Sheepdog image %s already exists") % image_id) location = StoreLocation({'image': image_id}) checksum = hashlib.md5() image.create(image_size) total = left = image_size while left > 0: length = min(self.chunk_size, left) data = image_file.read(length) image.write(data, total - left, length) left -= length checksum.update(data) return (location.get_uri(), image_size, checksum.hexdigest(), {})
def task_create(context, values): """Create a task object""" global DATA task_values = copy.deepcopy(values) task_id = task_values.get('id', str(uuid.uuid4())) required_attributes = ['type', 'status', 'input'] allowed_attributes = ['id', 'type', 'status', 'input', 'result', 'owner', 'message', 'expires_at', 'created_at', 'updated_at', 'deleted_at', 'deleted'] if task_id in DATA['tasks']: raise exception.Duplicate() for key in required_attributes: if key not in task_values: raise exception.Invalid('%s is a required attribute' % key) incorrect_keys = set(task_values.keys()) - set(allowed_attributes) if incorrect_keys: raise exception.Invalid( 'The keys %s are not valid' % str(incorrect_keys)) task_info_values = _pop_task_info_values(task_values) task = _task_format(task_id, **task_values) DATA['tasks'][task_id] = task task_info = _task_info_create(task['id'], task_info_values) return _format_task_from_db(task, task_info)
def image_create(context, image_values): global DATA image_id = image_values.get('id', str(uuid.uuid4())) if image_id in DATA['images']: raise exception.Duplicate() if 'status' not in image_values: raise exception.Invalid('status is a required attribute') allowed_keys = set(['id', 'name', 'status', 'min_ram', 'min_disk', 'size', 'virtual_size', 'checksum', 'locations', 'owner', 'protected', 'is_public', 'container_format', 'disk_format', 'created_at', 'updated_at', 'deleted', 'deleted_at', 'properties', 'tags']) incorrect_keys = set(image_values.keys()) - allowed_keys if incorrect_keys: raise exception.Invalid( 'The keys %s are not valid' % str(incorrect_keys)) image = _image_format(image_id, **image_values) DATA['images'][image_id] = image location_data = image_values.get('locations', None) if location_data is not None: _image_locations_set(image_id, location_data) DATA['tags'][image_id] = image.pop('tags', []) return _normalize_locations(copy.deepcopy(image))
def test_catch_error_unhandled(self, mock_function): mock_function.side_effect = exception.Duplicate() my_mock = mock.Mock() my_mock.debug = False self.assertEqual(cache_manage.FAILURE, cache_manage.list_cached(my_mock))
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ loc = StoreLocation({'image_id': image_id}) if self.fs.exists(image_id): raise exception.Duplicate( _("GridFS already has an image at " "location %s") % loc.get_uri()) LOG.debug( _("Adding a new image to GridFS with id %s and size %s") % (image_id, image_size)) self.fs.put(image_file, _id=image_id) image = self._get_file(loc) LOG.debug( _("Uploaded image %s, md5 %s, length %s to GridFS") % (image._id, image.md5, image.length)) return (loc.get_uri(), image.length, image.md5, {})
def _image_update(context, values, image_id, purge_props=False): """ Used internally by image_create and image_update :param context: Request context :param values: A dict of attributes to set :param image_id: If None, create the image, otherwise, find and update it """ session = get_session() with session.begin(): # Remove the properties passed in the values mapping. We # handle properties separately from base image attributes, # and leaving properties in the values mapping will cause # a SQLAlchemy model error because SQLAlchemy expects the # properties attribute of an Image model to be a list and # not a dict. properties = values.pop('properties', {}) if image_id: image_ref = image_get(context, image_id, session=session) # Perform authorization check check_mutate_authorization(context, image_ref) else: if 'size' in values: values['size'] = int(values['size']) if 'min_ram' in values: values['min_ram'] = int(values['min_ram']) if 'min_disk' in values: values['min_disk'] = int(values['min_disk']) values['is_public'] = bool(values.get('is_public', False)) image_ref = models.Image() # Need to canonicalize ownership if 'owner' in values and not values['owner']: values['owner'] = None if image_id: # Don't drop created_at if we're passing it in... _drop_protected_attrs(models.Image, values) image_ref.update(values) # Validate the attributes before we go any further. From my # investigation, the @validates decorator does not validate # on new records, only on existing records, which is, well, # idiotic. validate_image(image_ref.to_dict()) try: image_ref.save(session=session) except IntegrityError, e: raise exception.Duplicate("Image ID %s already exists!" % values['id']) _set_properties_for_image(context, image_ref, properties, purge_props, session)
def add(self, image_member): try: self.get(image_member.member_id) except exception.NotFound: pass else: msg = _('The target member %(member_id)s is already ' 'associated with image %(image_id)s.') % { 'member_id': image_member.member_id, 'image_id': self.image.image_id} raise exception.Duplicate(msg) image_member_values = self._format_image_member_to_db(image_member) # Note(shalq): find the image member including the member marked with # deleted. We will use only one record to represent membership between # the same image and member. The record of the deleted image member # will be reused, if it exists, update its properties instead of # creating a new one. members = self.db_api.image_member_find(self.context, image_id=self.image.image_id, member=image_member.member_id, include_deleted=True) if members: new_values = self.db_api.image_member_update(self.context, members[0]['id'], image_member_values) else: new_values = self.db_api.image_member_create(self.context, image_member_values) image_member.created_at = new_values['created_at'] image_member.updated_at = new_values['updated_at'] image_member.id = new_values['id']
def schedule_delete_from_backend(uri, conf, context, image_id, **kwargs): """ Given a uri and a time, schedule the deletion of an image. """ conf.register_opts(delete_opts) if not conf.delayed_delete: registry.update_image_metadata(context, image_id, {'status': 'deleted'}) try: return delete_from_backend(uri, **kwargs) except (UnsupportedBackend, exception.NotFound): msg = _("Failed to delete image from store (%(uri)s).") % locals() logger.error(msg) datadir = get_scrubber_datadir(conf) delete_time = time.time() + conf.scrub_time file_path = os.path.join(datadir, str(image_id)) utils.safe_mkdirs(datadir) if os.path.exists(file_path): msg = _("Image id %(image_id)s already queued for delete") % { 'image_id': image_id } raise exception.Duplicate(msg) with open(file_path, 'w') as f: f.write('\n'.join([uri, str(int(delete_time))])) os.chmod(file_path, 0600) os.utime(file_path, (delete_time, delete_time)) registry.update_image_metadata(context, image_id, {'status': 'pending_delete'})
def add_to_backend(self, context, scheme, image_id, data, size): for location in self.data.keys(): if image_id in location: raise exception.Duplicate() self.data[image_id] = (data, size or len(data)) checksum = 'Z' return (image_id, size, checksum)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed :note By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ datadir = self._find_best_datadir(image_size) filepath = os.path.join(datadir, str(image_id)) if os.path.exists(filepath): raise exception.Duplicate( _("Image file %s already exists!") % filepath) checksum = hashlib.md5() bytes_written = 0 try: with open(filepath, 'wb') as f: for buf in utils.chunkreadable(image_file, ChunkedFile.CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) f.write(buf) except IOError as e: if e.errno != errno.EACCES: self._delete_partial(filepath, image_id) exceptions = { errno.EFBIG: exception.StorageFull(), errno.ENOSPC: exception.StorageFull(), errno.EACCES: exception.StorageWriteDenied() } raise exceptions.get(e.errno, e) except Exception: with excutils.save_and_reraise_exception(): self._delete_partial(filepath, image_id) checksum_hex = checksum.hexdigest() metadata = self._get_metadata() LOG.debug( _("Wrote %(bytes_written)d bytes to %(filepath)s with " "checksum %(checksum_hex)s"), { 'bytes_written': bytes_written, 'filepath': filepath, 'checksum_hex': checksum_hex }) return ('file://%s' % filepath, bytes_written, checksum_hex, metadata)
def add(self, image_id, image_file, image_size): """Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed `glance.common.exception.UnexpectedStatus` if the upload request returned an unexpected status. The expected responses are 201 Created and 200 OK. """ checksum = hashlib.md5() image_file = _Reader(image_file, checksum) loc = StoreLocation({ 'scheme': self.scheme, 'server_host': self.server_host, 'image_dir': self.store_image_dir, 'datacenter_path': self.datacenter_path, 'datastore_name': self.datastore_name, 'image_id': image_id }) cookie = self._build_vim_cookie_header( self._session.vim.client.options.transport.cookiejar) headers = { 'Connection': 'Keep-Alive', 'Cookie': cookie, 'Transfer-Encoding': 'chunked' } try: conn = self._get_http_conn('PUT', loc, headers, content=image_file) res = conn.getresponse() except Exception: with excutils.save_and_reraise_exception(): LOG.exception( _('Failed to upload content of image ' '%(image)s') % {'image': image_id}) if res.status == httplib.CONFLICT: raise exception.Duplicate( _("Image file %(image_id)s already " "exists!") % {'image_id': image_id}) if res.status not in (httplib.CREATED, httplib.OK): msg = (_('Failed to upload content of image %(image)s') % { 'image': image_id }) LOG.error(msg) raise exception.UnexpectedStatus(status=res.status, body=res.read()) return (loc.get_uri(), image_file.size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns an `glance.store.ImageAddResult` object containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval `glance.store.ImageAddResult` object :raises `glance.common.exception.Duplicate` if the image already existed :note By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ filepath = os.path.join(self.datadir, str(image_id)) if os.path.exists(filepath): raise exception.Duplicate( _("Image file %s already exists!") % filepath) checksum = hashlib.md5() bytes_written = 0 try: with open(filepath, 'wb') as f: for buf in utils.chunkreadable(image_file, ChunkedFile.CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) f.write(buf) except IOError as e: if e.errno in [errno.EFBIG, errno.ENOSPC]: try: os.unlink(filepath) except Exception: msg = _('Unable to remove partial image data for image %s') LOG.error(msg % image_id) raise exception.StorageFull() elif e.errno == errno.EACCES: raise exception.StorageWriteDenied() else: raise checksum_hex = checksum.hexdigest() LOG.debug( _("Wrote %(bytes_written)d bytes to %(filepath)s with " "checksum %(checksum_hex)s") % locals()) return ('file://%s' % filepath, bytes_written, checksum_hex)
def add_to_backend(self, context, scheme, image_id, data, size): store_max_size = 7 current_store_size = 2 for location in self.data.keys(): if image_id in location: raise exception.Duplicate() if size and (current_store_size + size) > store_max_size: raise exception.StorageFull() if context.user == USER2: raise exception.Forbidden() if context.user == USER3: raise exception.StorageWriteDenied() self.data[image_id] = (data, size or len(data)) checksum = 'Z' return (image_id, size, checksum)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ checksum = hashlib.md5() image_name = str(image_id) with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn: fsid = None if hasattr(conn, 'get_fsid'): fsid = conn.get_fsid() with conn.open_ioctx(self.pool) as ioctx: order = int(math.log(self.chunk_size, 2)) LOG.debug('creating image %s with order %d', image_name, order) try: loc = self._create_image(fsid, ioctx, image_name, image_size, order) except rbd.ImageExists: raise exception.Duplicate( _('RBD image %s already exists') % image_id) try: with rbd.Image(ioctx, image_name) as image: offset = 0 chunks = utils.chunkreadable(image_file, self.chunk_size) for chunk in chunks: offset += image.write(chunk, offset) checksum.update(chunk) if loc.snapshot: image.create_snap(loc.snapshot) image.protect_snap(loc.snapshot) except: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exception. with excutils.save_and_reraise_exception(): self._delete_image(loc.image, loc.snapshot) return (loc.get_uri(), image_size, checksum.hexdigest(), {})
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, checksum and a dictionary with storage system specific information :raises `glance.common.exception.Duplicate` if the image already existed """ loc = StoreLocation({'image_id': image_id}) if self.fs.exists(image_id): raise exception.Duplicate( _("GridFS already has an image at " "location %s") % loc.get_uri()) LOG.debug( _("Adding a new image to GridFS with id %(id)s and " "size %(size)s") % { 'id': image_id, 'size': image_size }) try: self.fs.put(image_file, _id=image_id) image = self._get_file(loc) except Exception: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exception. with excutils.save_and_reraise_exception(): self.fs.delete(image_id) LOG.debug( _("Uploaded image %(id)s, md5 %(md5)s, length %(length)s " "to GridFS") % { 'id': image._id, 'md5': image.md5, 'length': image.length }) return (loc.get_uri(), image.length, image.md5, {})
def schedule_delayed_delete_from_backend(uri, image_id, **kwargs): """Given a uri, schedule the deletion of an image.""" datadir = CONF.scrubber_datadir delete_time = time.time() + CONF.scrub_time file_path = os.path.join(datadir, str(image_id)) utils.safe_mkdirs(datadir) if os.path.exists(file_path): msg = _("Image id %(image_id)s already queued for delete") % { 'image_id': image_id } raise exception.Duplicate(msg) with open(file_path, 'w') as f: f.write('\n'.join([uri, str(int(delete_time))])) os.chmod(file_path, 0600) os.utime(file_path, (delete_time, delete_time))
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns an `glance.store.ImageAddResult` object containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval `glance.store.ImageAddResult` object :raises `glance.common.exception.Duplicate` if the image already existed :note By default, the backend writes the image data to a file `/<DATADIR>/<ID>`, where <DATADIR> is the value of the filesystem_store_datadir configuration option and <ID> is the supplied image ID. """ filepath = os.path.join(self.datadir, str(image_id)) if os.path.exists(filepath): raise exception.Duplicate( _("Image file %s already exists!") % filepath) checksum = hashlib.md5() bytes_written = 0 try: with open(filepath, 'wb') as f: while True: buf = image_file.read(ChunkedFile.CHUNKSIZE) if not buf: break bytes_written += len(buf) checksum.update(buf) f.write(buf) except IOError: raise exception.StorageFull() checksum_hex = checksum.hexdigest() logger.debug( _("Wrote %(bytes_written)d bytes to %(filepath)s with " "checksum %(checksum_hex)s") % locals()) return ('file://%s' % filepath, bytes_written, checksum_hex)
def add(self, image_member): try: self.get(image_member.member_id) except exception.NotFound: pass else: msg = _('The target member %(member_id)s is already ' 'associated with image %(image_id)s.') % { 'member_id': image_member.member_id, 'image_id': self.image.image_id} raise exception.Duplicate(msg) image_member_values = self._format_image_member_to_db(image_member) new_values = self.db_api.image_member_create(self.context, image_member_values) image_member.created_at = new_values['created_at'] image_member.updated_at = new_values['updated_at'] image_member.id = new_values['id']
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns an `glance.store.ImageAddResult` object containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval `glance.store.ImageAddResult` object :raises `glance.common.exception.Duplicate` if the image already existed """ checksum = hashlib.md5() image_name = str(image_id) with rados.Rados(conffile=self.conf_file, rados_id=self.user) as conn: fsid = None if hasattr(conn, 'get_fsid'): fsid = conn.get_fsid() with conn.open_ioctx(self.pool) as ioctx: order = int(math.log(self.chunk_size, 2)) LOG.debug('creating image %s with order %d', image_name, order) try: location = self._create_image(fsid, ioctx, image_name, image_size, order) except rbd.ImageExists: raise exception.Duplicate( _('RBD image %s already exists') % image_id) with rbd.Image(ioctx, image_name) as image: bytes_left = image_size while bytes_left > 0: length = min(self.chunk_size, bytes_left) data = image_file.read(length) image.write(data, image_size - bytes_left) bytes_left -= length checksum.update(data) if location.snapshot: image.create_snap(location.snapshot) image.protect_snap(location.snapshot) return (location.get_uri(), image_size, checksum.hexdigest())
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval tuple of URL in backing store, bytes written, and checksum :raises `glance.common.exception.Duplicate` if the image already existed """ image = SheepdogImage(self.addr, self.port, image_id, self.WRITE_CHUNKSIZE) if image.exist(): raise exception.Duplicate( _("Sheepdog image %s already exists") % image_id) location = StoreLocation({'image': image_id}) checksum = hashlib.md5() image.create(image_size) try: total = left = image_size while left > 0: length = min(self.WRITE_CHUNKSIZE, left) data = image_file.read(length) image.write(data, total - left, length) left -= length checksum.update(data) except Exception: # Note(zhiyan): clean up already received data when # error occurs such as ImageSizeLimitExceeded exception. with excutils.save_and_reraise_exception(): image.delete() return (location.get_uri(), image_size, checksum.hexdigest(), {})
def image_create(context, image_values): global DATA image_id = image_values.get('id', uuidutils.generate_uuid()) if image_id in DATA['images']: raise exception.Duplicate() if 'status' not in image_values: raise exception.Invalid('status is a required attribute') allowed_keys = set(['id', 'name', 'status', 'min_ram', 'min_disk', 'size', 'checksum', 'locations', 'owner', 'protected', 'is_public', 'container_format', 'disk_format', 'created_at', 'updated_at', 'deleted_at', 'deleted', 'properties', 'tags']) if set(image_values.keys()) - allowed_keys: raise exception.Invalid() image = _image_format(image_id, **image_values) DATA['images'][image_id] = image DATA['tags'][image_id] = image.pop('tags', []) return image
def schedule_delete_from_backend(uri, context, image_id, **kwargs): """ Given a uri and a time, schedule the deletion of an image. """ if not CONF.delayed_delete: registry.update_image_metadata(context, image_id, {'status': 'deleted'}) try: return delete_from_backend(context, uri, **kwargs) except (UnsupportedBackend, exception.StoreDeleteNotSupported, exception.NotFound): exc_type = sys.exc_info()[0].__name__ msg = (_("Failed to delete image at %s from store (%s)") % (uri, exc_type)) LOG.error(msg) finally: # avoid falling through to the delayed deletion logic return datadir = CONF.scrubber_datadir delete_time = time.time() + CONF.scrub_time file_path = os.path.join(datadir, str(image_id)) utils.safe_mkdirs(datadir) if os.path.exists(file_path): msg = _("Image id %(image_id)s already queued for delete") % { 'image_id': image_id } raise exception.Duplicate(msg) with open(file_path, 'w') as f: f.write('\n'.join([uri, str(int(delete_time))])) os.chmod(file_path, 0600) os.utime(file_path, (delete_time, delete_time)) registry.update_image_metadata(context, image_id, {'status': 'pending_delete'})
def add(self, image_id, image_file, image_size, connection=None): location = self.create_location(image_id) if not connection: connection = self.get_connection(location) self._create_container_if_missing(location.container, connection) LOG.debug( _("Adding image object '%(obj_name)s' " "to Swift") % dict(obj_name=location.obj)) try: if image_size > 0 and image_size < self.large_object_size: # Image size is known, and is less than large_object_size. # Send to Swift with regular PUT. obj_etag = connection.put_object(location.container, location.obj, image_file, content_length=image_size) else: # Write the image into Swift in chunks. chunk_id = 1 if image_size > 0: total_chunks = str( int( math.ceil( float(image_size) / float(self.large_object_chunk_size)))) else: # image_size == 0 is when we don't know the size # of the image. This can occur with older clients # that don't inspect the payload size. LOG.debug( _("Cannot determine image size. Adding as a " "segmented object to Swift.")) total_chunks = '?' checksum = hashlib.md5() written_chunks = [] combined_chunks_size = 0 while True: chunk_size = self.large_object_chunk_size if image_size == 0: content_length = None else: left = image_size - combined_chunks_size if left == 0: break if chunk_size > left: chunk_size = left content_length = chunk_size chunk_name = "%s-%05d" % (location.obj, chunk_id) reader = ChunkReader(image_file, checksum, chunk_size) try: chunk_etag = connection.put_object( location.container, chunk_name, reader, content_length=content_length) written_chunks.append(chunk_name) except Exception: # Delete orphaned segments from swift backend self._delete_stale_chunks(connection, location.container, written_chunks) raise bytes_read = reader.bytes_read msg = _("Wrote chunk %(chunk_name)s (%(chunk_id)d/" "%(total_chunks)s) of length %(bytes_read)d " "to Swift returning MD5 of content: " "%(chunk_etag)s") LOG.debug(msg % locals()) if bytes_read == 0: # Delete the last chunk, because it's of zero size. # This will happen if size == 0. LOG.debug(_("Deleting final zero-length chunk")) connection.delete_object(location.container, chunk_name) break chunk_id += 1 combined_chunks_size += bytes_read # In the case we have been given an unknown image size, # set the size to the total size of the combined chunks. if image_size == 0: image_size = combined_chunks_size # Now we write the object manifest and return the # manifest's etag... manifest = "%s/%s-" % (location.container, location.obj) headers = { 'ETag': hashlib.md5("").hexdigest(), 'X-Object-Manifest': manifest } # The ETag returned for the manifest is actually the # MD5 hash of the concatenated checksums of the strings # of each chunk...so we ignore this result in favour of # the MD5 of the entire image file contents, so that # users can verify the image file contents accordingly connection.put_object(location.container, location.obj, None, headers=headers) obj_etag = checksum.hexdigest() # NOTE: We return the user and key here! Have to because # location is used by the API server to return the actual # image data. We *really* should consider NOT returning # the location attribute from GET /images/<ID> and # GET /images/details return (location.get_uri(), image_size, obj_etag) except swiftclient.ClientException as e: if e.http_status == httplib.CONFLICT: raise exception.Duplicate( _("Swift already has an image at " "this location")) msg = (_("Failed to add object to Swift.\n" "Got error from Swift: %(e)s") % locals()) LOG.error(msg) raise glance.store.BackendException(msg)
def data_iterator(): self.notifier.log = [] yield 'abcde' raise exception.Duplicate('Cant have duplicates')
def _image_update(context, values, image_id, purge_props=False): """ Used internally by image_create and image_update :param context: Request context :param values: A dict of attributes to set :param image_id: If None, create the image, otherwise, find and update it """ session = _get_session() with session.begin(): # Remove the properties passed in the values mapping. We # handle properties separately from base image attributes, # and leaving properties in the values mapping will cause # a SQLAlchemy model error because SQLAlchemy expects the # properties attribute of an Image model to be a list and # not a dict. properties = values.pop('properties', {}) try: locations = values.pop('locations') locations_provided = True except KeyError: locations_provided = False if image_id: image_ref = _image_get(context, image_id, session=session) # Perform authorization check _check_mutate_authorization(context, image_ref) else: if values.get('size') is not None: values['size'] = int(values['size']) if 'min_ram' in values: values['min_ram'] = int(values['min_ram'] or 0) if 'min_disk' in values: values['min_disk'] = int(values['min_disk'] or 0) values['is_public'] = bool(values.get('is_public', False)) values['protected'] = bool(values.get('protected', False)) image_ref = models.Image() # Need to canonicalize ownership if 'owner' in values and not values['owner']: values['owner'] = None if image_id: # Don't drop created_at if we're passing it in... _drop_protected_attrs(models.Image, values) #NOTE(iccha-sethi): updated_at must be explicitly set in case # only ImageProperty table was modifited values['updated_at'] = timeutils.utcnow() image_ref.update(values) # Validate the attributes before we go any further. From my # investigation, the @validates decorator does not validate # on new records, only on existing records, which is, well, # idiotic. values = _validate_image(image_ref.to_dict()) _update_values(image_ref, values) try: image_ref.save(session=session) except sqlalchemy.exc.IntegrityError: raise exception.Duplicate("Image ID %s already exists!" % values['id']) _set_properties_for_image(context, image_ref, properties, purge_props, session) if locations_provided: _image_locations_set(image_ref.id, locations, session) return image_get(context, image_ref.id)
def add(self, image_id, image_file, image_size): """ Stores an image file with supplied identifier to the backend storage system and returns an `glance.store.ImageAddResult` object containing information about the stored image. :param image_id: The opaque image identifier :param image_file: The image data to write, as a file-like object :param image_size: The size of the image data to write, in bytes :retval `glance.store.ImageAddResult` object :raises `glance.common.exception.Duplicate` if the image already existed Swift writes the image data using the scheme: ``swift://<USER>:<KEY>@<AUTH_ADDRESS>/<CONTAINER>/<ID>` where: <USER> = ``swift_store_user`` <KEY> = ``swift_store_key`` <AUTH_ADDRESS> = ``swift_store_auth_address`` <CONTAINER> = ``swift_store_container`` <ID> = The id of the image being added :note Swift auth URLs by default use HTTPS. To specify an HTTP auth URL, you can specify http://someurl.com for the swift_store_auth_address config option :note Swift cannot natively/transparently handle objects >5GB in size. So, if the image is greater than 5GB, we write chunks of image data to Swift and then write an manifest to Swift that contains information about the chunks. This same chunking process is used by default for images of an unknown size, as pushing them directly to swift would fail if the image turns out to be greater than 5GB. """ swift_conn = self._make_swift_connection(self.full_auth_address, self.user, self.key, storage_url=self.storage_url, token=self.token) obj_name = str(image_id) if self.multi_tenant: # NOTE: When using multi-tenant we create containers for each # image so we can set permissions on each image in swift container = self.container + '_' + obj_name auth_or_store_url = self.storage_url else: container = self.container auth_or_store_url = self.auth_address create_container_if_missing(container, swift_conn) location = StoreLocation({ 'scheme': self.scheme, 'container': container, 'obj': obj_name, 'auth_or_store_url': auth_or_store_url, 'user': self.user, 'key': self.key }) LOG.debug( _("Adding image object '%(obj_name)s' " "to Swift") % locals()) try: if image_size > 0 and image_size < self.large_object_size: # Image size is known, and is less than large_object_size. # Send to Swift with regular PUT. obj_etag = swift_conn.put_object(container, obj_name, image_file, content_length=image_size) else: # Write the image into Swift in chunks. chunk_id = 1 if image_size > 0: total_chunks = str( int( math.ceil( float(image_size) / float(self.large_object_chunk_size)))) else: # image_size == 0 is when we don't know the size # of the image. This can occur with older clients # that don't inspect the payload size. LOG.debug( _("Cannot determine image size. Adding as a " "segmented object to Swift.")) total_chunks = '?' checksum = hashlib.md5() combined_chunks_size = 0 while True: chunk_size = self.large_object_chunk_size if image_size == 0: content_length = None else: left = image_size - combined_chunks_size if left == 0: break if chunk_size > left: chunk_size = left content_length = chunk_size chunk_name = "%s-%05d" % (obj_name, chunk_id) reader = ChunkReader(image_file, checksum, chunk_size) chunk_etag = swift_conn.put_object( container, chunk_name, reader, content_length=content_length) bytes_read = reader.bytes_read msg = _("Wrote chunk %(chunk_name)s (%(chunk_id)d/" "%(total_chunks)s) of length %(bytes_read)d " "to Swift returning MD5 of content: " "%(chunk_etag)s") LOG.debug(msg % locals()) if bytes_read == 0: # Delete the last chunk, because it's of zero size. # This will happen if image_size == 0. LOG.debug(_("Deleting final zero-length chunk")) swift_conn.delete_object(container, chunk_name) break chunk_id += 1 combined_chunks_size += bytes_read # In the case we have been given an unknown image size, # set the image_size to the total size of the combined chunks. if image_size == 0: image_size = combined_chunks_size # Now we write the object manifest and return the # manifest's etag... manifest = "%s/%s" % (container, obj_name) headers = { 'ETag': hashlib.md5("").hexdigest(), 'X-Object-Manifest': manifest } # The ETag returned for the manifest is actually the # MD5 hash of the concatenated checksums of the strings # of each chunk...so we ignore this result in favour of # the MD5 of the entire image file contents, so that # users can verify the image file contents accordingly swift_conn.put_object(container, obj_name, None, headers=headers) obj_etag = checksum.hexdigest() # NOTE: We return the user and key here! Have to because # location is used by the API server to return the actual # image data. We *really* should consider NOT returning # the location attribute from GET /images/<ID> and # GET /images/details return (location.get_uri(), image_size, obj_etag) except swiftclient.ClientException, e: if e.http_status == httplib.CONFLICT: raise exception.Duplicate( _("Swift already has an image at " "location %s") % location.get_uri()) msg = (_("Failed to add object to Swift.\n" "Got error from Swift: %(e)s") % locals()) LOG.error(msg) raise glance.store.BackendException(msg)
def _do_request(self, method, url, body, headers): """ Connects to the server and issues a request. Handles converting any returned HTTP error status codes to OpenStack/Glance exceptions and closing the server connection. Returns the result data, or raises an appropriate exception. :param method: HTTP method ("GET", "POST", "PUT", etc...) :param url: urlparse.ParsedResult object with URL information :param body: data to send (as string, filelike or iterable), or None (default) :param headers: mapping of key/value pairs to add as headers :note If the body param has a read attribute, and method is either POST or PUT, this method will automatically conduct a chunked-transfer encoding and use the body as a file object or iterable, transferring chunks of data using the connection's send() method. This allows large objects to be transferred efficiently without buffering the entire body in memory. """ if url.query: path = url.path + "?" + url.query else: path = url.path try: connection_type = self.get_connection_type() headers = self._encode_headers(headers or {}) if 'x-auth-token' not in headers and self.auth_tok: headers['x-auth-token'] = self.auth_tok c = connection_type(url.hostname, url.port, **self.connect_kwargs) def _pushing(method): return method.lower() in ('post', 'put') def _simple(body): return body is None or isinstance(body, six.string_types) def _filelike(body): return hasattr(body, 'read') def _sendbody(connection, iter): connection.endheaders() for sent in iter: # iterator has done the heavy lifting pass def _chunkbody(connection, iter): connection.putheader('Transfer-Encoding', 'chunked') connection.endheaders() for chunk in iter: connection.send('%x\r\n%s\r\n' % (len(chunk), chunk)) connection.send('0\r\n\r\n') # Do a simple request or a chunked request, depending # on whether the body param is file-like or iterable and # the method is PUT or POST # if not _pushing(method) or _simple(body): # Simple request... c.request(method, path, body, headers) elif _filelike(body) or self._iterable(body): c.putrequest(method, path) use_sendfile = self._sendable(body) # According to HTTP/1.1, Content-Length and Transfer-Encoding # conflict. for header, value in headers.items(): if use_sendfile or header.lower() != 'content-length': c.putheader(header, str(value)) iter = self.image_iterator(c, headers, body) if use_sendfile: # send actual file without copying into userspace _sendbody(c, iter) else: # otherwise iterate and chunk _chunkbody(c, iter) else: raise TypeError('Unsupported image type: %s' % body.__class__) res = c.getresponse() def _retry(res): return res.getheader('Retry-After') status_code = self.get_status_code(res) if status_code in self.OK_RESPONSE_CODES: return res elif status_code in self.REDIRECT_RESPONSE_CODES: raise exception.RedirectException(res.getheader('Location')) elif status_code == httplib.UNAUTHORIZED: raise exception.NotAuthenticated(res.read()) elif status_code == httplib.FORBIDDEN: raise exception.Forbidden(res.read()) elif status_code == httplib.NOT_FOUND: raise exception.NotFound(res.read()) elif status_code == httplib.CONFLICT: raise exception.Duplicate(res.read()) elif status_code == httplib.BAD_REQUEST: raise exception.Invalid(res.read()) elif status_code == httplib.MULTIPLE_CHOICES: raise exception.MultipleChoices(body=res.read()) elif status_code == httplib.REQUEST_ENTITY_TOO_LARGE: raise exception.LimitExceeded(retry=_retry(res), body=res.read()) elif status_code == httplib.INTERNAL_SERVER_ERROR: raise exception.ServerError() elif status_code == httplib.SERVICE_UNAVAILABLE: raise exception.ServiceUnavailable(retry=_retry(res)) else: raise exception.UnexpectedStatus(status=status_code, body=res.read()) except (socket.error, IOError) as e: raise exception.ClientConnectionError(e)