def delete(self, req, id):
"""Deletes an existing image with the registry.
:param req: wsgi Request object
:param id: The opaque internal identifier for the image
:returns: 200 if delete was successful, a fault if not. On
success, the body contains the deleted image
information as a mapping.
"""
try:
deleted_image = self.db_api.image_destroy(req.context, id)
LOG.info(_LI("Successfully deleted image %(id)s"), {'id': id})
return dict(image=make_image_dict(deleted_image))
except exception.ForbiddenPublicImage:
LOG.info(_LI("Delete denied for public image %(id)s"), {'id': id})
raise exc.HTTPForbidden()
except exception.Forbidden:
# If it's private and doesn't belong to them, don't let on
# that it exists
LOG.info(
_LI("Access denied to image %(id)s but returning"
" 'not found'"), {'id': id})
return exc.HTTPNotFound()
except exception.ImageNotFound:
LOG.info(_LI("Image %(id)s not found"), {'id': id})
return exc.HTTPNotFound()
except Exception:
LOG.exception(_LE("Unable to delete image %s") % id)
raise
def delete(self, req, id):
"""Deletes an existing image with the registry.
:param req: wsgi Request object
:param id: The opaque internal identifier for the image
:returns: 200 if delete was successful, a fault if not. On
success, the body contains the deleted image
information as a mapping.
"""
try:
deleted_image = self.db_api.image_destroy(req.context, id)
LOG.info(_LI("Successfully deleted image %(id)s"), {'id': id})
return dict(image=make_image_dict(deleted_image))
except exception.ForbiddenPublicImage:
LOG.info(_LI("Delete denied for public image %(id)s"), {'id': id})
raise exc.HTTPForbidden()
except exception.Forbidden:
# If it's private and doesn't belong to them, don't let on
# that it exists
LOG.info(_LI("Access denied to image %(id)s but returning"
" 'not found'"), {'id': id})
return exc.HTTPNotFound()
except exception.ImageNotFound:
LOG.info(_LI("Image %(id)s not found"), {'id': id})
return exc.HTTPNotFound()
except Exception:
LOG.exception(_LE("Unable to delete image %s") % id)
raise
def set_image_data(image, uri, task_id):
data_iter = None
try:
LOG.info(
_LI("Task %(task_id)s: Got image data uri %(data_uri)s to be "
"imported"), {
"data_uri": uri,
"task_id": task_id
})
data_iter = script_utils.get_image_data_iter(uri)
image.set_data(data_iter)
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.warn(
_LW("Task %(task_id)s failed with exception %(error)s") % {
"error": encodeutils.exception_to_unicode(e),
"task_id": task_id
})
LOG.info(
_LI("Task %(task_id)s: Could not import image file"
" %(image_data)s"), {
"image_data": uri,
"task_id": task_id
})
finally:
if hasattr(data_iter, 'close'):
data_iter.close()
def queue_image(self, image_id):
"""
This adds a image to be cache to the queue.
If the image already exists in the queue or has already been
cached, we return False, True otherwise
:param image_id: Image ID
"""
if self.is_cached(image_id):
LOG.info(_LI("Not queueing image '%s'. Already cached."), image_id)
return False
if self.is_being_cached(image_id):
LOG.info(_LI("Not queueing image '%s'. Already being "
"written to cache"), image_id)
return False
if self.is_queued(image_id):
LOG.info(_LI("Not queueing image '%s'. Already queued."), image_id)
return False
path = self.get_image_filepath(image_id, 'queue')
# Touch the file to add it to the queue
with open(path, "w"):
pass
return True
def queue_image(self, image_id):
"""
This adds a image to be cache to the queue.
If the image already exists in the queue or has already been
cached, we return False, True otherwise
:param image_id: Image ID
"""
if self.is_cached(image_id):
LOG.info(_LI("Not queueing image '%s'. Already cached."), image_id)
return False
if self.is_being_cached(image_id):
LOG.info(
_LI("Not queueing image '%s'. Already being "
"written to cache"), image_id)
return False
if self.is_queued(image_id):
LOG.info(_LI("Not queueing image '%s'. Already queued."), image_id)
return False
path = self.get_image_filepath(image_id, 'queue')
# Touch the file to add it to the queue
with open(path, "w"):
pass
return True
def run_child(self):
def child_hup(*args):
"""Shuts down child processes, existing requests are handled."""
signal.signal(signal.SIGHUP, signal.SIG_IGN)
eventlet.wsgi.is_accepting = False
self.sock.close()
pid = os.fork()
if pid == 0:
signal.signal(signal.SIGHUP, child_hup)
signal.signal(signal.SIGTERM, signal.SIG_DFL)
# ignore the interrupt signal to avoid a race whereby
# a child worker receives the signal before the parent
# and is respawned unnecessarily as a result
signal.signal(signal.SIGINT, signal.SIG_IGN)
# The child has no need to stash the unwrapped
# socket, and the reference prevents a clean
# exit on sighup
self._sock = None
self.run_server()
LOG.info(_LI('Child %d exiting normally'), os.getpid())
# self.pool.waitall() is now called in wsgi's server so
# it's safe to exit here
sys.exit(0)
else:
LOG.info(_LI('Started child %s'), pid)
self.children.add(pid)
def _remove_children(self, pid):
if pid in self.children:
self.children.remove(pid)
LOG.info(_LI('Removed dead child %s'), pid)
elif pid in self.stale_children:
self.stale_children.remove(pid)
LOG.info(_LI('Removed stale child %s'), pid)
else:
LOG.warn(_LW('Unrecognised child %s') % pid)
def create(self, req, body):
"""Registers a new image with the registry.
:param req: wsgi Request object
:param body: Dictionary of information about the image
:returns: The newly-created image information as a mapping,
which will include the newly-created image's internal id
in the 'id' field
"""
image_data = body['image']
# Ensure the image has a status set
image_data.setdefault('status', 'active')
# Set up the image owner
if not req.context.is_admin or 'owner' not in image_data:
image_data['owner'] = req.context.owner
image_id = image_data.get('id')
if image_id and not uuidutils.is_uuid_like(image_id):
LOG.info(
_LI("Rejecting image creation request for invalid image "
"id '%(bad_id)s'"), {'bad_id': image_id})
msg = _("Invalid image id format")
return exc.HTTPBadRequest(explanation=msg)
if 'location' in image_data:
image_data['locations'] = [image_data.pop('location')]
try:
image_data = _normalize_image_location_for_db(image_data)
image_data = self.db_api.image_create(req.context, image_data)
image_data = dict(image=make_image_dict(image_data))
LOG.info(_LI("Successfully created image %(id)s"),
{'id': image_data['image']['id']})
return image_data
except exception.Duplicate:
msg = _("Image with identifier %s already exists!") % image_id
LOG.warn(msg)
return exc.HTTPConflict(msg)
except exception.Invalid as e:
msg = (_("Failed to add image metadata. "
"Got error: %s") % encodeutils.exception_to_unicode(e))
LOG.error(msg)
return exc.HTTPBadRequest(msg)
except Exception:
LOG.exception(_LE("Unable to create image %s"), image_id)
raise
def create(self, req, body):
"""Registers a new image with the registry.
:param req: wsgi Request object
:param body: Dictionary of information about the image
:returns: The newly-created image information as a mapping,
which will include the newly-created image's internal id
in the 'id' field
"""
image_data = body['image']
# Ensure the image has a status set
image_data.setdefault('status', 'active')
# Set up the image owner
if not req.context.is_admin or 'owner' not in image_data:
image_data['owner'] = req.context.owner
image_id = image_data.get('id')
if image_id and not uuidutils.is_uuid_like(image_id):
LOG.info(_LI("Rejecting image creation request for invalid image "
"id '%(bad_id)s'"), {'bad_id': image_id})
msg = _("Invalid image id format")
return exc.HTTPBadRequest(explanation=msg)
if 'location' in image_data:
image_data['locations'] = [image_data.pop('location')]
try:
image_data = _normalize_image_location_for_db(image_data)
image_data = self.db_api.image_create(req.context, image_data)
image_data = dict(image=make_image_dict(image_data))
LOG.info(_LI("Successfully created image %(id)s"),
{'id': image_data['image']['id']})
return image_data
except exception.Duplicate:
msg = _("Image with identifier %s already exists!") % image_id
LOG.warn(msg)
return exc.HTTPConflict(msg)
except exception.Invalid as e:
msg = (_("Failed to add image metadata. "
"Got error: %s") % encodeutils.exception_to_unicode(e))
LOG.error(msg)
return exc.HTTPBadRequest(msg)
except Exception:
LOG.exception(_LE("Unable to create image %s"), image_id)
raise
def __init__(self, store_api):
LOG.info(_LI("Initializing scrubber with configuration: %s"),
six.text_type({'registry_host': CONF.registry_host,
'registry_port': CONF.registry_port}))
self.store_api = store_api
registry.configure_registry_client()
registry.configure_registry_admin_creds()
# Here we create a request context with credentials to support
# delayed delete when using multi-tenant backend storage
admin_user = CONF.admin_user
admin_tenant = CONF.admin_tenant_name
if CONF.send_identity_headers:
# When registry is operating in trusted-auth mode
roles = [CONF.admin_role]
self.admin_context = context.RequestContext(user=admin_user,
tenant=admin_tenant,
auth_token=None,
roles=roles)
self.registry = registry.get_registry_client(self.admin_context)
else:
ctxt = context.RequestContext()
self.registry = registry.get_registry_client(ctxt)
auth_token = self.registry.auth_token
self.admin_context = context.RequestContext(user=admin_user,
tenant=admin_tenant,
auth_token=auth_token)
self.db_queue = get_scrub_queue()
self.pool = eventlet.greenpool.GreenPool(CONF.scrub_pool_size)
def run_task(task_id, task_type, context,
task_repo=None, image_repo=None, image_factory=None):
# TODO(nikhil): if task_repo is None get new task repo
# TODO(nikhil): if image_repo is None get new image repo
# TODO(nikhil): if image_factory is None get new image factory
LOG.info(_LI("Loading known task scripts for task_id %(task_id)s "
"of type %(task_type)s"), {'task_id': task_id,
'task_type': task_type})
if task_type == 'import':
image_import.run(task_id, context, task_repo,
image_repo, image_factory)
else:
msg = _LE("This task type %(task_type)s is not supported by the "
"current deployment of Glance. Please refer the "
"documentation provided by OpenStack or your operator "
"for more information.") % {'task_type': task_type}
LOG.error(msg)
task = task_repo.get(task_id)
task.fail(msg)
if task_repo:
task_repo.save(task)
else:
LOG.error(_LE("Failed to save task %(task_id)s in DB as task_repo "
"is %(task_repo)s"), {"task_id": task_id,
"task_repo": task_repo})
def delete_invalid_files(self):
"""
Removes any invalid cache entries
"""
for path in self.get_cache_files(self.invalid_dir):
os.unlink(path)
LOG.info(_LI("Removed invalid cache file %s"), path)
def delete_invalid_files(self):
"""
Removes any invalid cache entries
"""
for path in self.get_cache_files(self.invalid_dir):
os.unlink(path)
LOG.info(_LI("Removed invalid cache file %s"), path)
def execute(self, image_id):
"""Finishing the task flow
:param image_id: Glance Image ID
"""
task = script_utils.get_task(self.task_repo, self.task_id)
if task is None:
return
try:
task.succeed({'image_id': image_id})
except Exception as e:
# Note: The message string contains Error in it to indicate
# in the task.message that it's a error message for the user.
# TODO(nikhil): need to bring back save_and_reraise_exception when
# necessary
err_msg = ("Error: " + six.text_type(type(e)) + ': ' +
encodeutils.exception_to_unicode(e))
log_msg = err_msg + _LE("Task ID %s") % task.task_id
LOG.exception(log_msg)
task.fail(err_msg)
finally:
self.task_repo.save(task)
LOG.info(_LI("%(task_id)s of %(task_type)s completed"),
{'task_id': self.task_id, 'task_type': self.task_type})
def execute(self, image_id):
"""Finishing the task flow
:param image_id: Glance Image ID
"""
task = script_utils.get_task(self.task_repo, self.task_id)
if task is None:
return
try:
task.succeed({'image_id': image_id})
except Exception as e:
# Note: The message string contains Error in it to indicate
# in the task.message that it's a error message for the user.
# TODO(nikhil): need to bring back save_and_reraise_exception when
# necessary
err_msg = ("Error: " + six.text_type(type(e)) + ': ' +
encodeutils.exception_to_unicode(e))
log_msg = err_msg + _LE("Task ID %s") % task.task_id
LOG.exception(log_msg)
task.fail(err_msg)
finally:
self.task_repo.save(task)
LOG.info(_LI("%(task_id)s of %(task_type)s completed"), {
'task_id': self.task_id,
'task_type': self.task_type
})
def __init__(self, wakeup_time=300, threads=100):
LOG.info(_LI("Starting Daemon: wakeup_time=%(wakeup_time)s "
"threads=%(threads)s"),
{'wakeup_time': wakeup_time, 'threads': threads})
self.wakeup_time = wakeup_time
self.event = eventlet.event.Event()
# This pool is used for periodic instantiation of scrubber
self.daemon_pool = eventlet.greenpool.GreenPool(threads)
def set_image_data(image, uri, task_id):
data_iter = None
try:
LOG.info(_LI("Task %(task_id)s: Got image data uri %(data_uri)s to be "
"imported"), {"data_uri": uri, "task_id": task_id})
data_iter = script_utils.get_image_data_iter(uri)
image.set_data(data_iter)
except Exception as e:
with excutils.save_and_reraise_exception():
LOG.warn(_LW("Task %(task_id)s failed with exception %(error)s") %
{"error": encodeutils.exception_to_unicode(e),
"task_id": task_id})
LOG.info(_LI("Task %(task_id)s: Could not import image file"
" %(image_data)s"), {"image_data": uri,
"task_id": task_id})
finally:
if hasattr(data_iter, 'close'):
data_iter.close()
def show(self, req, id):
"""Return data about the given image id."""
try:
image = self.db_api.image_get(req.context, id)
LOG.debug("Successfully retrieved image %(id)s", {'id': id})
except exception.ImageNotFound:
LOG.info(_LI("Image %(id)s not found"), {'id': id})
raise exc.HTTPNotFound()
except exception.Forbidden:
# If it's private and doesn't belong to them, don't let on
# that it exists
LOG.info(_LI("Access denied to image %(id)s but returning"
" 'not found'"), {'id': id})
raise exc.HTTPNotFound()
except Exception:
LOG.exception(_LE("Unable to show image %s") % id)
raise
return dict(image=make_image_dict(image))
def _single_run(self, application, sock):
"""Start a WSGI server in a new green thread."""
LOG.info(_LI("Starting single process server"))
eventlet.wsgi.server(sock,
application,
custom_pool=self.pool,
log=self._logger,
debug=False,
keepalive=CONF.http_keepalive,
socket_timeout=self.client_socket_timeout)
def show(self, req, id):
"""Return data about the given image id."""
try:
image = self.db_api.image_get(req.context, id)
LOG.debug("Successfully retrieved image %(id)s", {'id': id})
except exception.ImageNotFound:
LOG.info(_LI("Image %(id)s not found"), {'id': id})
raise exc.HTTPNotFound()
except exception.Forbidden:
# If it's private and doesn't belong to them, don't let on
# that it exists
LOG.info(
_LI("Access denied to image %(id)s but returning"
" 'not found'"), {'id': id})
raise exc.HTTPNotFound()
except Exception:
LOG.exception(_LE("Unable to show image %s") % id)
raise
return dict(image=make_image_dict(image))
def init_driver(self):
"""
Create the driver for the cache
"""
driver_name = CONF.image_cache_driver
driver_module = (__name__ + '.drivers.' + driver_name + '.Driver')
try:
self.driver_class = importutils.import_class(driver_module)
LOG.info(_LI("Image cache loaded driver '%s'."), driver_name)
except ImportError as import_err:
LOG.warn(_LW("Image cache driver "
"'%(driver_name)s' failed to load. "
"Got error: '%(import_err)s."),
{'driver_name': driver_name,
'import_err': import_err})
driver_module = __name__ + '.drivers.sqlite.Driver'
LOG.info(_LI("Defaulting to SQLite driver."))
self.driver_class = importutils.import_class(driver_module)
self.configure_driver()
def _check_ext(ext):
try:
next(n for n, v in available
if n == ext.plugin.metadata.type_name and (
v is None or v == ext.plugin.metadata.type_version))
except StopIteration:
LOG.warn(
_LW("Can't load artifact %s: not in"
" available_plugins list") % ext.name)
raise exception.ArtifactLoadError(name=ext.name)
LOG.info(_LI("Artifact %s has been successfully loaded"), ext.name)
return True
def _check_ext(ext):
try:
next(n for n, v in available
if n == ext.plugin.metadata.type_name and
(v is None or v == ext.plugin.metadata.type_version))
except StopIteration:
LOG.warn(_LW("Can't load artifact %s: not in"
" available_plugins list") % ext.name)
raise exception.ArtifactLoadError(name=ext.name)
LOG.info(
_LI("Artifact %s has been successfully loaded"), ext.name)
return True
def _clear_metadata(meta):
metadef_tables = [
get_metadef_properties_table(meta),
get_metadef_objects_table(meta),
get_metadef_tags_table(meta),
get_metadef_namespace_resource_types_table(meta),
get_metadef_namespaces_table(meta),
get_metadef_resource_types_table(meta)
]
for table in metadef_tables:
table.delete().execute()
LOG.info(_LI("Table %s has been cleared"), table)
def _verify_and_respawn_children(self, pid, status):
if len(self.stale_children) == 0:
LOG.debug('No stale children')
if os.WIFEXITED(status) and os.WEXITSTATUS(status) != 0:
LOG.error(
_LE('Not respawning child %d, cannot '
'recover from termination') % pid)
if not self.children and not self.stale_children:
LOG.info(_LI('All workers have terminated. Exiting'))
self.running = False
else:
if len(self.children) < get_num_workers():
self.run_child()
def _scrub_image(self, image_id, delete_jobs):
if len(delete_jobs) == 0:
return
LOG.info(_LI("Scrubbing image %(id)s from %(count)d locations."),
{'id': image_id, 'count': len(delete_jobs)})
success = True
for img_id, loc_id, uri in delete_jobs:
try:
self._delete_image_location_from_backend(img_id, loc_id, uri)
except Exception:
success = False
if success:
image = self.registry.get_image(image_id)
if image['status'] == 'pending_delete':
self.registry.update_image(image_id, {'status': 'deleted'})
LOG.info(_LI("Image %s has been scrubbed successfully"), image_id)
else:
LOG.warn(_LW("One or more image locations couldn't be scrubbed "
"from backend. Leaving image '%s' in 'pending_delete'"
" status") % image_id)
def start_wsgi(self):
workers = get_num_workers()
if workers == 0:
# Useful for profiling, test, debug etc.
self.pool = self.create_pool()
self.pool.spawn_n(self._single_run, self.application, self.sock)
return
else:
LOG.info(_LI("Starting %d workers"), workers)
signal.signal(signal.SIGTERM, self.kill_children)
signal.signal(signal.SIGINT, self.kill_children)
signal.signal(signal.SIGHUP, self.hup)
while len(self.children) < workers:
self.run_child()
def reactivate(self, req, image_id):
image_repo = self.gateway.get_repo(req.context)
try:
image = image_repo.get(image_id)
image.reactivate()
image_repo.save(image)
LOG.info(_LI("Image %s is reactivated"), image_id)
except exception.NotFound as e:
raise webob.exc.HTTPNotFound(explanation=e.msg)
except exception.Forbidden as e:
LOG.debug("User not permitted to reactivate image '%s'", image_id)
raise webob.exc.HTTPForbidden(explanation=e.msg)
except exception.InvalidImageStatusTransition as e:
raise webob.exc.HTTPBadRequest(explanation=e.msg)
def _delete_image_location_from_backend(self, image_id, loc_id, uri):
if CONF.metadata_encryption_key:
uri = crypt.urlsafe_decrypt(CONF.metadata_encryption_key, uri)
try:
LOG.debug("Scrubbing image %s from a location.", image_id)
try:
self.store_api.delete_from_backend(uri, self.admin_context)
except store_exceptions.NotFound:
LOG.info(_LI("Image location for image '%s' not found in "
"backend; Marking image location deleted in "
"db."), image_id)
if loc_id != '-':
db_api.get_api().image_location_delete(self.admin_context,
image_id,
int(loc_id),
'deleted')
LOG.info(_LI("Image %s is scrubbed from a location."), image_id)
except Exception as e:
LOG.error(_LE("Unable to scrub image %(id)s from a location. "
"Reason: %(exc)s ") %
{'id': image_id,
'exc': encodeutils.exception_to_unicode(e)})
raise
def set_data(self, data, size=None):
remaining = xmonitor.api.common.check_quota(
self.context, size, self.db_api, image_id=self.image.image_id)
if remaining is not None:
# NOTE(jbresnah) we are trying to enforce a quota, put a limit
# reader on the data
data = utils.LimitingReader(data, remaining)
try:
self.image.set_data(data, size=size)
except exception.ImageSizeLimitExceeded:
raise exception.StorageQuotaFull(image_size=size,
remaining=remaining)
# NOTE(jbresnah) If two uploads happen at the same time and neither
# properly sets the size attribute[1] then there is a race condition
# that will allow for the quota to be broken[2]. Thus we must recheck
# the quota after the upload and thus after we know the size.
#
# Also, when an upload doesn't set the size properly then the call to
# check_quota above returns None and so utils.LimitingReader is not
# used above. Hence the store (e.g. filesystem store) may have to
# download the entire file before knowing the actual file size. Here
# also we need to check for the quota again after the image has been
# downloaded to the store.
#
# [1] For e.g. when using chunked transfers the 'Content-Length'
# header is not set.
# [2] For e.g.:
# - Upload 1 does not exceed quota but upload 2 exceeds quota.
# Both uploads are to different locations
# - Upload 2 completes before upload 1 and writes image.size.
# - Immediately, upload 1 completes and (over)writes image.size
# with the smaller size.
# - Now, to xmonitor, image has not exceeded quota but, in
# reality, the quota has been exceeded.
try:
xmonitor.api.common.check_quota(self.context,
self.image.size,
self.db_api,
image_id=self.image.image_id)
except exception.StorageQuotaFull:
with excutils.save_and_reraise_exception():
LOG.info(_LI('Cleaning up %s after exceeding the quota.'),
self.image.image_id)
self.store_utils.safe_delete_from_backend(
self.context, self.image.image_id, self.image.locations[0])
def replication_dump(options, args):
"""%(prog)s dump <server:port> <path>
Dump the contents of a xmonitor instance to local disk.
server:port: the location of the xmonitor instance.
path: a directory on disk to contain the data.
"""
# Make sure server and path are provided
if len(args) < 2:
raise TypeError(_("Too few arguments."))
path = args.pop()
server, port = utils.parse_valid_host_port(args.pop())
imageservice = get_image_service()
client = imageservice(http_client.HTTPConnection(server, port),
options.mastertoken)
for image in client.get_images():
LOG.debug('Considering: %s', image['id'])
data_path = os.path.join(path, image['id'])
if not os.path.exists(data_path):
LOG.info(_LI('Storing: %s'), image['id'])
# Dump xmonitor information
if six.PY3:
f = open(data_path, 'w', encoding='utf-8')
else:
f = open(data_path, 'w')
with f:
f.write(jsonutils.dumps(image))
if image['status'] == 'active' and not options.metaonly:
# Now fetch the image. The metadata returned in headers here
# is the same as that which we got from the detailed images
# request earlier, so we can ignore it here. Note that we also
# only dump active images.
LOG.debug('Image %s is active', image['id'])
image_response = client.get_image(image['id'])
with open(data_path + '.img', 'wb') as f:
while True:
chunk = image_response.read(options.chunksize)
if not chunk:
break
f.write(chunk)
def _set_task_status(self, new_status):
if self._validate_task_status_transition(self.status, new_status):
self._status = new_status
LOG.info(_LI("Task [%(task_id)s] status changing from "
"%(cur_status)s to %(new_status)s"),
{'task_id': self.task_id, 'cur_status': self.status,
'new_status': new_status})
self._status = new_status
else:
LOG.error(_LE("Task [%(task_id)s] status failed to change from "
"%(cur_status)s to %(new_status)s"),
{'task_id': self.task_id, 'cur_status': self.status,
'new_status': new_status})
raise exception.InvalidTaskStatusTransition(
cur_status=self.status,
new_status=new_status
)
def delete_stalled_files(self, older_than):
"""
Removes any incomplete cache entries older than a
supplied modified time.
:param older_than: Files written to on or before this timestamp
will be deleted.
"""
for path in self.get_cache_files(self.incomplete_dir):
if os.path.getmtime(path) < older_than:
try:
os.unlink(path)
LOG.info(_LI("Removed stalled cache file %s"), path)
except Exception as e:
msg = (_LW("Failed to delete file %(path)s. "
"Got error: %(e)s"), dict(path=path, e=e))
LOG.warn(msg)
def set_data(self, data, size=None):
remaining = xmonitor.api.common.check_quota(
self.context, size, self.db_api, image_id=self.image.image_id)
if remaining is not None:
# NOTE(jbresnah) we are trying to enforce a quota, put a limit
# reader on the data
data = utils.LimitingReader(data, remaining)
try:
self.image.set_data(data, size=size)
except exception.ImageSizeLimitExceeded:
raise exception.StorageQuotaFull(image_size=size,
remaining=remaining)
# NOTE(jbresnah) If two uploads happen at the same time and neither
# properly sets the size attribute[1] then there is a race condition
# that will allow for the quota to be broken[2]. Thus we must recheck
# the quota after the upload and thus after we know the size.
#
# Also, when an upload doesn't set the size properly then the call to
# check_quota above returns None and so utils.LimitingReader is not
# used above. Hence the store (e.g. filesystem store) may have to
# download the entire file before knowing the actual file size. Here
# also we need to check for the quota again after the image has been
# downloaded to the store.
#
# [1] For e.g. when using chunked transfers the 'Content-Length'
# header is not set.
# [2] For e.g.:
# - Upload 1 does not exceed quota but upload 2 exceeds quota.
# Both uploads are to different locations
# - Upload 2 completes before upload 1 and writes image.size.
# - Immediately, upload 1 completes and (over)writes image.size
# with the smaller size.
# - Now, to xmonitor, image has not exceeded quota but, in
# reality, the quota has been exceeded.
try:
xmonitor.api.common.check_quota(
self.context, self.image.size, self.db_api,
image_id=self.image.image_id)
except exception.StorageQuotaFull:
with excutils.save_and_reraise_exception():
LOG.info(_LI('Cleaning up %s after exceeding the quota.'),
self.image.image_id)
self.store_utils.safe_delete_from_backend(
self.context, self.image.image_id, self.image.locations[0])
def delete(self, req, image_id, id):
"""
Removes a membership from the image.
"""
self._check_can_access_image_members(req.context)
# Make sure the image exists
try:
image = self.db_api.image_get(req.context, image_id)
except exception.NotFound:
msg = _("Image %(id)s not found") % {'id': image_id}
LOG.warn(msg)
raise webob.exc.HTTPNotFound(msg)
except exception.Forbidden:
# If it's private and doesn't belong to them, don't let on
# that it exists
msg = _LW("Access denied to image %(id)s but returning"
" 'not found'") % {'id': image_id}
LOG.warn(msg)
raise webob.exc.HTTPNotFound()
# Can they manipulate the membership?
if not self.is_image_sharable(req.context, image):
msg = (_LW("User lacks permission to share image %(id)s") %
{'id': image_id})
LOG.warn(msg)
msg = _("No permission to share that image")
raise webob.exc.HTTPForbidden(msg)
# Look up an existing membership
members = self.db_api.image_member_find(req.context,
image_id=image_id,
member=id)
if members:
self.db_api.image_member_delete(req.context, members[0]['id'])
else:
LOG.debug("%(id)s is not a member of image %(image_id)s",
{'id': id, 'image_id': image_id})
msg = _("Membership could not be found.")
raise webob.exc.HTTPNotFound(explanation=msg)
# Make an appropriate result
LOG.info(_LI("Successfully deleted a membership from image %(id)s"),
{'id': image_id})
return webob.exc.HTTPNoContent()
def delete_stalled_files(self, older_than):
"""
Removes any incomplete cache entries older than a
supplied modified time.
:param older_than: Files written to on or before this timestamp
will be deleted.
"""
for path in self.get_cache_files(self.incomplete_dir):
if os.path.getmtime(path) < older_than:
try:
os.unlink(path)
LOG.info(_LI("Removed stalled cache file %s"), path)
except Exception as e:
msg = (_LW("Failed to delete file %(path)s. "
"Got error: %(e)s"),
dict(path=path, e=e))
LOG.warn(msg)
def __init__(self, app):
mapper = routes.Mapper()
resource = cached_images.create_resource()
mapper.connect("/v1/cached_images",
controller=resource,
action="get_cached_images",
conditions=dict(method=["GET"]))
mapper.connect("/v1/cached_images/{image_id}",
controller=resource,
action="delete_cached_image",
conditions=dict(method=["DELETE"]))
mapper.connect("/v1/cached_images",
controller=resource,
action="delete_cached_images",
conditions=dict(method=["DELETE"]))
mapper.connect("/v1/queued_images/{image_id}",
controller=resource,
action="queue_image",
conditions=dict(method=["PUT"]))
mapper.connect("/v1/queued_images",
controller=resource,
action="get_queued_images",
conditions=dict(method=["GET"]))
mapper.connect("/v1/queued_images/{image_id}",
controller=resource,
action="delete_queued_image",
conditions=dict(method=["DELETE"]))
mapper.connect("/v1/queued_images",
controller=resource,
action="delete_queued_images",
conditions=dict(method=["DELETE"]))
self._mapper = mapper
self._resource = resource
LOG.info(_LI("Initialized image cache management middleware"))
super(CacheManageFilter, self).__init__(app)
def __init__(self, app):
mapper = routes.Mapper()
resource = cached_images.create_resource()
mapper.connect("/v1/cached_images",
controller=resource,
action="get_cached_images",
conditions=dict(method=["GET"]))
mapper.connect("/v1/cached_images/{image_id}",
controller=resource,
action="delete_cached_image",
conditions=dict(method=["DELETE"]))
mapper.connect("/v1/cached_images",
controller=resource,
action="delete_cached_images",
conditions=dict(method=["DELETE"]))
mapper.connect("/v1/queued_images/{image_id}",
controller=resource,
action="queue_image",
conditions=dict(method=["PUT"]))
mapper.connect("/v1/queued_images",
controller=resource,
action="get_queued_images",
conditions=dict(method=["GET"]))
mapper.connect("/v1/queued_images/{image_id}",
controller=resource,
action="delete_queued_image",
conditions=dict(method=["DELETE"]))
mapper.connect("/v1/queued_images",
controller=resource,
action="delete_queued_images",
conditions=dict(method=["DELETE"]))
self._mapper = mapper
self._resource = resource
LOG.info(_LI("Initialized image cache management middleware"))
super(CacheManageFilter, self).__init__(app)
def wait_on_children(self):
while self.running:
try:
pid, status = os.wait()
if os.WIFEXITED(status) or os.WIFSIGNALED(status):
self._remove_children(pid)
self._verify_and_respawn_children(pid, status)
except OSError as err:
if err.errno not in (errno.EINTR, errno.ECHILD):
raise
except KeyboardInterrupt:
LOG.info(_LI('Caught keyboard interrupt. Exiting.'))
break
except exception.SIGHUPInterrupt:
self.reload()
continue
eventlet.greenio.shutdown_safe(self.sock)
self.sock.close()
LOG.debug('Exited')
def set_data(self, data, size=None):
if size is None:
size = 0 # NOTE(markwash): zero -> unknown size
# Create the verifier for signature verification (if correct properties
# are present)
if (signature_utils.should_create_verifier(
self.image.extra_properties)):
# NOTE(bpoulos): if creating verifier fails, exception will be
# raised
verifier = signature_utils.get_verifier(
self.context, self.image.extra_properties)
else:
verifier = None
location, size, checksum, loc_meta = self.store_api.add_to_backend(
CONF,
self.image.image_id,
utils.LimitingReader(utils.CooperativeReader(data),
CONF.image_size_cap),
size,
context=self.context,
verifier=verifier)
# NOTE(bpoulos): if verification fails, exception will be raised
if verifier:
try:
verifier.verify()
LOG.info(_LI("Successfully verified signature for image %s"),
self.image.image_id)
except crypto_exception.InvalidSignature:
raise exception.SignatureVerificationError(
_('Signature verification failed'))
self.image.locations = [{
'url': location,
'metadata': loc_meta,
'status': 'active'
}]
self.image.size = size
self.image.checksum = checksum
self.image.status = 'active'
def run(self):
images = self.cache.get_queued_images()
if not images:
LOG.debug("Nothing to prefetch.")
return True
num_images = len(images)
LOG.debug("Found %d images to prefetch", num_images)
pool = eventlet.GreenPool(num_images)
results = pool.imap(self.fetch_image_into_cache, images)
successes = sum([1 for r in results if r is True])
if successes != num_images:
LOG.warn(_LW("Failed to successfully cache all "
"images in queue."))
return False
LOG.info(_LI("Successfully cached all %d images"), num_images)
return True
def configure_driver(self):
"""
Configure the driver for the cache and, if it fails to configure,
fall back to using the SQLite driver which has no odd dependencies
"""
try:
self.driver = self.driver_class()
self.driver.configure()
except exception.BadDriverConfiguration as config_err:
driver_module = self.driver_class.__module__
LOG.warn(_LW("Image cache driver "
"'%(driver_module)s' failed to configure. "
"Got error: '%(config_err)s"),
{'driver_module': driver_module,
'config_err': config_err})
LOG.info(_LI("Defaulting to SQLite driver."))
default_module = __name__ + '.drivers.sqlite.Driver'
self.driver_class = importutils.import_class(default_module)
self.driver = self.driver_class()
self.driver.configure()
def migrate_location_credentials(migrate_engine, to_quoted):
"""
Migrate location credentials for encrypted swift uri's between the
quoted and unquoted forms.
:param migrate_engine: The configured db engine
:param to_quoted: If True, migrate location credentials from
unquoted to quoted form. If False, do the
reverse.
"""
if not CONF.metadata_encryption_key:
msg = _LI("'metadata_encryption_key' was not specified in the config"
" file or a config file was not specified. This means that"
" this migration is a NOOP.")
LOG.info(msg)
return
meta = sqlalchemy.schema.MetaData()
meta.bind = migrate_engine
images_table = sqlalchemy.Table('images', meta, autoload=True)
images = list(images_table.select().execute())
for image in images:
try:
fixed_uri = fix_uri_credentials(image['location'], to_quoted)
images_table.update().where(
images_table.c.id == image['id']).values(
location=fixed_uri).execute()
except exception.Invalid:
msg = _LW("Failed to decrypt location value for image"
" %(image_id)s") % {'image_id': image['id']}
LOG.warn(msg)
except exception.BadStoreUri as e:
reason = encodeutils.exception_to_unicode(e)
msg = _LE("Invalid store uri for image: %(image_id)s. "
"Details: %(reason)s") % {'image_id': image.id,
'reason': reason}
LOG.exception(msg)
raise
def _reap_old_files(self, dirpath, entry_type, grace=None):
now = time.time()
reaped = 0
for path in get_all_regular_files(dirpath):
mtime = os.path.getmtime(path)
age = now - mtime
if not grace:
LOG.debug("No grace period, reaping '%(path)s'"
" immediately", {'path': path})
delete_cached_file(path)
reaped += 1
elif age > grace:
LOG.debug("Cache entry '%(path)s' exceeds grace period, "
"(%(age)i s > %(grace)i s)",
{'path': path, 'age': age, 'grace': grace})
delete_cached_file(path)
reaped += 1
LOG.info(_LI("Reaped %(reaped)s %(entry_type)s cache entries"),
{'reaped': reaped, 'entry_type': entry_type})
return reaped
def set_data(self, data, size=None):
if size is None:
size = 0 # NOTE(markwash): zero -> unknown size
# Create the verifier for signature verification (if correct properties
# are present)
if (signature_utils.should_create_verifier(
self.image.extra_properties)):
# NOTE(bpoulos): if creating verifier fails, exception will be
# raised
verifier = signature_utils.get_verifier(
self.context, self.image.extra_properties)
else:
verifier = None
location, size, checksum, loc_meta = self.store_api.add_to_backend(
CONF,
self.image.image_id,
utils.LimitingReader(utils.CooperativeReader(data),
CONF.image_size_cap),
size,
context=self.context,
verifier=verifier)
# NOTE(bpoulos): if verification fails, exception will be raised
if verifier:
try:
verifier.verify()
LOG.info(_LI("Successfully verified signature for image %s"),
self.image.image_id)
except crypto_exception.InvalidSignature:
raise exception.SignatureVerificationError(
_('Signature verification failed')
)
self.image.locations = [{'url': location, 'metadata': loc_meta,
'status': 'active'}]
self.image.size = size
self.image.checksum = checksum
self.image.status = 'active'
def __init__(self, app):
LOG.info(_LI("Initialized gzip middleware"))
super(GzipMiddleware, self).__init__(app)
def __init__(self, app):
self.cache = image_cache.ImageCache()
self.serializer = images.ImageSerializer()
self.policy = policy.Enforcer()
LOG.info(_LI("Initialized image cache middleware"))
super(CacheFilter, self).__init__(app)
def create_tables(tables):
for table in tables:
LOG.info(_LI("creating table %(table)s"), {'table': table})
table.create()
def replication_load(options, args):
"""%(prog)s load <server:port> <path>
Load the contents of a local directory into xmonitor.
server:port: the location of the xmonitor instance.
path: a directory on disk containing the data.
"""
# Make sure server and path are provided
if len(args) < 2:
raise TypeError(_("Too few arguments."))
path = args.pop()
server, port = utils.parse_valid_host_port(args.pop())
imageservice = get_image_service()
client = imageservice(http_client.HTTPConnection(server, port),
options.slavetoken)
updated = []
for ent in os.listdir(path):
if uuidutils.is_uuid_like(ent):
image_uuid = ent
LOG.info(_LI('Considering: %s'), image_uuid)
meta_file_name = os.path.join(path, image_uuid)
with open(meta_file_name) as meta_file:
meta = jsonutils.loads(meta_file.read())
# Remove keys which don't make sense for replication
for key in options.dontreplicate.split(' '):
if key in meta:
LOG.debug('Stripping %(header)s from saved '
'metadata', {'header': key})
del meta[key]
if _image_present(client, image_uuid):
# NOTE(mikal): Perhaps we just need to update the metadata?
# Note that we don't attempt to change an image file once it
# has been uploaded.
LOG.debug('Image %s already present', image_uuid)
headers = client.get_image_meta(image_uuid)
for key in options.dontreplicate.split(' '):
if key in headers:
LOG.debug('Stripping %(header)s from slave '
'metadata', {'header': key})
del headers[key]
if _dict_diff(meta, headers):
LOG.info(_LI('Image %s metadata has changed'), image_uuid)
headers, body = client.add_image_meta(meta)
_check_upload_response_headers(headers, body)
updated.append(meta['id'])
else:
if not os.path.exists(os.path.join(path, image_uuid + '.img')):
LOG.debug('%s dump is missing image data, skipping',
image_uuid)
continue
# Upload the image itself
with open(os.path.join(path, image_uuid + '.img')) as img_file:
try:
headers, body = client.add_image(meta, img_file)
_check_upload_response_headers(headers, body)
updated.append(meta['id'])
except exc.HTTPConflict:
LOG.error(_LE(IMAGE_ALREADY_PRESENT_MESSAGE)
% image_uuid) # noqa
return updated
def replication_livecopy(options, args):
"""%(prog)s livecopy <fromserver:port> <toserver:port>
Load the contents of one xmonitor instance into another.
fromserver:port: the location of the master xmonitor instance.
toserver:port: the location of the slave xmonitor instance.
"""
# Make sure from-server and to-server are provided
if len(args) < 2:
raise TypeError(_("Too few arguments."))
imageservice = get_image_service()
slave_server, slave_port = utils.parse_valid_host_port(args.pop())
slave_conn = http_client.HTTPConnection(slave_server, slave_port)
slave_client = imageservice(slave_conn, options.slavetoken)
master_server, master_port = utils.parse_valid_host_port(args.pop())
master_conn = http_client.HTTPConnection(master_server, master_port)
master_client = imageservice(master_conn, options.mastertoken)
updated = []
for image in master_client.get_images():
LOG.debug('Considering %(id)s', {'id': image['id']})
for key in options.dontreplicate.split(' '):
if key in image:
LOG.debug('Stripping %(header)s from master metadata',
{'header': key})
del image[key]
if _image_present(slave_client, image['id']):
# NOTE(mikal): Perhaps we just need to update the metadata?
# Note that we don't attempt to change an image file once it
# has been uploaded.
headers = slave_client.get_image_meta(image['id'])
if headers['status'] == 'active':
for key in options.dontreplicate.split(' '):
if key in image:
LOG.debug('Stripping %(header)s from master '
'metadata', {'header': key})
del image[key]
if key in headers:
LOG.debug('Stripping %(header)s from slave '
'metadata', {'header': key})
del headers[key]
if _dict_diff(image, headers):
LOG.info(_LI('Image %s metadata has changed'), image['id'])
headers, body = slave_client.add_image_meta(image)
_check_upload_response_headers(headers, body)
updated.append(image['id'])
elif image['status'] == 'active':
LOG.info(_LI('Image %s is being synced'), image['id'])
if not options.metaonly:
image_response = master_client.get_image(image['id'])
try:
headers, body = slave_client.add_image(image,
image_response)
_check_upload_response_headers(headers, body)
updated.append(image['id'])
except exc.HTTPConflict:
LOG.error(_LE(IMAGE_ALREADY_PRESENT_MESSAGE) % image['id']) # noqa
return updated
def upload_data_to_store(req, image_meta, image_data, store, notifier):
"""
Upload image data to specified store.
Upload image data to the store and cleans up on error.
"""
image_id = image_meta['id']
db_api = xmonitor.db.get_api()
image_size = image_meta.get('size')
try:
# By default image_data will be passed as CooperativeReader object.
# But if 'user_storage_quota' is enabled and 'remaining' is not None
# then it will be passed as object of LimitingReader to
# 'store_add_to_backend' method.
image_data = utils.CooperativeReader(image_data)
remaining = xmonitor.api.common.check_quota(
req.context, image_size, db_api, image_id=image_id)
if remaining is not None:
image_data = utils.LimitingReader(image_data, remaining)
(uri,
size,
checksum,
location_metadata) = store_api.store_add_to_backend(
image_meta['id'],
image_data,
image_meta['size'],
store,
context=req.context)
location_data = {'url': uri,
'metadata': location_metadata,
'status': 'active'}
try:
# recheck the quota in case there were simultaneous uploads that
# did not provide the size
xmonitor.api.common.check_quota(
req.context, size, db_api, image_id=image_id)
except exception.StorageQuotaFull:
with excutils.save_and_reraise_exception():
LOG.info(_LI('Cleaning up %s after exceeding '
'the quota'), image_id)
store_utils.safe_delete_from_backend(
req.context, image_meta['id'], location_data)
def _kill_mismatched(image_meta, attr, actual):
supplied = image_meta.get(attr)
if supplied and supplied != actual:
msg = (_("Supplied %(attr)s (%(supplied)s) and "
"%(attr)s generated from uploaded image "
"(%(actual)s) did not match. Setting image "
"status to 'killed'.") % {'attr': attr,
'supplied': supplied,
'actual': actual})
LOG.error(msg)
safe_kill(req, image_id, 'saving')
initiate_deletion(req, location_data, image_id)
raise webob.exc.HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
# Verify any supplied size/checksum value matches size/checksum
# returned from store when adding image
_kill_mismatched(image_meta, 'size', size)
_kill_mismatched(image_meta, 'checksum', checksum)
# Update the database with the checksum returned
# from the backend store
LOG.debug("Updating image %(image_id)s data. "
"Checksum set to %(checksum)s, size set "
"to %(size)d", {'image_id': image_id,
'checksum': checksum,
'size': size})
update_data = {'checksum': checksum,
'size': size}
try:
try:
state = 'saving'
image_meta = registry.update_image_metadata(req.context,
image_id,
update_data,
from_state=state)
except exception.Duplicate:
image = registry.get_image_metadata(req.context, image_id)
if image['status'] == 'deleted':
raise exception.ImageNotFound()
else:
raise
except exception.NotAuthenticated as e:
# Delete image data due to possible token expiration.
LOG.debug("Authentication error - the token may have "
"expired during file upload. Deleting image data for "
" %s " % image_id)
initiate_deletion(req, location_data, image_id)
raise webob.exc.HTTPUnauthorized(explanation=e.msg, request=req)
except exception.ImageNotFound:
msg = _("Image %s could not be found after upload. The image may"
" have been deleted during the upload.") % image_id
LOG.info(msg)
# NOTE(jculp): we need to clean up the datastore if an image
# resource is deleted while the image data is being uploaded
#
# We get "location_data" from above call to store.add(), any
# exceptions that occur there handle this same issue internally,
# Since this is store-agnostic, should apply to all stores.
initiate_deletion(req, location_data, image_id)
raise webob.exc.HTTPPreconditionFailed(explanation=msg,
request=req,
content_type='text/plain')
except store_api.StoreAddDisabled:
msg = _("Error in store configuration. Adding images to store "
"is disabled.")
LOG.exception(msg)
safe_kill(req, image_id, 'saving')
notifier.error('image.upload', msg)
raise webob.exc.HTTPGone(explanation=msg, request=req,
content_type='text/plain')
except (store_api.Duplicate, exception.Duplicate) as e:
msg = (_("Attempt to upload duplicate image: %s") %
encodeutils.exception_to_unicode(e))
LOG.warn(msg)
# NOTE(dosaboy): do not delete the image since it is likely that this
# conflict is a result of another concurrent upload that will be
# successful.
notifier.error('image.upload', msg)
raise webob.exc.HTTPConflict(explanation=msg,
request=req,
content_type="text/plain")
except exception.Forbidden as e:
msg = (_("Forbidden upload attempt: %s") %
encodeutils.exception_to_unicode(e))
LOG.warn(msg)
safe_kill(req, image_id, 'saving')
notifier.error('image.upload', msg)
raise webob.exc.HTTPForbidden(explanation=msg,
request=req,
content_type="text/plain")
except store_api.StorageFull as e:
msg = (_("Image storage media is full: %s") %
encodeutils.exception_to_unicode(e))
LOG.error(msg)
safe_kill(req, image_id, 'saving')
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except store_api.StorageWriteDenied as e:
msg = (_("Insufficient permissions on image storage media: %s") %
encodeutils.exception_to_unicode(e))
LOG.error(msg)
safe_kill(req, image_id, 'saving')
notifier.error('image.upload', msg)
raise webob.exc.HTTPServiceUnavailable(explanation=msg,
request=req,
content_type='text/plain')
except exception.ImageSizeLimitExceeded as e:
msg = (_("Denying attempt to upload image larger than %d bytes.")
% CONF.image_size_cap)
LOG.warn(msg)
safe_kill(req, image_id, 'saving')
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except exception.StorageQuotaFull as e:
msg = (_("Denying attempt to upload image because it exceeds the "
"quota: %s") % encodeutils.exception_to_unicode(e))
LOG.warn(msg)
safe_kill(req, image_id, 'saving')
notifier.error('image.upload', msg)
raise webob.exc.HTTPRequestEntityTooLarge(explanation=msg,
request=req,
content_type='text/plain')
except webob.exc.HTTPError:
# NOTE(bcwaldon): Ideally, we would just call 'raise' here,
# but something in the above function calls is affecting the
# exception context and we must explicitly re-raise the
# caught exception.
msg = _LE("Received HTTP error while uploading image %s") % image_id
notifier.error('image.upload', msg)
with excutils.save_and_reraise_exception():
LOG.exception(msg)
safe_kill(req, image_id, 'saving')
except (ValueError, IOError) as e:
msg = _("Client disconnected before sending all data to backend")
LOG.warn(msg)
safe_kill(req, image_id, 'saving')
raise webob.exc.HTTPBadRequest(explanation=msg,
content_type="text/plain",
request=req)
except Exception as e:
msg = _("Failed to upload image %s") % image_id
LOG.exception(msg)
safe_kill(req, image_id, 'saving')
notifier.error('image.upload', msg)
raise webob.exc.HTTPInternalServerError(explanation=msg,
request=req,
content_type='text/plain')
return image_meta, location_data
