def revert(self, result, **kwargs):
if isinstance(result, failure.Failure):
LOG.error(
_LE('Task: %(task_id)s failed to import image '
'%(image_id)s to the filesystem.'), {
'task_id': self.task_id,
'image_id': self.image_id
})
# NOTE(abhishekk): Revert image state back to 'queued' as
# something went wrong.
image = self.image_repo.get(self.image_id)
image.status = 'queued'
self.image_repo.save(image)
# NOTE(abhishekk): Deleting partial image data from staging area
if self._path is not None:
LOG.debug(('Deleting image %(image_id)s from staging '
'area.'), {'image_id': self.image_id})
try:
if CONF.enabled_backends:
store_api.delete(self._path, None)
else:
store_api.delete_from_backend(self._path)
except Exception:
LOG.exception(
_LE("Error reverting web-download "
"task: %(task_id)s"), {'task_id': self.task_id})
def _load_config(self):
try:
conf_file = CONF.find_file(CONF.swift_store_config_file)
CONFIG.read(conf_file)
except Exception as e:
msg = (_LE("swift config file %(conf_file)s:%(exc)s not found") % {
'conf_file': CONF.swift_store_config_file,
'exc': e
})
LOG.error(msg)
raise exception.InvalidSwiftStoreConfiguration()
account_params = {}
account_references = CONFIG.sections()
for ref in account_references:
reference = {}
try:
reference['auth_address'] = CONFIG.get(ref, 'auth_address')
reference['user'] = CONFIG.get(ref, 'user')
reference['key'] = CONFIG.get(ref, 'key')
account_params[ref] = reference
except (ValueError, SyntaxError, configparser.NoOptionError) as e:
LOG.exception(
_LE("Invalid format of swift store config "
"cfg"))
return account_params
def size_checked_iter(response, image_meta, expected_size, image_iter,
notifier):
image_id = image_meta['id']
bytes_written = 0
def notify_image_sent_hook(env):
image_send_notification(bytes_written, expected_size,
image_meta, response.request, notifier)
# Add hook to process after response is fully sent
if 'eventlet.posthooks' in response.request.environ:
response.request.environ['eventlet.posthooks'].append(
(notify_image_sent_hook, (), {}))
try:
for chunk in image_iter:
yield chunk
bytes_written += len(chunk)
except Exception as err:
with excutils.save_and_reraise_exception():
msg = (_LE("An error occurred reading from backend storage for "
"image %(image_id)s: %(err)s") % {'image_id': image_id,
'err': err})
LOG.error(msg)
if expected_size != bytes_written:
msg = (_LE("Backend storage for image %(image_id)s "
"disconnected after writing only %(bytes_written)d "
"bytes") % {'image_id': image_id,
'bytes_written': bytes_written})
LOG.error(msg)
raise exception.GlanceException(_("Corrupt image download for "
"image %(image_id)s") %
{'image_id': image_id})
def main():
"""The main function."""
try:
config.parse_args()
except RuntimeError as e:
sys.exit("ERROR: %s" % encodeutils.exception_to_unicode(e))
except SystemExit as e:
sys.exit("Please specify one command")
# Setup logging
logging.setup(CONF, 'glance')
if CONF.token:
CONF.slavetoken = CONF.token
CONF.mastertoken = CONF.token
command = lookup_command(CONF.command)
try:
command(CONF, CONF.args)
except TypeError as e:
LOG.error(_LE(command.__doc__) % {'prog': command.__name__}) # noqa
sys.exit("ERROR: %s" % encodeutils.exception_to_unicode(e))
except ValueError as e:
LOG.error(_LE(command.__doc__) % {'prog': command.__name__}) # noqa
sys.exit("ERROR: %s" % encodeutils.exception_to_unicode(e))
def run_task(task_id, task_type, context,
task_repo=None, image_repo=None, image_factory=None):
# TODO(nikhil): if task_repo is None get new task repo
# TODO(nikhil): if image_repo is None get new image repo
# TODO(nikhil): if image_factory is None get new image factory
LOG.info(_LI("Loading known task scripts for task_id %(task_id)s "
"of type %(task_type)s"), {'task_id': task_id,
'task_type': task_type})
if task_type == 'import':
image_import.run(task_id, context, task_repo,
image_repo, image_factory)
elif task_type == 'api_image_import':
api_image_import.run(task_id,
context,
task_repo,
image_repo,
image_factory)
else:
msg = _LE("This task type %(task_type)s is not supported by the "
"current deployment of Glance. Please refer the "
"documentation provided by OpenStack or your operator "
"for more information.") % {'task_type': task_type}
LOG.error(msg)
task = task_repo.get(task_id)
task.fail(msg)
if task_repo:
task_repo.save(task)
else:
LOG.error(_LE("Failed to save task %(task_id)s in DB as task_repo "
"is %(task_repo)s"), {"task_id": task_id,
"task_repo": task_repo})
def _execute(t_id, task_repo, image_repo, image_factory):
task = script_utils.get_task(task_repo, t_id)
if task is None:
# NOTE: This happens if task is not found in the database. In
# such cases, there is no way to update the task status so,
# it's ignored here.
return
try:
task_input = script_utils.unpack_task_input(task)
uri = script_utils.validate_location_uri(task_input.get('import_from'))
image_id = import_image(image_repo, image_factory, task_input, t_id,
uri)
task.succeed({'image_id': image_id})
except Exception as e:
# Note: The message string contains Error in it to indicate
# in the task.message that it's a error message for the user.
# TODO(nikhil): need to bring back save_and_reraise_exception when
# necessary
err_msg = ("Error: " + six.text_type(type(e)) + ': ' +
encodeutils.exception_to_unicode(e))
log_msg = _LE(err_msg + ("Task ID %s" % task.task_id)) # noqa
LOG.exception(log_msg)
task.fail(_LE(err_msg)) # noqa
finally:
task_repo.save(task)
def revert(self, result, **kwargs):
if isinstance(result, failure.Failure):
LOG.error(_LE('Task: %(task_id)s failed to import image '
'%(image_id)s to the filesystem.'),
{'task_id': self.task_id,
'image_id': self.image_id})
# NOTE(abhishekk): Revert image state back to 'queued' as
# something went wrong.
# NOTE(danms): If we failed to stage the image, then none
# of the _ImportToStore() tasks could have run, so we need
# to move all stores out of "importing" and into "failed".
with self.action_wrapper as action:
action.set_image_attribute(status='queued')
action.remove_importing_stores(self.stores)
action.add_failed_stores(self.stores)
# NOTE(abhishekk): Deleting partial image data from staging area
if self._path is not None:
LOG.debug(('Deleting image %(image_id)s from staging '
'area.'), {'image_id': self.image_id})
try:
if CONF.enabled_backends:
store_api.delete(self._path, None)
else:
store_api.delete_from_backend(self._path)
except Exception:
LOG.exception(_LE("Error reverting web-download "
"task: %(task_id)s"), {
'task_id': self.task_id})
def _execute(t_id, task_repo, image_repo, image_factory):
task = script_utils.get_task(task_repo, t_id)
if task is None:
# NOTE: This happens if task is not found in the database. In
# such cases, there is no way to update the task status so,
# it's ignored here.
return
try:
task_input = script_utils.unpack_task_input(task)
uri = script_utils.validate_location_uri(task_input.get('import_from'))
image_id = import_image(image_repo, image_factory, task_input, t_id,
uri)
task.succeed({'image_id': image_id})
except Exception as e:
# Note: The message string contains Error in it to indicate
# in the task.message that it's a error message for the user.
# TODO(nikhil): need to bring back save_and_reraise_exception when
# necessary
err_msg = ("Error: " + six.text_type(type(e)) + ': ' +
encodeutils.exception_to_unicode(e))
log_msg = _LE(err_msg + ("Task ID %s" % task.task_id)) # noqa
LOG.exception(log_msg)
task.fail(_LE(err_msg)) # noqa
finally:
task_repo.save(task)
def size_checked_iter(response, image_meta, expected_size, image_iter,
notifier):
image_id = image_meta['id']
bytes_written = 0
def notify_image_sent_hook(env):
image_send_notification(bytes_written, expected_size,
image_meta, response.request, notifier)
# Add hook to process after response is fully sent
if 'eventlet.posthooks' in response.request.environ:
response.request.environ['eventlet.posthooks'].append(
(notify_image_sent_hook, (), {}))
try:
for chunk in image_iter:
yield chunk
bytes_written += len(chunk)
except Exception as err:
with excutils.save_and_reraise_exception():
msg = (_LE("An error occurred reading from backend storage for "
"image %(image_id)s: %(err)s") % {'image_id': image_id,
'err': err})
LOG.error(msg)
if expected_size != bytes_written:
msg = (_LE("Backend storage for image %(image_id)s "
"disconnected after writing only %(bytes_written)d "
"bytes") % {'image_id': image_id,
'bytes_written': bytes_written})
LOG.error(msg)
raise exception.GlanceException(_("Corrupt image download for "
"image %(image_id)s") %
{'image_id': image_id})
def main():
"""The main function."""
try:
config.parse_args()
except RuntimeError as e:
sys.exit("ERROR: %s" % encodeutils.exception_to_unicode(e))
except SystemExit as e:
sys.exit("Please specify one command")
# Setup logging
logging.setup(CONF, 'glance')
if CONF.token:
CONF.slavetoken = CONF.token
CONF.mastertoken = CONF.token
command = lookup_command(CONF.command)
try:
command(CONF, CONF.args)
except TypeError as e:
LOG.error(_LE(command.__doc__) % {'prog': command.__name__}) # noqa
sys.exit("ERROR: %s" % encodeutils.exception_to_unicode(e))
except ValueError as e:
LOG.error(_LE(command.__doc__) % {'prog': command.__name__}) # noqa
sys.exit("ERROR: %s" % encodeutils.exception_to_unicode(e))
def _run(self, task_id, task_type):
task = self.task_repo.get(task_id)
msg = _LE("This execution of Tasks is not setup. Please consult the "
"project documentation for more information on the "
"executors available.")
LOG.error(msg)
task.fail(_LE("Internal error occurred while trying to process task."))
self.task_repo.save(task)
def _run(self, task_id, task_type):
task = self.task_repo.get(task_id)
msg = _LE("This execution of Tasks is not setup. Please consult the "
"project documentation for more information on the "
"executors available.")
LOG.error(msg)
task.fail(_LE("Internal error occurred while trying to process task."))
self.task_repo.save(task)
def execute(self):
"""Create temp file into store and return path to it
:param image_id: Glance Image ID
"""
image = self.image_repo.get(self.image_id)
# NOTE (abhishekk): If ``all_stores_must_succeed`` is set to True
# and copying task fails then we keep data in staging area as it
# is so that if second call is made to copy the same image then
# no need to copy the data in staging area again.
file_path = "%s/%s" % (getattr(
CONF,
'os_glance_staging_store').filesystem_store_datadir, self.image_id)
if os.path.exists(file_path):
return file_path, 0
# At first search image in default_backend
default_store = CONF.glance_store.default_backend
for loc in image.locations:
if loc['metadata'].get('store') == default_store:
try:
return self._copy_to_staging_store(loc)
except store_api.exceptions.NotFound:
msg = (_LE("Image not present in default store, searching "
"in all glance-api specific available "
"stores"))
LOG.error(msg)
break
available_backends = CONF.enabled_backends
for loc in image.locations:
image_backend = loc['metadata'].get('store')
if (image_backend in available_backends.keys()
and image_backend != default_store):
try:
return self._copy_to_staging_store(loc)
except store_api.exceptions.NotFound:
LOG.error(
_LE('Image: %(img_id)s is not present in store '
'%(store)s.'), {
'img_id': self.image_id,
'store': image_backend
})
continue
raise exception.NotFound(
_("Image not found in any configured "
"store"))
def _run(self, task_id, task_type):
LOG.debug('Taskflow executor picked up the execution of task ID '
'%(task_id)s of task type '
'%(task_type)s', {'task_id': task_id,
'task_type': task_type})
task = script_utils.get_task(self.task_repo, task_id)
if task is None:
# NOTE: This happens if task is not found in the database. In
# such cases, there is no way to update the task status so,
# it's ignored here.
return
flow = self._get_flow(task)
executor = self._fetch_an_executor()
try:
engine = engines.load(
flow,
engine=CONF.taskflow_executor.engine_mode, executor=executor,
max_workers=CONF.taskflow_executor.max_workers)
with llistener.DynamicLoggingListener(engine, log=LOG):
engine.run()
except Exception as exc:
with excutils.save_and_reraise_exception():
LOG.error(_LE('Failed to execute task %(task_id)s: %(exc)s') %
{'task_id': task_id,
'exc': encodeutils.exception_to_unicode(exc)})
# TODO(sabari): Check for specific exceptions and update the
# task failure message.
task.fail(_('Task failed due to Internal Error'))
self.task_repo.save(task)
finally:
if executor is not None:
executor.shutdown()
def process_response(self, resp):
"""
We intercept the response coming back from the main
images Resource, removing image file from the cache
if necessary
"""
status_code = self.get_status_code(resp)
if not 200 <= status_code < 300:
return resp
try:
(image_id, method, version) = self._fetch_request_info(
resp.request)
except TypeError:
return resp
if method == 'GET' and status_code == http.NO_CONTENT:
# Bugfix:1251055 - Don't cache non-existent image files.
# NOTE: Both GET for an image without locations and DELETE return
# 204 but DELETE should be processed.
return resp
method_str = '_process_%s_response' % method
try:
process_response_method = getattr(self, method_str)
except AttributeError:
LOG.error(_LE('could not find %s') % method_str)
# Nothing to do here, move along
return resp
else:
return process_response_method(resp, image_id, version=version)
def _process_GET_response(self, resp, image_id, version=None):
image_checksum = resp.headers.get('Content-MD5')
if not image_checksum:
# API V1 stores the checksum in a different header:
image_checksum = resp.headers.get('x-image-meta-checksum')
if not image_checksum:
LOG.error(_LE("Checksum header is missing."))
# fetch image_meta on the basis of version
image = None
if version:
method = getattr(self, '_get_%s_image_metadata' % version)
image, metadata = method(resp.request, image_id)
# NOTE(zhiyan): image_cache return a generator object and set to
# response.app_iter, it will be called by eventlet.wsgi later.
# So we need enforce policy firstly but do it by application
# since eventlet.wsgi could not catch webob.exc.HTTPForbidden and
# return 403 error to client then.
# FIXME(abhishekk): This policy check here is not necessary as this
# will hit only during first image download i.e. while image is not
# present in cache. We already enforced same check in API layer and
# enforcing same check here again makes no sense.
self._enforce(resp.request, image)
resp.app_iter = self.cache.get_caching_iter(image_id, image_checksum,
resp.app_iter)
return resp
def __call__(self, request):
"""WSGI method that controls (de)serialization and method dispatch."""
action_args = self.get_action_args(request.environ)
action = action_args.pop("action", None)
try:
deserialized_request = self.dispatch(self.deserializer, action, request)
action_args.update(deserialized_request)
action_result = self.dispatch(self.controller, action, request, **action_args)
except webob.exc.WSGIHTTPException as e:
exc_info = sys.exc_info()
e = translate_exception(request, e)
six.reraise(type(e), e, exc_info[2])
except Exception as e:
LOG.exception(_LE("Caught error: %s"), six.text_type(e))
response = webob.exc.HTTPInternalServerError()
return response
try:
response = webob.Response(request=request)
self.dispatch(self.serializer, action, response, action_result)
return response
except webob.exc.WSGIHTTPException as e:
return translate_exception(request, e)
except webob.exc.HTTPException as e:
return e
# return unserializable result (typically a webob exc)
except Exception:
return action_result
def _run(self, task_id, task_type):
LOG.debug('Taskflow executor picked up the execution of task ID '
'%(task_id)s of task type '
'%(task_type)s', {'task_id': task_id,
'task_type': task_type})
task = script_utils.get_task(self.task_repo, task_id)
if task is None:
# NOTE: This happens if task is not found in the database. In
# such cases, there is no way to update the task status so,
# it's ignored here.
return
flow = self._get_flow(task)
executor = self._fetch_an_executor()
try:
engine = engines.load(
flow,
engine=CONF.taskflow_executor.engine_mode, executor=executor,
max_workers=CONF.taskflow_executor.max_workers)
with llistener.DynamicLoggingListener(engine, log=LOG):
engine.run()
except Exception as exc:
with excutils.save_and_reraise_exception():
LOG.error(_LE('Failed to execute task %(task_id)s: %(exc)s') %
{'task_id': task_id,
'exc': encodeutils.exception_to_unicode(exc)})
# TODO(sabari): Check for specific exceptions and update the
# task failure message.
task.fail(_('Task failed due to Internal Error'))
self.task_repo.save(task)
finally:
if executor is not None:
executor.shutdown()
def do_request(self, method, action, **kwargs):
try:
kwargs['headers'] = kwargs.get('headers', {})
kwargs['headers'].update(self.identity_headers or {})
if self._passed_request_id:
request_id = self._passed_request_id
if six.PY3 and isinstance(request_id, bytes):
request_id = request_id.decode('utf-8')
kwargs['headers']['X-Openstack-Request-ID'] = request_id
res = super(RegistryClient,
self).do_request(method, action, **kwargs)
status = res.status
request_id = res.getheader('x-openstack-request-id')
if six.PY3 and isinstance(request_id, bytes):
request_id = request_id.decode('utf-8')
LOG.debug(
"Registry request %(method)s %(action)s HTTP %(status)s"
" request id %(request_id)s", {
'method': method,
'action': action,
'status': status,
'request_id': request_id
})
except Exception as exc:
with excutils.save_and_reraise_exception():
exc_name = exc.__class__.__name__
LOG.exception(
_LE("Registry client request %(method)s "
"%(action)s raised %(exc_name)s"), {
'method': method,
'action': action,
'exc_name': exc_name
})
return res
def db_load_metadefs(engine, metadata_path=None, merge=False,
prefer_new=False, overwrite=False):
meta = MetaData()
meta.bind = engine
if not merge and (prefer_new or overwrite):
LOG.error(_LE("To use --prefer_new or --overwrite you need to combine "
"of these options with --merge option."))
return
if prefer_new and overwrite and merge:
LOG.error(_LE("Please provide no more than one option from this list: "
"--prefer_new, --overwrite"))
return
_populate_metadata(meta, metadata_path, merge, prefer_new, overwrite)
def _finish_task(self, task):
try:
task.succeed({'image_id': self.action_wrapper.image_id})
except Exception as e:
# Note: The message string contains Error in it to indicate
# in the task.message that it's a error message for the user.
# TODO(nikhil): need to bring back save_and_reraise_exception when
# necessary
log_msg = _LE("Task ID %(task_id)s failed. Error: %(exc_type)s: "
"%(e)s")
LOG.exception(
log_msg, {
'exc_type': six.text_type(type(e)),
'e': encodeutils.exception_to_unicode(e),
'task_id': task.task_id
})
err_msg = _("Error: %(exc_type)s: %(e)s")
task.fail(
err_msg % {
'exc_type': six.text_type(type(e)),
'e': encodeutils.exception_to_unicode(e)
})
finally:
self.task_repo.save(task)
def build_image_owner_map(owner_map, db, context):
image_owner_map = {}
for image in db.image_get_all(context):
image_id = image['id']
owner_name = image['owner']
if not owner_name:
LOG.info(_LI('Image %s has no owner. Skipping.') % image_id)
continue
try:
owner_id = owner_map[owner_name]
except KeyError:
msg = (_LE('Image "%(image)s" owner "%(owner)s" was not found. '
'Skipping.'),
{'image': image_id, 'owner': owner_name})
LOG.error(msg)
continue
image_owner_map[image_id] = owner_id
LOG.info(_LI('Image "%(image)s" owner "%(owner)s" -> "%(owner_id)s"'),
{'image': image_id, 'owner': owner_name,
'owner_id': owner_id})
return image_owner_map
def _delete_image_location_from_backend(self, image_id, loc_id, uri,
backend=None):
try:
LOG.debug("Scrubbing image %s from a location.", image_id)
try:
if CONF.enabled_backends:
self.store_api.delete(uri, backend, self.admin_context)
else:
self.store_api.delete_from_backend(uri, self.admin_context)
except store_exceptions.NotFound:
LOG.info(_LI("Image location for image '%s' not found in "
"backend; Marking image location deleted in "
"db."), image_id)
if loc_id != '-':
db_api.get_api().image_location_delete(self.admin_context,
image_id,
int(loc_id),
'deleted')
LOG.info(_LI("Image %s is scrubbed from a location."), image_id)
except Exception as e:
LOG.error(_LE("Unable to scrub image %(id)s from a location. "
"Reason: %(exc)s ") %
{'id': image_id,
'exc': encodeutils.exception_to_unicode(e)})
raise
def new_task_executor(self, context):
try:
# NOTE(flaper87): Backwards compatibility layer.
# It'll allow us to provide a deprecation path to
# users that are currently consuming the `eventlet`
# executor.
task_executor = CONF.task.task_executor
if task_executor == 'eventlet':
# NOTE(jokke): Making sure we do not log the deprecation
# warning 1000 times or anything crazy like that.
if not TaskExecutorFactory.eventlet_deprecation_warned:
msg = _LW("The `eventlet` executor has been deprecated. "
"Use `taskflow` instead.")
LOG.warn(msg)
TaskExecutorFactory.eventlet_deprecation_warned = True
task_executor = 'taskflow'
executor_cls = ('glance.async_.%s_executor.'
'TaskExecutor' % task_executor)
LOG.debug("Loading %s executor", task_executor)
executor = importutils.import_class(executor_cls)
return executor(context,
self.task_repo,
self.image_repo,
self.image_factory)
except ImportError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE("Failed to load the %s executor provided "
"in the config.") % CONF.task.task_executor)
def image_send_notification(bytes_written, expected_size, image_meta, request,
notifier):
"""Send an image.send message to the notifier."""
try:
context = request.context
payload = {
'bytes_sent': bytes_written,
'image_id': image_meta['id'],
'owner_id': image_meta['owner'],
'receiver_tenant_id': context.project_id,
'receiver_user_id': context.user_id,
'destination_ip': request.remote_addr,
}
if bytes_written != expected_size:
notify = notifier.error
else:
notify = notifier.info
notify('image.send', payload)
except Exception as err:
msg = (_LE("An error occurred during image.send"
" notification: %(err)s") % {
'err': err
})
LOG.error(msg)
def _on_load_failure(self, manager, ep, exc):
msg = (_LE("Could not load plugin from %(module)s: %(msg)s") % {
"module": ep.module_name,
"msg": exc
})
LOG.error(msg)
raise exc
def __call__(self, request):
"""WSGI method that controls (de)serialization and method dispatch."""
action_args = self.get_action_args(request.environ)
action = action_args.pop('action', None)
try:
deserialized_request = self.dispatch(self.deserializer, action,
request)
action_args.update(deserialized_request)
action_result = self.dispatch(self.controller, action, request,
**action_args)
except webob.exc.WSGIHTTPException as e:
exc_info = sys.exc_info()
e = translate_exception(request, e)
six.reraise(type(e), e, exc_info[2])
except Exception as e:
LOG.exception(_LE("Caught error: %s"), six.text_type(e))
response = webob.exc.HTTPInternalServerError()
return response
try:
response = webob.Response(request=request)
self.dispatch(self.serializer, action, response, action_result)
return response
except webob.exc.WSGIHTTPException as e:
return translate_exception(request, e)
except webob.exc.HTTPException as e:
return e
# return unserializable result (typically a webob exc)
except Exception:
return action_result
def cache_tee_iter(self, image_id, image_iter, image_checksum):
try:
current_checksum = hashlib.md5()
with self.driver.open_for_write(image_id) as cache_file:
for chunk in image_iter:
try:
cache_file.write(chunk)
finally:
current_checksum.update(chunk)
yield chunk
cache_file.flush()
if (image_checksum and
image_checksum != current_checksum.hexdigest()):
msg = _("Checksum verification failed. Aborted "
"caching of image '%s'.") % image_id
raise exception.GlanceException(msg)
except exception.GlanceException as e:
with excutils.save_and_reraise_exception():
# image_iter has given us bad, (size_checked_iter has found a
# bad length), or corrupt data (checksum is wrong).
LOG.exception(encodeutils.exception_to_unicode(e))
except Exception as e:
LOG.exception(_LE("Exception encountered while tee'ing "
"image '%(image_id)s' into cache: %(error)s. "
"Continuing with response.") %
{'image_id': image_id,
'error': encodeutils.exception_to_unicode(e)})
# If no checksum provided continue responding even if
# caching failed.
for chunk in image_iter:
yield chunk
def _run(self, task_id, task_type):
LOG.debug('Taskflow executor picked up the execution of task ID '
'%(task_id)s of task type '
'%(task_type)s', {'task_id': task_id,
'task_type': task_type})
task = script_utils.get_task(self.task_repo, task_id)
if task is None:
# NOTE: This happens if task is not found in the database. In
# such cases, there is no way to update the task status so,
# it's ignored here.
return
flow = self._get_flow(task)
try:
with self._executor() as executor:
engine = engines.load(flow, self.engine_conf,
executor=executor, **self.engine_kwargs)
with llistener.DynamicLoggingListener(engine, log=LOG):
engine.run()
except Exception as exc:
with excutils.save_and_reraise_exception():
LOG.error(_LE('Failed to execute task %(task_id)s: %(exc)s') %
{'task_id': task_id, 'exc': six.text_type(exc)})
# TODO(sabari): Check for specific exceptions and update the
# task failure message.
task.fail(_('Task failed due to Internal Error'))
self.task_repo.save(task)
def migrate_location_credentials(migrate_engine, to_quoted):
"""
Migrate location credentials for swift uri's between the quoted
and unquoted forms.
:param migrate_engine: The configured db engine
:param to_quoted: If True, migrate location credentials from
unquoted to quoted form. If False, do the
reverse.
"""
meta = sqlalchemy.schema.MetaData()
meta.bind = migrate_engine
images_table = sqlalchemy.Table('images', meta, autoload=True)
images = list(images_table.select(images_table.c.location.startswith(
'swift')).execute())
for image in images:
try:
fixed_uri = legacy_parse_uri(image['location'], to_quoted)
images_table.update().where(
images_table.c.id == image['id']).values(
location=fixed_uri).execute()
except exception.BadStoreUri as e:
reason = encodeutils.exception_to_unicode(e)
msg = _LE("Invalid store uri for image: %(image_id)s. "
"Details: %(reason)s") % {'image_id': image.id,
'reason': reason}
LOG.exception(msg)
raise
def safe_delete_from_backend(context, image_id, location):
"""
Given a location, delete an image from the store and
update location status to db.
This function try to handle all known exceptions which might be raised
by those calls on store and DB modules in its implementation.
:param context: The request context
:param image_id: The image identifier
:param location: The image location entry
"""
try:
ret = store_api.delete_from_backend(location['url'], context=context)
location['status'] = 'deleted'
if 'id' in location:
db_api.get_api().image_location_delete(context, image_id,
location['id'], 'deleted')
return ret
except store_api.NotFound:
msg = _LW('Failed to delete image %s in store from URI') % image_id
LOG.warn(msg)
except store_api.StoreDeleteNotSupported as e:
LOG.warn(encodeutils.exception_to_unicode(e))
except store_api.UnsupportedBackend:
exc_type = sys.exc_info()[0].__name__
msg = (_LE('Failed to delete image %(image_id)s from store: %(exc)s') %
dict(image_id=image_id, exc=exc_type))
LOG.error(msg)
def do_request(self, method, action, **kwargs):
try:
kwargs['headers'] = kwargs.get('headers', {})
kwargs['headers'].update(self.identity_headers or {})
if self._passed_request_id:
kwargs['headers']['X-Openstack-Request-ID'] = (
self._passed_request_id)
res = super(RegistryClient, self).do_request(method,
action,
**kwargs)
status = res.status
request_id = res.getheader('x-openstack-request-id')
LOG.debug("Registry request %(method)s %(action)s HTTP %(status)s"
" request id %(request_id)s",
{'method': method, 'action': action,
'status': status, 'request_id': request_id})
except Exception as exc:
with excutils.save_and_reraise_exception():
exc_name = exc.__class__.__name__
LOG.exception(_LE("Registry client request %(method)s "
"%(action)s raised %(exc_name)s"),
{'method': method, 'action': action,
'exc_name': exc_name})
return res
def cache_tee_iter(self, image_id, image_iter, image_checksum):
try:
current_checksum = hashlib.md5()
with self.driver.open_for_write(image_id) as cache_file:
for chunk in image_iter:
try:
cache_file.write(chunk)
finally:
current_checksum.update(chunk)
yield chunk
cache_file.flush()
if (image_checksum and
image_checksum != current_checksum.hexdigest()):
msg = _("Checksum verification failed. Aborted "
"caching of image '%s'.") % image_id
raise exception.GlanceException(msg)
except exception.GlanceException as e:
with excutils.save_and_reraise_exception():
# image_iter has given us bad, (size_checked_iter has found a
# bad length), or corrupt data (checksum is wrong).
LOG.exception(encodeutils.exception_to_unicode(e))
except Exception as e:
LOG.exception(_LE("Exception encountered while tee'ing "
"image '%(image_id)s' into cache: %(error)s. "
"Continuing with response.") %
{'image_id': image_id,
'error': encodeutils.exception_to_unicode(e)})
# If no checksum provided continue responding even if
# caching failed.
for chunk in image_iter:
yield chunk
def get_data(self, offset=0, chunk_size=None):
if not self.image.locations:
# NOTE(mclaren): This is the only set of arguments
# which work with this exception currently, see:
# https://bugs.launchpad.net/glance-store/+bug/1501443
# When the above glance_store bug is fixed we can
# add a msg as usual.
raise store.NotFound(image=None)
err = None
for loc in self.image.locations:
try:
data, size = self.store_api.get_from_backend(
loc['url'],
offset=offset,
chunk_size=chunk_size,
context=self.context)
return data
except Exception as e:
LOG.warn(_LW('Get image %(id)s data failed: '
'%(err)s.')
% {'id': self.image.image_id,
'err': encodeutils.exception_to_unicode(e)})
err = e
# tried all locations
LOG.error(_LE('Glance tried all active locations to get data for '
'image %s but all have failed.') % self.image.image_id)
raise err
def migrate_location_credentials(migrate_engine, to_quoted):
"""
Migrate location credentials for swift uri's between the quoted
and unquoted forms.
:param migrate_engine: The configured db engine
:param to_quoted: If True, migrate location credentials from
unquoted to quoted form. If False, do the
reverse.
"""
meta = sqlalchemy.schema.MetaData()
meta.bind = migrate_engine
images_table = sqlalchemy.Table('images', meta, autoload=True)
images = list(
images_table.select(
images_table.c.location.startswith('swift')).execute())
for image in images:
try:
fixed_uri = legacy_parse_uri(image['location'], to_quoted)
images_table.update().where(
images_table.c.id == image['id']).values(
location=fixed_uri).execute()
except exception.BadStoreUri as e:
reason = encodeutils.exception_to_unicode(e)
msg = _LE("Invalid store uri for image: %(image_id)s. "
"Details: %(reason)s") % {
'image_id': image.id,
'reason': reason
}
LOG.exception(msg)
raise
def execute(self, image_id, file_path):
"""Does the actual introspection
:param image_id: Glance image ID
:param file_path: Path to the file being introspected
"""
try:
stdout, stderr = putils.trycmd('qemu-img', 'info',
'--output=json', file_path,
log_errors=putils.LOG_ALL_ERRORS)
except OSError as exc:
# NOTE(flaper87): errno == 2 means the executable file
# was not found. For now, log an error and move forward
# until we have a better way to enable/disable optional
# tasks.
if exc.errno != 2:
with excutils.save_and_reraise_exception():
msg = (_LE('Failed to execute introspection '
'%(task_id)s: %(exc)s') %
{'task_id': self.task_id, 'exc': exc.message})
LOG.error(msg)
return
if stderr:
raise RuntimeError(stderr)
metadata = json.loads(stdout)
new_image = self.image_repo.get(image_id)
new_image.virtual_size = metadata.get('virtual-size', 0)
new_image.disk_format = metadata.get('format')
self.image_repo.save(new_image)
LOG.debug("%(task_id)s: Introspection successful: %(file)s",
{'task_id': self.task_id, 'file': file_path})
return new_image
def db_load_metadefs(engine, metadata_path=None, merge=False,
prefer_new=False, overwrite=False):
meta = MetaData()
meta.bind = engine
if not merge and (prefer_new or overwrite):
LOG.error(_LE("To use --prefer_new or --overwrite you need to combine "
"of these options with --merge option."))
return
if prefer_new and overwrite and merge:
LOG.error(_LE("Please provide no more than one option from this list: "
"--prefer_new, --overwrite"))
return
_populate_metadata(meta, metadata_path, merge, prefer_new, overwrite)
def get_data(self, offset=0, chunk_size=None):
if not self.image.locations:
# NOTE(mclaren): This is the only set of arguments
# which work with this exception currently, see:
# https://bugs.launchpad.net/glance-store/+bug/1501443
# When the above glance_store bug is fixed we can
# add a msg as usual.
raise store.NotFound(image=None)
err = None
for loc in self.image.locations:
try:
#自backend中取数据
data, size = self.store_api.get_from_backend(
loc['url'],
offset=offset,
chunk_size=chunk_size,
context=self.context)
return data
except Exception as e:
LOG.warn(
_LW('Get image %(id)s data failed: '
'%(err)s.') % {
'id': self.image.image_id,
'err': encodeutils.exception_to_unicode(e)
})
err = e
# tried all locations
LOG.error(
_LE('Glance tried all active locations to get data for '
'image %s but all have failed.') % self.image.image_id)
raise err
def delete(self, req, id):
"""Deletes an existing image with the registry.
:param req: wsgi Request object
:param id: The opaque internal identifier for the image
:retval Returns 200 if delete was successful, a fault if not. On
success, the body contains the deleted image information as a mapping.
"""
try:
deleted_image = self.db_api.image_destroy(req.context, id)
LOG.info(_LI("Successfully deleted image %(id)s"), {'id': id})
return dict(image=make_image_dict(deleted_image))
except exception.ForbiddenPublicImage:
LOG.info(_LI("Delete denied for public image %(id)s"), {'id': id})
raise exc.HTTPForbidden()
except exception.Forbidden:
# If it's private and doesn't belong to them, don't let on
# that it exists
LOG.info(
_LI("Access denied to image %(id)s but returning"
" 'not found'"), {'id': id})
return exc.HTTPNotFound()
except exception.ImageNotFound:
LOG.info(_LI("Image %(id)s not found"), {'id': id})
return exc.HTTPNotFound()
except Exception:
LOG.exception(_LE("Unable to delete image %s") % id)
raise
def delete(self, req, id):
"""Deletes an existing image with the registry.
:param req: wsgi Request object
:param id: The opaque internal identifier for the image
:retval Returns 200 if delete was successful, a fault if not. On
success, the body contains the deleted image information as a mapping.
"""
try:
deleted_image = self.db_api.image_destroy(req.context, id)
LOG.info(_LI("Successfully deleted image %(id)s"), {'id': id})
return dict(image=make_image_dict(deleted_image))
except exception.ForbiddenPublicImage:
LOG.info(_LI("Delete denied for public image %(id)s"), {'id': id})
raise exc.HTTPForbidden()
except exception.Forbidden:
# If it's private and doesn't belong to them, don't let on
# that it exists
LOG.info(_LI("Access denied to image %(id)s but returning"
" 'not found'"), {'id': id})
return exc.HTTPNotFound()
except exception.ImageNotFound:
LOG.info(_LI("Image %(id)s not found"), {'id': id})
return exc.HTTPNotFound()
except Exception:
LOG.exception(_LE("Unable to delete image %s") % id)
raise
def safe_delete_from_backend(context, image_id, location):
"""
Given a location, delete an image from the store and
update location status to db.
This function try to handle all known exceptions which might be raised
by those calls on store and DB modules in its implementation.
:param context: The request context
:param image_id: The image identifier
:param location: The image location entry
"""
try:
ret = store_api.delete_from_backend(location['url'], context=context)
location['status'] = 'deleted'
if 'id' in location:
db_api.get_api().image_location_delete(context, image_id,
location['id'], 'deleted')
return ret
except store_api.NotFound:
msg = _LW('Failed to delete image %s in store from URI') % image_id
LOG.warn(msg)
except store_api.StoreDeleteNotSupported as e:
LOG.warn(encodeutils.exception_to_unicode(e))
except store_api.UnsupportedBackend:
exc_type = sys.exc_info()[0].__name__
msg = (_LE('Failed to delete image %(image_id)s from store: %(exc)s') %
dict(image_id=image_id, exc=exc_type))
LOG.error(msg)
def _get_images(self, context, filters, **params):
"""Get images, wrapping in exception if necessary."""
# NOTE(markwash): for backwards compatibility, is_public=True for
# admins actually means "treat me as if I'm not an admin and show me
# all my images"
if context.is_admin and params.get('is_public') is True:
params['admin_as_user'] = True
del params['is_public']
try:
return self.db_api.image_get_all(context,
filters=filters,
**params)
except exception.ImageNotFound:
LOG.warn(
_LW("Invalid marker. Image %(id)s could not be "
"found.") % {'id': params.get('marker')})
msg = _("Invalid marker. Image could not be found.")
raise exc.HTTPBadRequest(explanation=msg)
except exception.Forbidden:
LOG.warn(
_LW("Access denied to image %(id)s but returning "
"'not found'") % {'id': params.get('marker')})
msg = _("Invalid marker. Image could not be found.")
raise exc.HTTPBadRequest(explanation=msg)
except Exception:
LOG.exception(_LE("Unable to get images"))
raise
def new_task_executor(self, context):
try:
# NOTE(flaper87): Backwards compatibility layer.
# It'll allow us to provide a deprecation path to
# users that are currently consuming the `eventlet`
# executor.
task_executor = CONF.task.task_executor
if task_executor == 'eventlet':
# NOTE(jokke): Making sure we do not log the deprecation
# warning 1000 times or anything crazy like that.
if not TaskExecutorFactory.eventlet_deprecation_warned:
msg = _LW("The `eventlet` executor has been deprecated. "
"Use `taskflow` instead.")
LOG.warn(msg)
TaskExecutorFactory.eventlet_deprecation_warned = True
task_executor = 'taskflow'
executor_cls = ('glance.async.%s_executor.'
'TaskExecutor' % task_executor)
LOG.debug("Loading %s executor", task_executor)
executor = importutils.import_class(executor_cls)
return executor(context,
self.task_repo,
self.image_repo,
self.image_factory)
except ImportError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE("Failed to load the %s executor provided "
"in the config.") % CONF.task.task_executor)
def _load_strategies():
"""Load all strategy modules."""
modules = {}
namespace = "glance.common.image_location_strategy.modules"
ex = stevedore.extension.ExtensionManager(namespace)
for module_name in ex.names():
try:
mgr = stevedore.driver.DriverManager(namespace=namespace, name=module_name, invoke_on_load=False)
# Obtain module name
strategy_name = str(mgr.driver.get_strategy_name())
if strategy_name in modules:
msg = _("%(strategy)s is registered as a module twice. " "%(module)s is not being used.") % {
"strategy": strategy_name,
"module": module_name,
}
LOG.warn(msg)
else:
# Initialize strategy module
mgr.driver.init()
modules[strategy_name] = mgr.driver
except Exception as e:
LOG.error(
_LE("Failed to load location strategy module " "%(module)s: %(e)s") % {"module": module_name, "e": e}
)
return modules
def process_response(self, resp):
"""
We intercept the response coming back from the main
images Resource, removing image file from the cache
if necessary
"""
status_code = self.get_status_code(resp)
if not 200 <= status_code < 300:
return resp
# Note(dharinic): Bug: 1664709: Do not cache partial images.
if status_code == http.PARTIAL_CONTENT:
return resp
try:
(image_id, method,
version) = self._fetch_request_info(resp.request)
except TypeError:
return resp
if method == 'GET' and status_code == http.NO_CONTENT:
# Bugfix:1251055 - Don't cache non-existent image files.
# NOTE: Both GET for an image without locations and DELETE return
# 204 but DELETE should be processed.
return resp
method_str = '_process_%s_response' % method
try:
process_response_method = getattr(self, method_str)
except AttributeError:
LOG.error(_LE('could not find %s'), method_str)
# Nothing to do here, move along
return resp
else:
return process_response_method(resp, image_id, version=version)
def execute(self, image_id):
"""Finishing the task flow
:param image_id: Glance Image ID
"""
task = script_utils.get_task(self.task_repo, self.task_id)
if task is None:
return
try:
task.succeed({'image_id': image_id})
except Exception as e:
# Note: The message string contains Error in it to indicate
# in the task.message that it's a error message for the user.
# TODO(nikhil): need to bring back save_and_reraise_exception when
# necessary
log_msg = _LE("Task ID %(task_id)s failed. Error: %(exc_type)s: "
"%(e)s")
LOG.exception(log_msg, {'exc_type': six.text_type(type(e)),
'e': encodeutils.exception_to_unicode(e),
'task_id': task.task_id})
err_msg = _("Error: %(exc_type)s: %(e)s")
task.fail(err_msg % {'exc_type': six.text_type(type(e)),
'e': encodeutils.exception_to_unicode(e)})
finally:
self.task_repo.save(task)
LOG.info(_LI("%(task_id)s of %(task_type)s completed"),
{'task_id': self.task_id, 'task_type': self.task_type})
def _load_strategies():
"""Load all strategy modules."""
modules = {}
namespace = "glance.common.image_location_strategy.modules"
ex = stevedore.extension.ExtensionManager(namespace)
for module_name in ex.names():
try:
mgr = stevedore.driver.DriverManager(namespace=namespace,
name=module_name,
invoke_on_load=False)
# Obtain module name
strategy_name = str(mgr.driver.get_strategy_name())
if strategy_name in modules:
msg = (_('%(strategy)s is registered as a module twice. '
'%(module)s is not being used.') % {
'strategy': strategy_name,
'module': module_name
})
LOG.warn(msg)
else:
# Initialize strategy module
mgr.driver.init()
modules[strategy_name] = mgr.driver
except Exception as e:
LOG.error(
_LE("Failed to load location strategy module "
"%(module)s: %(e)s") % {
'module': module_name,
'e': e
})
return modules
def _delete_image_location_from_backend(self,
image_id,
loc_id,
uri,
backend=None):
try:
LOG.debug("Scrubbing image %s from a location.", image_id)
try:
if CONF.enabled_backends:
self.store_api.delete(uri, backend, self.admin_context)
else:
self.store_api.delete_from_backend(uri, self.admin_context)
except store_exceptions.NotFound:
LOG.info(
_LI("Image location for image '%s' not found in "
"backend; Marking image location deleted in "
"db."), image_id)
if loc_id != '-':
db_api.get_api().image_location_delete(self.admin_context,
image_id, int(loc_id),
'deleted')
LOG.info(_LI("Image %s is scrubbed from a location."), image_id)
except Exception as e:
LOG.error(
_LE("Unable to scrub image %(id)s from a location. "
"Reason: %(exc)s ") % {
'id': image_id,
'exc': encodeutils.exception_to_unicode(e)
})
raise
def execute(self, image_id):
"""Finishing the task flow
:param image_id: Glance Image ID
"""
task = script_utils.get_task(self.task_repo, self.task_id)
if task is None:
return
try:
task.succeed({'image_id': image_id})
except Exception as e:
# Note: The message string contains Error in it to indicate
# in the task.message that it's a error message for the user.
# TODO(nikhil): need to bring back save_and_reraise_exception when
# necessary
err_msg = ("Error: " + six.text_type(type(e)) + ': ' +
encodeutils.exception_to_unicode(e))
log_msg = err_msg + _LE("Task ID %s") % task.task_id
LOG.exception(log_msg)
task.fail(err_msg)
finally:
self.task_repo.save(task)
LOG.info(_LI("%(task_id)s of %(task_type)s completed"), {
'task_id': self.task_id,
'task_type': self.task_type
})
def revert(self, result, **kwargs):
if isinstance(result, failure.Failure):
LOG.error(
_LE('Task: %(task_id)s failed to copy image '
'%(image_id)s.'), {
'task_id': self.task_id,
'image_id': self.image_id
})
def revert(self, result, **kwargs):
if isinstance(result, failure.Failure):
LOG.exception(
_LE('Task: %(task_id)s failed to import image '
'%(image_id)s to the filesystem.'), {
'task_id': self.task_id,
'image_id': self.image_id
})
def _upload(self, req, image_meta):
"""
Uploads the payload of the request to a backend store in
Glance. If the `x-image-meta-store` header is set, Glance
will attempt to use that scheme; if not, Glance will use the
scheme set by the flag `default_store` to find the backing store.
:param req: The WSGI/Webob Request object
:param image_meta: Mapping of metadata about image
:raises HTTPConflict if image already exists
:retval The location where the image was stored
"""
scheme = req.headers.get("x-image-meta-store", CONF.glance_store.default_store)
store = self.get_store_or_400(req, scheme)
copy_from = self._copy_from(req)
if copy_from:
try:
image_data, image_size = self._get_from_store(req.context, copy_from, dest=store)
except Exception:
upload_utils.safe_kill(req, image_meta["id"], "queued")
msg = _LE("Copy from external source '%(scheme)s' failed for " "image: %(image)s") % {
"scheme": scheme,
"image": image_meta["id"],
}
LOG.exception(msg)
return
image_meta["size"] = image_size or image_meta["size"]
else:
try:
req.get_content_type(("application/octet-stream",))
except exception.InvalidContentType:
upload_utils.safe_kill(req, image_meta["id"], "queued")
msg = "Content-Type must be application/octet-stream"
LOG.debug(msg)
raise HTTPBadRequest(explanation=msg)
image_data = req.body_file
image_id = image_meta["id"]
LOG.debug("Setting image %s to status 'saving'", image_id)
registry.update_image_metadata(req.context, image_id, {"status": "saving"})
LOG.debug(
"Uploading image data for image %(image_id)s " "to %(scheme)s store",
{"image_id": image_id, "scheme": scheme},
)
self.notifier.info("image.prepare", redact_loc(image_meta))
image_meta, location_data = upload_utils.upload_data_to_store(req, image_meta, image_data, store, self.notifier)
self.notifier.info("image.upload", redact_loc(image_meta))
return location_data
def revert(self, image_id, result, **kwargs):
if isinstance(result, failure.Failure):
LOG.exception(_LE('Task: %(task_id)s failed to import image '
'%(image_id)s to the filesystem.'),
{'task_id': self.task_id, 'image_id': image_id})
return
if os.path.exists(result.split("file://")[-1]):
store_api.delete_from_backend(result)
def begin_processing(self, task_id):
try:
super(TaskExecutor, self).begin_processing(task_id)
except exception.ImportTaskError as exc:
LOG.error(_LE('Failed to execute task %(task_id)s: %(exc)s') %
{'task_id': task_id, 'exc': exc.msg})
task = self.task_repo.get(task_id)
task.fail(exc.msg)
self.task_repo.save(task)
def _compile_rule(self, rule):
try:
return re.compile(rule)
except Exception as e:
msg = (_LE("Encountered a malformed property protection rule"
" %(rule)s: %(error)s.") % {'rule': rule,
'error': e})
LOG.error(msg)
raise InvalidPropProtectConf()
def _cleanup_namespace(self, namespace_repo, namespace, namespace_created):
if namespace_created:
try:
namespace_obj = namespace_repo.get(namespace.namespace)
namespace_obj.delete()
namespace_repo.remove(namespace_obj)
LOG.debug("Cleaned up namespace %(namespace)s ", {"namespace": namespace.namespace})
except exception:
msg = _LE("Failed to delete namespace %(namespace)s ") % {"namespace": namespace.namespace}
LOG.error(msg)
def get_catalog_search_repo(self, context):
if self.es_api is None:
LOG.error(_LE('The search and index services are not available. '
'Ensure you have the necessary prerequisite '
'dependencies installed like elasticsearch to use '
'these services.'))
raise exception.SearchNotAvailable()
search_repo = glance.search.CatalogSearchRepo(context, self.es_api)
policy_search_repo = policy.CatalogSearchRepoProxy(
search_repo, context, self.policy)
return policy_search_repo
def revert(self, result, **kwargs):
if isinstance(result, failure.Failure):
LOG.error(_LE('Task: %(task_id)s failed to import image '
'%(image_id)s to the filesystem.'),
{'task_id': self.task_id,
'image_id': self.image_id})
# NOTE(abhishekk): Revert image state back to 'queued' as
# something went wrong.
image = self.image_repo.get(self.image_id)
image.status = 'queued'
self.image_repo.save(image)
def _verify_and_respawn_children(self, pid, status):
if len(self.stale_children) == 0:
LOG.debug("No stale children")
if os.WIFEXITED(status) and os.WEXITSTATUS(status) != 0:
LOG.error(_LE("Not respawning child %d, cannot " "recover from termination") % pid)
if not self.children and not self.stale_children:
LOG.info(_LI("All workers have terminated. Exiting"))
self.running = False
else:
if len(self.children) < CONF.workers:
self.run_child()