def size_checked_iter(response, image_meta, expected_size, image_iter,
notifier):
image_id = image_meta['id']
bytes_written = 0
def notify_image_sent_hook(env):
image_send_notification(bytes_written, expected_size,
image_meta, response.request, notifier)
# Add hook to process after response is fully sent
if 'eventlet.posthooks' in response.request.environ:
response.request.environ['eventlet.posthooks'].append(
(notify_image_sent_hook, (), {}))
try:
for chunk in image_iter:
yield chunk
bytes_written += len(chunk)
except Exception as err:
with excutils.save_and_reraise_exception():
msg = (_LE("An error occurred reading from backend storage for "
"image %(image_id)s: %(err)s") % {'image_id': image_id,
'err': err})
LOG.error(msg)
if expected_size != bytes_written:
msg = (_LE("Backend storage for image %(image_id)s "
"disconnected after writing only %(bytes_written)d "
"bytes") % {'image_id': image_id,
'bytes_written': bytes_written})
LOG.error(msg)
raise exception.GlanceException(_("Corrupt image download for "
"image %(image_id)s") %
{'image_id': image_id})
def run_task(task_id, task_type, context,
task_repo=None, image_repo=None, image_factory=None):
# TODO(nikhil): if task_repo is None get new task repo
# TODO(nikhil): if image_repo is None get new image repo
# TODO(nikhil): if image_factory is None get new image factory
LOG.info(_LI("Loading known task scripts for task_id %(task_id)s "
"of type %(task_type)s"), {'task_id': task_id,
'task_type': task_type})
if task_type == 'import':
image_import.run(task_id, context, task_repo,
image_repo, image_factory)
else:
msg = _LE("This task type %(task_type)s is not supported by the "
"current deployment of Glance. Please refer the "
"documentation provided by OpenStack or your operator "
"for more information.") % {'task_type': task_type}
LOG.error(msg)
task = task_repo.get(task_id)
task.fail(msg)
if task_repo:
task_repo.save(task)
else:
LOG.error(_LE("Failed to save task %(task_id)s in DB as task_repo "
"is %(task_repo)s"), {"task_id": task_id,
"task_repo": task_repo})
def _execute(t_id, task_repo, image_repo, image_factory):
task = script_utils.get_task(task_repo, t_id)
if task is None:
# NOTE: This happens if task is not found in the database. In
# such cases, there is no way to update the task status so,
# it's ignored here.
return
try:
task_input = script_utils.unpack_task_input(task)
uri = script_utils.validate_location_uri(task_input.get('import_from'))
image_id = import_image(image_repo, image_factory, task_input, t_id,
uri)
task.succeed({'image_id': image_id})
except Exception as e:
# Note: The message string contains Error in it to indicate
# in the task.message that it's a error message for the user.
# TODO(nikhil): need to bring back save_and_reraise_exception when
# necessary
err_msg = ("Error: " + six.text_type(type(e)) + ': ' +
encodeutils.exception_to_unicode(e))
log_msg = _LE(err_msg + ("Task ID %s" % task.task_id)) # noqa
LOG.exception(log_msg)
task.fail(_LE(err_msg)) # noqa
finally:
task_repo.save(task)
def _load_config(self):
try:
conf_file = CONF.find_file(CONF.swift_store_config_file)
CONFIG.read(conf_file)
except Exception as e:
msg = (_LE("swift config file %(conf_file)s:%(exc)s not found") % {
'conf_file': CONF.swift_store_config_file,
'exc': e
})
LOG.error(msg)
raise exception.InvalidSwiftStoreConfiguration()
account_params = {}
account_references = CONFIG.sections()
for ref in account_references:
reference = {}
try:
reference['auth_address'] = CONFIG.get(ref, 'auth_address')
reference['user'] = CONFIG.get(ref, 'user')
reference['key'] = CONFIG.get(ref, 'key')
account_params[ref] = reference
except (ValueError, SyntaxError, configparser.NoOptionError) as e:
LOG.exception(
_LE("Invalid format of swift store config "
"cfg"))
return account_params
def _execute(t_id, task_repo, image_repo, image_factory):
task = script_utils.get_task(task_repo, t_id)
if task is None:
# NOTE: This happens if task is not found in the database. In
# such cases, there is no way to update the task status so,
# it's ignored here.
return
try:
task_input = script_utils.unpack_task_input(task)
uri = script_utils.validate_location_uri(task_input.get('import_from'))
image_id = import_image(image_repo, image_factory, task_input, t_id,
uri)
task.succeed({'image_id': image_id})
except Exception as e:
# Note: The message string contains Error in it to indicate
# in the task.message that it's a error message for the user.
# TODO(nikhil): need to bring back save_and_reraise_exception when
# necessary
err_msg = ("Error: " + six.text_type(type(e)) + ': ' +
encodeutils.exception_to_unicode(e))
log_msg = _LE(err_msg + ("Task ID %s" % task.task_id)) # noqa
LOG.exception(log_msg)
task.fail(_LE(err_msg)) # noqa
finally:
task_repo.save(task)
def main():
"""The main function."""
try:
config.parse_args()
except RuntimeError as e:
sys.exit("ERROR: %s" % encodeutils.exception_to_unicode(e))
except SystemExit as e:
sys.exit("Please specify one command")
# Setup logging
logging.setup(CONF, 'xmonitor')
if CONF.token:
CONF.slavetoken = CONF.token
CONF.mastertoken = CONF.token
command = lookup_command(CONF.command)
try:
command(CONF, CONF.args)
except TypeError as e:
LOG.error(_LE(command.__doc__) % {'prog': command.__name__}) # noqa
sys.exit("ERROR: %s" % encodeutils.exception_to_unicode(e))
except ValueError as e:
LOG.error(_LE(command.__doc__) % {'prog': command.__name__}) # noqa
sys.exit("ERROR: %s" % encodeutils.exception_to_unicode(e))
def _run(self, task_id, task_type):
task = self.task_repo.get(task_id)
msg = _LE("This execution of Tasks is not setup. Please consult the "
"project documentation for more information on the "
"executors available.")
LOG.error(msg)
task.fail(_LE("Internal error occurred while trying to process task."))
self.task_repo.save(task)
def delete(self, req, id):
"""Deletes an existing image with the registry.
:param req: wsgi Request object
:param id: The opaque internal identifier for the image
:returns: 200 if delete was successful, a fault if not. On
success, the body contains the deleted image
information as a mapping.
"""
try:
deleted_image = self.db_api.image_destroy(req.context, id)
LOG.info(_LI("Successfully deleted image %(id)s"), {'id': id})
return dict(image=make_image_dict(deleted_image))
except exception.ForbiddenPublicImage:
LOG.info(_LI("Delete denied for public image %(id)s"), {'id': id})
raise exc.HTTPForbidden()
except exception.Forbidden:
# If it's private and doesn't belong to them, don't let on
# that it exists
LOG.info(
_LI("Access denied to image %(id)s but returning"
" 'not found'"), {'id': id})
return exc.HTTPNotFound()
except exception.ImageNotFound:
LOG.info(_LI("Image %(id)s not found"), {'id': id})
return exc.HTTPNotFound()
except Exception:
LOG.exception(_LE("Unable to delete image %s") % id)
raise
def _get_images(self, context, filters, **params):
"""Get images, wrapping in exception if necessary."""
# NOTE(markwash): for backwards compatibility, is_public=True for
# admins actually means "treat me as if I'm not an admin and show me
# all my images"
if context.is_admin and params.get('is_public') is True:
params['admin_as_user'] = True
del params['is_public']
try:
return self.db_api.image_get_all(context,
filters=filters,
**params)
except exception.ImageNotFound:
LOG.warn(
_LW("Invalid marker. Image %(id)s could not be "
"found.") % {'id': params.get('marker')})
msg = _("Invalid marker. Image could not be found.")
raise exc.HTTPBadRequest(explanation=msg)
except exception.Forbidden:
LOG.warn(
_LW("Access denied to image %(id)s but returning "
"'not found'") % {'id': params.get('marker')})
msg = _("Invalid marker. Image could not be found.")
raise exc.HTTPBadRequest(explanation=msg)
except Exception:
LOG.exception(_LE("Unable to get images"))
raise
def process_response(self, resp):
"""
We intercept the response coming back from the main
images Resource, removing image file from the cache
if necessary
"""
status_code = self.get_status_code(resp)
if not 200 <= status_code < 300:
return resp
try:
(image_id, method,
version) = self._fetch_request_info(resp.request)
except TypeError:
return resp
if method == 'GET' and status_code == 204:
# Bugfix:1251055 - Don't cache non-existent image files.
# NOTE: Both GET for an image without locations and DELETE return
# 204 but DELETE should be processed.
return resp
method_str = '_process_%s_response' % method
try:
process_response_method = getattr(self, method_str)
except AttributeError:
LOG.error(_LE('could not find %s') % method_str)
# Nothing to do here, move along
return resp
else:
return process_response_method(resp, image_id, version=version)
def migrate_location_credentials(migrate_engine, to_quoted):
"""
Migrate location credentials for swift uri's between the quoted
and unquoted forms.
:param migrate_engine: The configured db engine
:param to_quoted: If True, migrate location credentials from
unquoted to quoted form. If False, do the
reverse.
"""
meta = sqlalchemy.schema.MetaData()
meta.bind = migrate_engine
images_table = sqlalchemy.Table('images', meta, autoload=True)
images = list(
images_table.select(
images_table.c.location.startswith('swift')).execute())
for image in images:
try:
fixed_uri = legacy_parse_uri(image['location'], to_quoted)
images_table.update().where(
images_table.c.id == image['id']).values(
location=fixed_uri).execute()
except exception.BadStoreUri as e:
reason = encodeutils.exception_to_unicode(e)
msg = _LE("Invalid store uri for image: %(image_id)s. "
"Details: %(reason)s") % {
'image_id': image.id,
'reason': reason
}
LOG.exception(msg)
raise
def delete(self, req, id):
"""Deletes an existing image with the registry.
:param req: wsgi Request object
:param id: The opaque internal identifier for the image
:returns: 200 if delete was successful, a fault if not. On
success, the body contains the deleted image
information as a mapping.
"""
try:
deleted_image = self.db_api.image_destroy(req.context, id)
LOG.info(_LI("Successfully deleted image %(id)s"), {'id': id})
return dict(image=make_image_dict(deleted_image))
except exception.ForbiddenPublicImage:
LOG.info(_LI("Delete denied for public image %(id)s"), {'id': id})
raise exc.HTTPForbidden()
except exception.Forbidden:
# If it's private and doesn't belong to them, don't let on
# that it exists
LOG.info(_LI("Access denied to image %(id)s but returning"
" 'not found'"), {'id': id})
return exc.HTTPNotFound()
except exception.ImageNotFound:
LOG.info(_LI("Image %(id)s not found"), {'id': id})
return exc.HTTPNotFound()
except Exception:
LOG.exception(_LE("Unable to delete image %s") % id)
raise
def do_request(self, method, action, **kwargs):
try:
kwargs['headers'] = kwargs.get('headers', {})
kwargs['headers'].update(self.identity_headers or {})
if self._passed_request_id:
request_id = self._passed_request_id
if six.PY3 and isinstance(request_id, bytes):
request_id = request_id.decode('utf-8')
kwargs['headers']['X-Openstack-Request-ID'] = request_id
res = super(RegistryClient, self).do_request(method,
action,
**kwargs)
status = res.status
request_id = res.getheader('x-openstack-request-id')
if six.PY3 and isinstance(request_id, bytes):
request_id = request_id.decode('utf-8')
LOG.debug("Registry request %(method)s %(action)s HTTP %(status)s"
" request id %(request_id)s",
{'method': method, 'action': action,
'status': status, 'request_id': request_id})
except Exception as exc:
with excutils.save_and_reraise_exception():
exc_name = exc.__class__.__name__
LOG.exception(_LE("Registry client request %(method)s "
"%(action)s raised %(exc_name)s"),
{'method': method, 'action': action,
'exc_name': exc_name})
return res
def get_data(self, offset=0, chunk_size=None):
if not self.image.locations:
# NOTE(mclaren): This is the only set of arguments
# which work with this exception currently, see:
# https://bugs.launchpad.net/glance-store/+bug/1501443
# When the above glance_store bug is fixed we can
# add a msg as usual.
raise store.NotFound(image=None)
err = None
for loc in self.image.locations:
try:
data, size = self.store_api.get_from_backend(
loc['url'],
offset=offset,
chunk_size=chunk_size,
context=self.context)
return data
except Exception as e:
LOG.warn(
_LW('Get image %(id)s data failed: '
'%(err)s.') % {
'id': self.image.image_id,
'err': encodeutils.exception_to_unicode(e)
})
err = e
# tried all locations
LOG.error(
_LE('Glance tried all active locations to get data for '
'image %s but all have failed.') % self.image.image_id)
raise err
def cache_tee_iter(self, image_id, image_iter, image_checksum):
try:
current_checksum = hashlib.md5()
with self.driver.open_for_write(image_id) as cache_file:
for chunk in image_iter:
try:
cache_file.write(chunk)
finally:
current_checksum.update(chunk)
yield chunk
cache_file.flush()
if (image_checksum and
image_checksum != current_checksum.hexdigest()):
msg = _("Checksum verification failed. Aborted "
"caching of image '%s'.") % image_id
raise exception.GlanceException(msg)
except exception.GlanceException as e:
with excutils.save_and_reraise_exception():
# image_iter has given us bad, (size_checked_iter has found a
# bad length), or corrupt data (checksum is wrong).
LOG.exception(encodeutils.exception_to_unicode(e))
except Exception as e:
LOG.exception(_LE("Exception encountered while tee'ing "
"image '%(image_id)s' into cache: %(error)s. "
"Continuing with response.") %
{'image_id': image_id,
'error': encodeutils.exception_to_unicode(e)})
# If no checksum provided continue responding even if
# caching failed.
for chunk in image_iter:
yield chunk
def get_data(self, offset=0, chunk_size=None):
if not self.image.locations:
# NOTE(mclaren): This is the only set of arguments
# which work with this exception currently, see:
# https://bugs.launchpad.net/glance-store/+bug/1501443
# When the above glance_store bug is fixed we can
# add a msg as usual.
raise store.NotFound(image=None)
err = None
for loc in self.image.locations:
try:
data, size = self.store_api.get_from_backend(
loc['url'],
offset=offset,
chunk_size=chunk_size,
context=self.context)
return data
except Exception as e:
LOG.warn(_LW('Get image %(id)s data failed: '
'%(err)s.')
% {'id': self.image.image_id,
'err': encodeutils.exception_to_unicode(e)})
err = e
# tried all locations
LOG.error(_LE('Glance tried all active locations to get data for '
'image %s but all have failed.') % self.image.image_id)
raise err
def new_task_executor(self, context):
try:
# NOTE(flaper87): Backwards compatibility layer.
# It'll allow us to provide a deprecation path to
# users that are currently consuming the `eventlet`
# executor.
task_executor = CONF.task.task_executor
if task_executor == 'eventlet':
# NOTE(jokke): Making sure we do not log the deprecation
# warning 1000 times or anything crazy like that.
if not TaskExecutorFactory.eventlet_deprecation_warned:
msg = _LW("The `eventlet` executor has been deprecated. "
"Use `taskflow` instead.")
LOG.warn(msg)
TaskExecutorFactory.eventlet_deprecation_warned = True
task_executor = 'taskflow'
executor_cls = ('xmonitor.async.%s_executor.'
'TaskExecutor' % task_executor)
LOG.debug("Loading %s executor", task_executor)
executor = importutils.import_class(executor_cls)
return executor(context, self.task_repo, self.image_repo,
self.image_factory)
except ImportError:
with excutils.save_and_reraise_exception():
LOG.exception(
_LE("Failed to load the %s executor provided "
"in the config.") % CONF.task.task_executor)
def safe_delete_from_backend(context, image_id, location):
"""
Given a location, delete an image from the store and
update location status to db.
This function try to handle all known exceptions which might be raised
by those calls on store and DB modules in its implementation.
:param context: The request context
:param image_id: The image identifier
:param location: The image location entry
"""
try:
ret = store_api.delete_from_backend(location['url'], context=context)
location['status'] = 'deleted'
if 'id' in location:
db_api.get_api().image_location_delete(context, image_id,
location['id'], 'deleted')
return ret
except store_api.NotFound:
msg = _LW('Failed to delete image %s in store from URI') % image_id
LOG.warn(msg)
except store_api.StoreDeleteNotSupported as e:
LOG.warn(encodeutils.exception_to_unicode(e))
except store_api.UnsupportedBackend:
exc_type = sys.exc_info()[0].__name__
msg = (_LE('Failed to delete image %(image_id)s from store: %(exc)s') %
dict(image_id=image_id, exc=exc_type))
LOG.error(msg)
def safe_delete_from_backend(context, image_id, location):
"""
Given a location, delete an image from the store and
update location status to db.
This function try to handle all known exceptions which might be raised
by those calls on store and DB modules in its implementation.
:param context: The request context
:param image_id: The image identifier
:param location: The image location entry
"""
try:
ret = store_api.delete_from_backend(location['url'], context=context)
location['status'] = 'deleted'
if 'id' in location:
db_api.get_api().image_location_delete(context, image_id,
location['id'], 'deleted')
return ret
except store_api.NotFound:
msg = _LW('Failed to delete image %s in store from URI') % image_id
LOG.warn(msg)
except store_api.StoreDeleteNotSupported as e:
LOG.warn(encodeutils.exception_to_unicode(e))
except store_api.UnsupportedBackend:
exc_type = sys.exc_info()[0].__name__
msg = (_LE('Failed to delete image %(image_id)s from store: %(exc)s') %
dict(image_id=image_id, exc=exc_type))
LOG.error(msg)
def _load_strategies():
"""Load all strategy modules."""
modules = {}
namespace = "xmonitor.common.image_location_strategy.modules"
ex = stevedore.extension.ExtensionManager(namespace)
for module_name in ex.names():
try:
mgr = stevedore.driver.DriverManager(namespace=namespace,
name=module_name,
invoke_on_load=False)
# Obtain module name
strategy_name = str(mgr.driver.get_strategy_name())
if strategy_name in modules:
msg = (_('%(strategy)s is registered as a module twice. '
'%(module)s is not being used.') % {
'strategy': strategy_name,
'module': module_name
})
LOG.warn(msg)
else:
# Initialize strategy module
mgr.driver.init()
modules[strategy_name] = mgr.driver
except Exception as e:
LOG.error(
_LE("Failed to load location strategy module "
"%(module)s: %(e)s") % {
'module': module_name,
'e': e
})
return modules
def migrate_location_credentials(migrate_engine, to_quoted):
"""
Migrate location credentials for swift uri's between the quoted
and unquoted forms.
:param migrate_engine: The configured db engine
:param to_quoted: If True, migrate location credentials from
unquoted to quoted form. If False, do the
reverse.
"""
meta = sqlalchemy.schema.MetaData()
meta.bind = migrate_engine
images_table = sqlalchemy.Table('images', meta, autoload=True)
images = list(images_table.select(images_table.c.location.startswith(
'swift')).execute())
for image in images:
try:
fixed_uri = legacy_parse_uri(image['location'], to_quoted)
images_table.update().where(
images_table.c.id == image['id']).values(
location=fixed_uri).execute()
except exception.BadStoreUri as e:
reason = encodeutils.exception_to_unicode(e)
msg = _LE("Invalid store uri for image: %(image_id)s. "
"Details: %(reason)s") % {'image_id': image.id,
'reason': reason}
LOG.exception(msg)
raise
def _load_strategies():
"""Load all strategy modules."""
modules = {}
namespace = "xmonitor.common.image_location_strategy.modules"
ex = stevedore.extension.ExtensionManager(namespace)
for module_name in ex.names():
try:
mgr = stevedore.driver.DriverManager(
namespace=namespace,
name=module_name,
invoke_on_load=False)
# Obtain module name
strategy_name = str(mgr.driver.get_strategy_name())
if strategy_name in modules:
msg = (_('%(strategy)s is registered as a module twice. '
'%(module)s is not being used.') %
{'strategy': strategy_name, 'module': module_name})
LOG.warn(msg)
else:
# Initialize strategy module
mgr.driver.init()
modules[strategy_name] = mgr.driver
except Exception as e:
LOG.error(_LE("Failed to load location strategy module "
"%(module)s: %(e)s") % {'module': module_name,
'e': e})
return modules
def execute(self, image_id):
"""Finishing the task flow
:param image_id: Glance Image ID
"""
task = script_utils.get_task(self.task_repo, self.task_id)
if task is None:
return
try:
task.succeed({'image_id': image_id})
except Exception as e:
# Note: The message string contains Error in it to indicate
# in the task.message that it's a error message for the user.
# TODO(nikhil): need to bring back save_and_reraise_exception when
# necessary
err_msg = ("Error: " + six.text_type(type(e)) + ': ' +
encodeutils.exception_to_unicode(e))
log_msg = err_msg + _LE("Task ID %s") % task.task_id
LOG.exception(log_msg)
task.fail(err_msg)
finally:
self.task_repo.save(task)
LOG.info(_LI("%(task_id)s of %(task_type)s completed"), {
'task_id': self.task_id,
'task_type': self.task_type
})
def process_response(self, resp):
"""
We intercept the response coming back from the main
images Resource, removing image file from the cache
if necessary
"""
status_code = self.get_status_code(resp)
if not 200 <= status_code < 300:
return resp
try:
(image_id, method, version) = self._fetch_request_info(
resp.request)
except TypeError:
return resp
if method == 'GET' and status_code == 204:
# Bugfix:1251055 - Don't cache non-existent image files.
# NOTE: Both GET for an image without locations and DELETE return
# 204 but DELETE should be processed.
return resp
method_str = '_process_%s_response' % method
try:
process_response_method = getattr(self, method_str)
except AttributeError:
LOG.error(_LE('could not find %s') % method_str)
# Nothing to do here, move along
return resp
else:
return process_response_method(resp, image_id, version=version)
def execute(self, image_id):
"""Finishing the task flow
:param image_id: Glance Image ID
"""
task = script_utils.get_task(self.task_repo, self.task_id)
if task is None:
return
try:
task.succeed({'image_id': image_id})
except Exception as e:
# Note: The message string contains Error in it to indicate
# in the task.message that it's a error message for the user.
# TODO(nikhil): need to bring back save_and_reraise_exception when
# necessary
err_msg = ("Error: " + six.text_type(type(e)) + ': ' +
encodeutils.exception_to_unicode(e))
log_msg = err_msg + _LE("Task ID %s") % task.task_id
LOG.exception(log_msg)
task.fail(err_msg)
finally:
self.task_repo.save(task)
LOG.info(_LI("%(task_id)s of %(task_type)s completed"),
{'task_id': self.task_id, 'task_type': self.task_type})
def new_task_executor(self, context):
try:
# NOTE(flaper87): Backwards compatibility layer.
# It'll allow us to provide a deprecation path to
# users that are currently consuming the `eventlet`
# executor.
task_executor = CONF.task.task_executor
if task_executor == 'eventlet':
# NOTE(jokke): Making sure we do not log the deprecation
# warning 1000 times or anything crazy like that.
if not TaskExecutorFactory.eventlet_deprecation_warned:
msg = _LW("The `eventlet` executor has been deprecated. "
"Use `taskflow` instead.")
LOG.warn(msg)
TaskExecutorFactory.eventlet_deprecation_warned = True
task_executor = 'taskflow'
executor_cls = ('xmonitor.async.%s_executor.'
'TaskExecutor' % task_executor)
LOG.debug("Loading %s executor", task_executor)
executor = importutils.import_class(executor_cls)
return executor(context,
self.task_repo,
self.image_repo,
self.image_factory)
except ImportError:
with excutils.save_and_reraise_exception():
LOG.exception(_LE("Failed to load the %s executor provided "
"in the config.") % CONF.task.task_executor)
def _run(self, task_id, task_type):
LOG.debug('Taskflow executor picked up the execution of task ID '
'%(task_id)s of task type '
'%(task_type)s', {'task_id': task_id,
'task_type': task_type})
task = script_utils.get_task(self.task_repo, task_id)
if task is None:
# NOTE: This happens if task is not found in the database. In
# such cases, there is no way to update the task status so,
# it's ignored here.
return
flow = self._get_flow(task)
executor = self._fetch_an_executor()
try:
engine = engines.load(
flow,
engine=CONF.taskflow_executor.engine_mode, executor=executor,
max_workers=CONF.taskflow_executor.max_workers)
with llistener.DynamicLoggingListener(engine, log=LOG):
engine.run()
except Exception as exc:
with excutils.save_and_reraise_exception():
LOG.error(_LE('Failed to execute task %(task_id)s: %(exc)s') %
{'task_id': task_id,
'exc': encodeutils.exception_to_unicode(exc)})
# TODO(sabari): Check for specific exceptions and update the
# task failure message.
task.fail(_('Task failed due to Internal Error'))
self.task_repo.save(task)
finally:
if executor is not None:
executor.shutdown()
def begin_processing(self, task_id):
try:
super(TaskExecutor, self).begin_processing(task_id)
except exception.ImportTaskError as exc:
LOG.error(_LE('Failed to execute task %(task_id)s: %(exc)s') %
{'task_id': task_id, 'exc': exc.msg})
task = self.task_repo.get(task_id)
task.fail(exc.msg)
self.task_repo.save(task)
def _compile_rule(self, rule):
try:
return re.compile(rule)
except Exception as e:
msg = (_LE("Encountered a malformed property protection rule"
" %(rule)s: %(error)s.") % {'rule': rule,
'error': e})
LOG.error(msg)
raise InvalidPropProtectConf()
def revert(self, image_id, result, **kwargs):
if isinstance(result, failure.Failure):
LOG.exception(_LE('Task: %(task_id)s failed to import image '
'%(image_id)s to the filesystem.') %
{'task_id': self.task_id, 'image_id': image_id})
return
if os.path.exists(result.split("file://")[-1]):
store_api.delete_from_backend(result)
def _compile_rule(self, rule):
try:
return re.compile(rule)
except Exception as e:
msg = (_LE("Encountered a malformed property protection rule"
" %(rule)s: %(error)s.") % {
'rule': rule,
'error': e
})
LOG.error(msg)
raise InvalidPropProtectConf()
def db_load_metadefs(engine,
metadata_path=None,
merge=False,
prefer_new=False,
overwrite=False):
meta = MetaData()
meta.bind = engine
if not merge and (prefer_new or overwrite):
LOG.error(
_LE("To use --prefer_new or --overwrite you need to combine "
"of these options with --merge option."))
return
if prefer_new and overwrite and merge:
LOG.error(
_LE("Please provide no more than one option from this list: "
"--prefer_new, --overwrite"))
return
_populate_metadata(meta, metadata_path, merge, prefer_new, overwrite)
def _cleanup_namespace(self, namespace_repo, namespace, namespace_created):
if namespace_created:
try:
namespace_obj = namespace_repo.get(namespace.namespace)
namespace_obj.delete()
namespace_repo.remove(namespace_obj)
LOG.debug("Cleaned up namespace %(namespace)s ",
{'namespace': namespace.namespace})
except exception:
msg = (_LE("Failed to delete namespace %(namespace)s ") %
{'namespace': namespace.namespace})
LOG.error(msg)
def revert(self, image_id, result, **kwargs):
if isinstance(result, failure.Failure):
LOG.exception(
_LE('Task: %(task_id)s failed to import image '
'%(image_id)s to the filesystem.') % {
'task_id': self.task_id,
'image_id': image_id
})
return
if os.path.exists(result.split("file://")[-1]):
store_api.delete_from_backend(result)
def _verify_and_respawn_children(self, pid, status):
if len(self.stale_children) == 0:
LOG.debug('No stale children')
if os.WIFEXITED(status) and os.WEXITSTATUS(status) != 0:
LOG.error(
_LE('Not respawning child %d, cannot '
'recover from termination') % pid)
if not self.children and not self.stale_children:
LOG.info(_LI('All workers have terminated. Exiting'))
self.running = False
else:
if len(self.children) < get_num_workers():
self.run_child()
def _load_config(self):
try:
conf_file = CONF.find_file(CONF.swift_store_config_file)
CONFIG.read(conf_file)
except Exception as e:
msg = (_LE("swift config file %(conf_file)s:%(exc)s not found") %
{'conf_file': CONF.swift_store_config_file, 'exc': e})
LOG.error(msg)
raise exception.InvalidSwiftStoreConfiguration()
account_params = {}
account_references = CONFIG.sections()
for ref in account_references:
reference = {}
try:
reference['auth_address'] = CONFIG.get(ref, 'auth_address')
reference['user'] = CONFIG.get(ref, 'user')
reference['key'] = CONFIG.get(ref, 'key')
account_params[ref] = reference
except (ValueError, SyntaxError, configparser.NoOptionError) as e:
LOG.exception(_LE("Invalid format of swift store config "
"cfg"))
return account_params
def _cleanup_namespace(self, namespace_repo, namespace, namespace_created):
if namespace_created:
try:
namespace_obj = namespace_repo.get(namespace.namespace)
namespace_obj.delete()
namespace_repo.remove(namespace_obj)
LOG.debug("Cleaned up namespace %(namespace)s ",
{'namespace': namespace.namespace})
except exception:
msg = (_LE("Failed to delete namespace %(namespace)s ") % {
'namespace': namespace.namespace
})
LOG.error(msg)
def run_task(task_id,
task_type,
context,
task_repo=None,
image_repo=None,
image_factory=None):
# TODO(nikhil): if task_repo is None get new task repo
# TODO(nikhil): if image_repo is None get new image repo
# TODO(nikhil): if image_factory is None get new image factory
LOG.info(
_LI("Loading known task scripts for task_id %(task_id)s "
"of type %(task_type)s"), {
'task_id': task_id,
'task_type': task_type
})
if task_type == 'import':
image_import.run(task_id, context, task_repo, image_repo,
image_factory)
else:
msg = _LE("This task type %(task_type)s is not supported by the "
"current deployment of Glance. Please refer the "
"documentation provided by OpenStack or your operator "
"for more information.") % {
'task_type': task_type
}
LOG.error(msg)
task = task_repo.get(task_id)
task.fail(msg)
if task_repo:
task_repo.save(task)
else:
LOG.error(
_LE("Failed to save task %(task_id)s in DB as task_repo "
"is %(task_repo)s"), {
"task_id": task_id,
"task_repo": task_repo
})
def get_task(task_repo, task_id):
"""Gets a TaskProxy object.
:param task_repo: TaskRepo object used to perform DB operations
:param task_id: ID of the Task
"""
task = None
try:
task = task_repo.get(task_id)
except exception.NotFound:
msg = _LE('Task not found for task_id %s') % task_id
LOG.exception(msg)
return task
def _get_delete_jobs(self):
try:
records = self.db_queue.get_all_locations()
except Exception as err:
LOG.error(_LE("Can not get scrub jobs from queue: %s") %
encodeutils.exception_to_unicode(err))
return {}
delete_jobs = {}
for image_id, loc_id, loc_uri in records:
if image_id not in delete_jobs:
delete_jobs[image_id] = []
delete_jobs[image_id].append((image_id, loc_id, loc_uri))
return delete_jobs
def get_task(task_repo, task_id):
"""Gets a TaskProxy object.
:param task_repo: TaskRepo object used to perform DB operations
:param task_id: ID of the Task
"""
task = None
try:
task = task_repo.get(task_id)
except exception.NotFound:
msg = _LE('Task not found for task_id %s') % task_id
LOG.exception(msg)
return task
def create(self, req, body):
"""Registers a new image with the registry.
:param req: wsgi Request object
:param body: Dictionary of information about the image
:returns: The newly-created image information as a mapping,
which will include the newly-created image's internal id
in the 'id' field
"""
image_data = body['image']
# Ensure the image has a status set
image_data.setdefault('status', 'active')
# Set up the image owner
if not req.context.is_admin or 'owner' not in image_data:
image_data['owner'] = req.context.owner
image_id = image_data.get('id')
if image_id and not uuidutils.is_uuid_like(image_id):
LOG.info(
_LI("Rejecting image creation request for invalid image "
"id '%(bad_id)s'"), {'bad_id': image_id})
msg = _("Invalid image id format")
return exc.HTTPBadRequest(explanation=msg)
if 'location' in image_data:
image_data['locations'] = [image_data.pop('location')]
try:
image_data = _normalize_image_location_for_db(image_data)
image_data = self.db_api.image_create(req.context, image_data)
image_data = dict(image=make_image_dict(image_data))
LOG.info(_LI("Successfully created image %(id)s"),
{'id': image_data['image']['id']})
return image_data
except exception.Duplicate:
msg = _("Image with identifier %s already exists!") % image_id
LOG.warn(msg)
return exc.HTTPConflict(msg)
except exception.Invalid as e:
msg = (_("Failed to add image metadata. "
"Got error: %s") % encodeutils.exception_to_unicode(e))
LOG.error(msg)
return exc.HTTPBadRequest(msg)
except Exception:
LOG.exception(_LE("Unable to create image %s"), image_id)
raise
def _restore(self, artifact_repo, artifact):
"""Restore the artifact to queued status.
:param artifact_repo: The instance of ArtifactRepo
:param artifact: The artifact will be restored
"""
try:
if artifact_repo and artifact:
artifact.state = 'creating'
artifact_repo.save(artifact)
except Exception as e:
msg = (_LE("Unable to restore artifact %(artifact_id)s: %(e)s") %
{'artifact_id': artifact.id,
'e': encodeutils.exception_to_unicode(e)})
LOG.exception(msg)
def cooperative_iter(iter):
"""
Return an iterator which schedules after each
iteration. This can prevent eventlet thread starvation.
:param iter: an iterator to wrap
"""
try:
for chunk in iter:
sleep(0)
yield chunk
except Exception as err:
with excutils.save_and_reraise_exception():
msg = _LE("Error: cooperative_iter exception %s") % err
LOG.error(msg)
def safe_kill(req, image_id, from_state):
"""
Mark image killed without raising exceptions if it fails.
Since _kill is meant to be called from exceptions handlers, it should
not raise itself, rather it should just log its error.
:param req: The WSGI/Webob Request object
:param image_id: Opaque image identifier
:param from_state: Permitted current status for transition to 'killed'
"""
try:
_kill(req, image_id, from_state)
except Exception:
LOG.exception(_LE("Unable to kill image %(id)s: ") % {'id': image_id})
def safe_kill(req, image_id, from_state):
"""
Mark image killed without raising exceptions if it fails.
Since _kill is meant to be called from exceptions handlers, it should
not raise itself, rather it should just log its error.
:param req: The WSGI/Webob Request object
:param image_id: Opaque image identifier
:param from_state: Permitted current status for transition to 'killed'
"""
try:
_kill(req, image_id, from_state)
except Exception:
LOG.exception(_LE("Unable to kill image %(id)s: ") % {'id': image_id})
def _delete(self, image_repo, image):
"""Delete the image.
:param image_repo: The instance of ImageRepo
:param image: The image that will be deleted
"""
try:
if image_repo and image:
image.status = 'killed'
image_repo.save(image)
except Exception as e:
msg = (_LE("Unable to delete image %(image_id)s: %(e)s") %
{'image_id': image.image_id,
'e': encodeutils.exception_to_unicode(e)})
LOG.exception(msg)
def create(self, req, body):
"""Registers a new image with the registry.
:param req: wsgi Request object
:param body: Dictionary of information about the image
:returns: The newly-created image information as a mapping,
which will include the newly-created image's internal id
in the 'id' field
"""
image_data = body['image']
# Ensure the image has a status set
image_data.setdefault('status', 'active')
# Set up the image owner
if not req.context.is_admin or 'owner' not in image_data:
image_data['owner'] = req.context.owner
image_id = image_data.get('id')
if image_id and not uuidutils.is_uuid_like(image_id):
LOG.info(_LI("Rejecting image creation request for invalid image "
"id '%(bad_id)s'"), {'bad_id': image_id})
msg = _("Invalid image id format")
return exc.HTTPBadRequest(explanation=msg)
if 'location' in image_data:
image_data['locations'] = [image_data.pop('location')]
try:
image_data = _normalize_image_location_for_db(image_data)
image_data = self.db_api.image_create(req.context, image_data)
image_data = dict(image=make_image_dict(image_data))
LOG.info(_LI("Successfully created image %(id)s"),
{'id': image_data['image']['id']})
return image_data
except exception.Duplicate:
msg = _("Image with identifier %s already exists!") % image_id
LOG.warn(msg)
return exc.HTTPConflict(msg)
except exception.Invalid as e:
msg = (_("Failed to add image metadata. "
"Got error: %s") % encodeutils.exception_to_unicode(e))
LOG.error(msg)
return exc.HTTPBadRequest(msg)
except Exception:
LOG.exception(_LE("Unable to create image %s"), image_id)
raise
def _restore(self, image_repo, image):
"""
Restore the image to queued status.
:param image_repo: The instance of ImageRepo
:param image: The image will be restored
"""
try:
if image_repo and image:
image.status = 'queued'
image_repo.save(image)
except Exception as e:
msg = (_LE("Unable to restore image %(image_id)s: %(e)s") %
{'image_id': image.image_id,
'e': encodeutils.exception_to_unicode(e)})
LOG.exception(msg)
def _delete(self, image_repo, image):
"""Delete the image.
:param image_repo: The instance of ImageRepo
:param image: The image that will be deleted
"""
try:
if image_repo and image:
image.status = 'killed'
image_repo.save(image)
except Exception as e:
msg = (_LE("Unable to delete image %(image_id)s: %(e)s") % {
'image_id': image.image_id,
'e': encodeutils.exception_to_unicode(e)
})
LOG.exception(msg)
def _restore(self, artifact_repo, artifact):
"""Restore the artifact to queued status.
:param artifact_repo: The instance of ArtifactRepo
:param artifact: The artifact will be restored
"""
try:
if artifact_repo and artifact:
artifact.state = 'creating'
artifact_repo.save(artifact)
except Exception as e:
msg = (_LE("Unable to restore artifact %(artifact_id)s: %(e)s") % {
'artifact_id': artifact.id,
'e': encodeutils.exception_to_unicode(e)
})
LOG.exception(msg)
def __init__(self, namespace):
self.mgr = enabled.EnabledExtensionManager(
check_func=self._gen_check_func(),
namespace=namespace,
propagate_map_exceptions=True,
on_load_failure_callback=self._on_load_failure)
self.plugin_map = {'by_typename': {},
'by_endpoint': {}}
def _add_extension(ext):
"""
Plugins can be loaded as entry_point=single plugin and
entry_point=PLUGIN_LIST, where PLUGIN_LIST is a python variable
holding a list of plugins
"""
def _load_one(plugin):
if issubclass(plugin, definitions.ArtifactType):
# make sure that have correct plugin name
art_name = plugin.metadata.type_name
if art_name != ext.name:
raise exception.ArtifactNonMatchingTypeName(
name=art_name, plugin=ext.name)
# make sure that no plugin with the same name and version
# already exists
exists = self._get_plugins(ext.name)
new_tv = plugin.metadata.type_version
if any(e.metadata.type_version == new_tv for e in exists):
raise exception.ArtifactDuplicateNameTypeVersion()
self._add_plugin("by_endpoint", plugin.metadata.endpoint,
plugin)
self._add_plugin("by_typename", plugin.metadata.type_name,
plugin)
if isinstance(ext.plugin, list):
for p in ext.plugin:
_load_one(p)
else:
_load_one(ext.plugin)
# (ivasilevskaya) that looks pretty bad as RuntimeError is too general,
# but stevedore has awful exception wrapping with no specific class
# for this very case (no extensions for given namespace found)
try:
self.mgr.map(_add_extension)
except RuntimeError as re:
LOG.error(_LE("Unable to load artifacts: %s") % re.message)
