def main():
CONF.register_cli_opt(command_opt)
try:
cfg_files = cfg.find_config_files(project='glance',
prog='glance-registry')
cfg_files.extend(cfg.find_config_files(project='glance',
prog='glance-api'))
config.parse_args(default_config_files=cfg_files,
usage="%(prog)s [options] <cmd>")
log.setup('glance')
except RuntimeError as e:
sys.exit("ERROR: %s" % e)
try:
if CONF.command.action.startswith('db'):
return CONF.command.action_fn()
else:
func_kwargs = {}
for k in CONF.command.action_kwargs:
v = getattr(CONF.command, 'action_kwarg_' + k)
if v is None:
continue
func_kwargs[k] = encodeutils.safe_decode(v)
func_args = [encodeutils.safe_decode(arg)
for arg in CONF.command.action_args]
return CONF.command.action_fn(*func_args, **func_kwargs)
except exception.GlanceException as e:
sys.exit("ERROR: %s" % utils.exception_to_str(e))
def _safe_header(self, name, value):
if name in SENSITIVE_HEADERS:
# because in python3 byte string handling is ... ug
v = value.encode("utf-8")
h = hashlib.sha1(v)
d = h.hexdigest()
return encodeutils.safe_decode(name), "{SHA1}%s" % d
else:
return (encodeutils.safe_decode(name), encodeutils.safe_decode(value))
def _safe_header(self, name, value):
if name in SENSITIVE_HEADERS:
# because in python3 byte string handling is ... ug
v = value.encode('utf-8')
h = hashlib.sha1(v)
d = h.hexdigest()
return encodeutils.safe_decode(name), "{SHA1}%s" % d
else:
return (encodeutils.safe_decode(name),
encodeutils.safe_decode(value))
def log_http_response(resp, body=None):
status = (resp.raw.version / 10.0, resp.status_code, resp.reason)
dump = ['\nHTTP/%.1f %s %s' % status]
headers = resp.headers.items()
dump.extend(['%s: %s' % safe_header(k, v) for k, v in headers])
dump.append('')
if body:
body = encodeutils.safe_decode(body)
dump.extend([body, ''])
LOG.debug('\n'.join([encodeutils.safe_decode(x, errors='ignore')
for x in dump]))
def log_curl_request(self, method, url, headers, data, kwargs):
curl = ['curl -g -i -X %s' % method]
headers = copy.deepcopy(headers)
headers.update(self.session.headers)
for (key, value) in six.iteritems(headers):
header = '-H \'%s: %s\'' % safe_header(key, value)
curl.append(header)
if not self.session.verify:
curl.append('-k')
else:
if isinstance(self.session.verify, six.string_types):
curl.append(' --cacert %s' % self.session.verify)
if self.session.cert:
curl.append(' --cert %s --key %s' % self.session.cert)
if data and isinstance(data, six.string_types):
curl.append('-d \'%s\'' % data)
curl.append(url)
msg = ' '.join([encodeutils.safe_decode(item, errors='ignore')
for item in curl])
LOG.debug(msg)
def _get_claim(self, message_id):
"""Gets minimal claim doc for a message.
:returns: {'id': cid, 'expires': ts} IFF the message is claimed,
and that claim has not expired.
"""
claim = self._client.hmget(message_id, 'c', 'c.e')
if claim == [None, None]:
# NOTE(kgriffs): message_id was not found
return None
info = {
# NOTE(kgriffs): A "None" claim is serialized as an empty str
'id': encodeutils.safe_decode(claim[0]) or None,
'expires': int(claim[1]),
}
# Is the message claimed?
now = timeutils.utcnow_ts()
if info['id'] and (now < info['expires']):
return info
# Not claimed
return None
def test_exception_to_dict_with_long_message(self):
# Generate Russian byte message whose length is 300
msg = encodeutils.safe_decode(' \xd0\xb2' * 100)
exc = exception.NovaException(message=msg)
fault_dict = compute_utils.exception_to_dict(exc)
byte_message = encodeutils.safe_encode(fault_dict["message"])
self.assertEqual(255, len(byte_message))
def exception_to_dict(fault):
"""Converts exceptions to a dict for use in notifications."""
# TODO(johngarbutt) move to nova/exception.py to share with wrap_exception
code = 500
if hasattr(fault, "kwargs"):
code = fault.kwargs.get('code', 500)
# get the message from the exception that was thrown
# if that does not exist, use the name of the exception class itself
try:
message = fault.format_message()
# These exception handlers are broad so we don't fail to log the fault
# just because there is an unexpected error retrieving the message
except Exception:
try:
message = unicode(fault)
except Exception:
message = None
if not message:
message = fault.__class__.__name__
# NOTE(dripton) The message field in the database is limited to 255 chars.
# MySQL silently truncates overly long messages, but PostgreSQL throws an
# error if we don't truncate it.
u_message = encodeutils.safe_decode(encodeutils.safe_encode(message)[:255])
fault_dict = dict(exception=fault)
fault_dict["message"] = u_message
fault_dict["code"] = code
return fault_dict
def exception_to_dict(fault):
"""Converts exceptions to a dict for use in notifications."""
# TODO(johngarbutt) move to nova/exception.py to share with wrap_exception
code = 500
if hasattr(fault, "kwargs"):
code = fault.kwargs.get('code', 500)
# get the message from the exception that was thrown
# if that does not exist, use the name of the exception class itself
try:
message = fault.format_message()
# These exception handlers are broad so we don't fail to log the fault
# just because there is an unexpected error retrieving the message
except Exception:
try:
message = unicode(fault)
except Exception:
message = None
if not message:
message = fault.__class__.__name__
# NOTE(dripton) The message field in the database is limited to 255 chars.
# MySQL silently truncates overly long messages, but PostgreSQL throws an
# error if we don't truncate it.
u_message = encodeutils.safe_decode(encodeutils.safe_encode(message)[:255])
fault_dict = dict(exception=fault)
fault_dict["message"] = u_message
fault_dict["code"] = code
return fault_dict
def test_exception_to_dict_with_long_message(self):
# Generate Russian byte message whose length is 300
msg = encodeutils.safe_decode(' \xd0\xb2' * 100)
exc = exception.NovaException(message=msg)
fault_dict = compute_utils.exception_to_dict(exc)
byte_message = encodeutils.safe_encode(fault_dict["message"])
self.assertEqual(255, len(byte_message))
def _get_claim(self, message_id):
"""Gets minimal claim doc for a message.
:returns: {'id': cid, 'expires': ts} IFF the message is claimed,
and that claim has not expired.
"""
claim = self._client.hmget(message_id, 'c', 'c.e')
if claim == [None, None]:
# NOTE(kgriffs): message_id was not found
return None
info = {
# NOTE(kgriffs): A "None" claim is serialized as an empty str
'id': encodeutils.safe_decode(claim[0]) or None,
'expires': int(claim[1]),
}
# Is the message claimed?
now = timeutils.utcnow_ts()
if info['id'] and (now < info['expires']):
return info
# Not claimed
return None
def find_resource(manager, name_or_id):
"""Helper for the _find_* methods."""
# first try to get entity as integer id
try:
if isinstance(name_or_id, int) or name_or_id.isdigit():
return manager.get(int(name_or_id))
except exc.NotFound:
pass
# now try to get entity as uuid
try:
# This must be unicode for Python 3 compatibility.
# If you pass a bytestring to uuid.UUID, you will get a TypeError
uuid.UUID(encodeutils.safe_decode(name_or_id))
return manager.get(name_or_id)
except (ValueError, exc.NotFound):
pass
# finally try to find entity by name
matches = list(manager.list(filters={'name': name_or_id}))
num_matches = len(matches)
if num_matches == 0:
msg = "No %s with a name or ID of '%s' exists." % \
(manager.resource_class.__name__.lower(), name_or_id)
raise exc.CommandError(msg)
elif num_matches > 1:
msg = ("Multiple %s matches found for '%s', use an ID to be more"
" specific." % (manager.resource_class.__name__.lower(),
name_or_id))
raise exc.CommandError(msg)
else:
return matches[0]
def print_dict(d, max_column_width=80):
pt = prettytable.PrettyTable(['Property', 'Value'], caching=False)
pt.align = 'l'
pt.max_width = max_column_width
for k, v in six.iteritems(d):
if isinstance(v, (dict, list)):
v = json.dumps(v)
pt.add_row([k, v])
print(encodeutils.safe_decode(pt.get_string(sortby='Property')))
def test_exception_to_dict_with_long_message_2_bytes(self):
# Generate Russian byte string whose length is 300. This Russian UTF-8
# character occupies 2 bytes. After truncating, the byte string length
# should be 254.
msg = encodeutils.safe_decode('\xd0\x92' * 150)
exc = exception.NovaException(message=msg)
fault_dict = compute_utils.exception_to_dict(exc)
byte_message = encodeutils.safe_encode(fault_dict["message"])
self.assertEqual(254, len(byte_message))
def test_exception_to_dict_with_long_message_3_bytes(self):
# Generate Chinese byte string whose length is 300. This Chinese UTF-8
# character occupies 3 bytes. After truncating, the byte string length
# should be 255.
msg = encodeutils.safe_decode('\xe8\xb5\xb5' * 100)
exc = exception.NovaException(message=msg)
fault_dict = compute_utils.exception_to_dict(exc)
byte_message = encodeutils.safe_encode(fault_dict["message"])
self.assertEqual(255, len(byte_message))
def loads(s, encoding='utf-8', **kwargs):
"""Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON
:param s: string to deserialize
:param encoding: encoding used to interpret the string
:param kwargs: extra named parameters, please see documentation
of `json.loads <https://docs.python.org/2/library/json.html#basic-usage>`_
:returns: python object
"""
return json.loads(encodeutils.safe_decode(s, encoding), **kwargs)
def test_exception_to_dict_with_long_message_2_bytes(self):
# Generate Russian byte string whose length is 300. This Russian UTF-8
# character occupies 2 bytes. After truncating, the byte string length
# should be 254.
msg = encodeutils.safe_decode('\xd0\x92' * 150)
exc = exception.NovaException(message=msg)
fault_dict = compute_utils.exception_to_dict(exc)
byte_message = encodeutils.safe_encode(fault_dict["message"])
self.assertEqual(254, len(byte_message))
def loads(s, encoding='utf-8', **kwargs):
"""Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON
:param s: string to deserialize
:param encoding: encoding used to interpret the string
:param kwargs: extra named parameters, please see documentation
of `json.loads <https://docs.python.org/2/library/json.html#basic-usage>`_
:returns: python object
"""
return json.loads(encodeutils.safe_decode(s, encoding), **kwargs)
def _hmap_to_msgenv_kwargs(hmap):
claim_id = hmap[b'c']
if claim_id:
claim_id = encodeutils.safe_decode(claim_id)
else:
claim_id = None
# NOTE(kgriffs): Under Py3K, redis-py converts all strings
# into binary. Woohoo!
return {
'id': encodeutils.safe_decode(hmap[b'id']),
'ttl': int(hmap[b't']),
'created': int(hmap[b'cr']),
'expires': int(hmap[b'e']),
'client_uuid': encodeutils.safe_decode(hmap[b'u']),
'claim_id': claim_id,
'claim_expires': int(hmap[b'c.e']),
}
def test_exception_to_dict_with_long_message_3_bytes(self):
# Generate Chinese byte string whose length is 300. This Chinese UTF-8
# character occupies 3 bytes. After truncating, the byte string length
# should be 255.
msg = encodeutils.safe_decode('\xe8\xb5\xb5' * 100)
exc = exception.NovaException(message=msg)
fault_dict = compute_utils.exception_to_dict(exc)
byte_message = encodeutils.safe_encode(fault_dict["message"])
self.assertEqual(255, len(byte_message))
def exception_to_str(exc):
try:
error = six.text_type(exc)
except UnicodeError:
try:
error = str(exc)
except UnicodeError:
error = ("Caught '%(exception)s' exception." %
{"exception": exc.__class__.__name__})
return encodeutils.safe_decode(error, errors='ignore')
def _hmap_to_msgenv_kwargs(hmap):
claim_id = hmap[b'c']
if claim_id:
claim_id = encodeutils.safe_decode(claim_id)
else:
claim_id = None
# NOTE(kgriffs): Under Py3K, redis-py converts all strings
# into binary. Woohoo!
return {
'id': encodeutils.safe_decode(hmap[b'id']),
'ttl': int(hmap[b't']),
'created': int(hmap[b'cr']),
'expires': int(hmap[b'e']),
'client_uuid': encodeutils.safe_decode(hmap[b'u']),
'claim_id': claim_id,
'claim_expires': int(hmap[b'c.e']),
}
def validate(self):
'''
Validates the template.
'''
# TODO(sdake) Should return line number of invalid reference
# validate overall template (top-level structure)
self.t.validate()
# Validate parameters
self.parameters.validate(context=self.context,
validate_value=self.strict_validate)
# Validate Parameter Groups
parameter_groups = param_groups.ParameterGroups(self.t)
parameter_groups.validate()
# Check duplicate names between parameters and resources
dup_names = set(self.parameters.keys()) & set(self.keys())
if dup_names:
LOG.debug("Duplicate names %s" % dup_names)
raise exception.StackValidationFailed(
message=_("Duplicate names %s") % dup_names)
for res in self.dependencies:
try:
result = res.validate()
except exception.HeatException as ex:
LOG.info(ex)
raise ex
except Exception as ex:
LOG.exception(ex)
raise exception.StackValidationFailed(
message=encodeutils.safe_decode(six.text_type(ex)))
if result:
raise exception.StackValidationFailed(message=result)
for val in self.outputs.values():
try:
if not val or not val.get('Value'):
message = _('Each Output must contain '
'a Value key.')
raise exception.StackValidationFailed(message=message)
function.validate(val.get('Value'))
except AttributeError:
message = _('Output validation error: '
'Outputs must contain Output. '
'Found a [%s] instead') % type(val)
raise exception.StackValidationFailed(message=message)
except Exception as ex:
reason = _('Output validation error: '
'%s') % six.text_type(ex)
raise exception.StackValidationFailed(message=reason)
def validate(self):
'''
Validates the template.
'''
# TODO(sdake) Should return line number of invalid reference
# validate overall template (top-level structure)
self.t.validate()
# Validate parameters
self.parameters.validate(context=self.context)
# Validate Parameter Groups
parameter_groups = param_groups.ParameterGroups(self.t)
parameter_groups.validate()
# Check duplicate names between parameters and resources
dup_names = set(self.parameters.keys()) & set(self.keys())
if dup_names:
LOG.debug("Duplicate names %s" % dup_names)
raise exception.StackValidationFailed(
message=_("Duplicate names %s") % dup_names)
for res in self.dependencies:
try:
result = res.validate()
except exception.HeatException as ex:
LOG.info(ex)
raise ex
except Exception as ex:
LOG.exception(ex)
raise exception.StackValidationFailed(
message=encodeutils.safe_decode(six.text_type(ex)))
if result:
raise exception.StackValidationFailed(message=result)
for val in self.outputs.values():
try:
if not val or not val.get('Value'):
message = _('Each Output must contain '
'a Value key.')
raise exception.StackValidationFailed(message=message)
function.validate(val.get('Value'))
except AttributeError:
message = _('Output validation error: '
'Outputs must contain Output. '
'Found a [%s] instead') % type(val)
raise exception.StackValidationFailed(message=message)
except Exception as ex:
reason = _('Output validation error: '
'%s') % six.text_type(ex)
raise exception.StackValidationFailed(message=reason)
def __unicode__(self):
if self.stack.id:
if self.resource_id:
text = '%s "%s" [%s] %s' % (self.__class__.__name__, self.name,
self.resource_id,
unicode(self.stack))
else:
text = '%s "%s" %s' % (self.__class__.__name__, self.name,
unicode(self.stack))
else:
text = '%s "%s"' % (self.__class__.__name__, self.name)
return encodeutils.safe_decode(text)
def __unicode__(self):
if self.stack.id:
if self.resource_id:
text = '%s "%s" [%s] %s' % (self.__class__.__name__, self.name,
self.resource_id,
unicode(self.stack))
else:
text = '%s "%s" %s' % (self.__class__.__name__, self.name,
unicode(self.stack))
else:
text = '%s "%s"' % (self.__class__.__name__, self.name)
return encodeutils.safe_decode(text)
def validate(self):
'''
Validates the template.
'''
# TODO(sdake) Should return line number of invalid reference
# validate overall template (top-level structure)
self.t.validate()
# Validate parameters
self.parameters.validate(context=self.context)
# Validate Parameter Groups
parameter_groups = ParameterGroups(self.t)
parameter_groups.validate()
# Check duplicate names between parameters and resources
dup_names = set(self.parameters.keys()) & set(self.keys())
if dup_names:
LOG.debug("Duplicate names %s" % dup_names)
raise StackValidationFailed(message=_("Duplicate names %s") %
dup_names)
for res in self.dependencies:
try:
result = res.validate()
except exception.HeatException as ex:
LOG.info(ex)
raise ex
except Exception as ex:
LOG.exception(ex)
raise StackValidationFailed(message=encodeutils.safe_decode(
six.text_type(ex)))
if result:
raise StackValidationFailed(message=result)
for val in self.outputs.values():
try:
if isinstance(val, six.string_types):
message = _('"Outputs" must contain '
'a map of output maps, '
'find a string "%s".') % val
raise StackValidationFailed(message=message)
if not val or not val.get('Value', ''):
msg = _('Every Output object must '
'contain a Value member.')
raise StackValidationFailed(message=msg)
function.validate(val.get('Value', ''))
except Exception as ex:
reason = 'Output validation error: %s' % six.text_type(ex)
raise StackValidationFailed(message=reason)
def read_json(stream, len):
"""Like json.load, but converts ValueError to MalformedJSON upon failure.
:param stream: a file-like object
:param len: the number of bytes to read from stream
"""
try:
content = encodeutils.safe_decode(stream.read(len), 'utf-8')
return json.loads(content, parse_int=_json_int)
except UnicodeDecodeError as ex:
raise MalformedJSON(ex)
except ValueError as ex:
raise MalformedJSON(ex)
def read_json(stream, len):
"""Like json.load, but converts ValueError to MalformedJSON upon failure.
:param stream: a file-like object
:param len: the number of bytes to read from stream
"""
try:
content = encodeutils.safe_decode(stream.read(len), 'utf-8')
return json.loads(content, parse_int=_json_int)
except UnicodeDecodeError as ex:
raise MalformedJSON(ex)
except ValueError as ex:
raise MalformedJSON(ex)
def main():
try:
argv = [encodeutils.safe_decode(a) for a in sys.argv[1:]]
OpenStackComputeShell().main(argv)
except Exception as e:
logger.debug(e, exc_info=1)
details = {'name': encodeutils.safe_encode(e.__class__.__name__),
'msg': encodeutils.safe_encode(six.text_type(e))}
print("ERROR (%(name)s): %(msg)s" % details,
file=sys.stderr)
sys.exit(1)
except KeyboardInterrupt as e:
print("Shutting down novaclient", file=sys.stderr)
sys.exit(1)
def validate(self):
'''
Validates the template.
'''
# TODO(sdake) Should return line number of invalid reference
# validate overall template (top-level structure)
self.t.validate()
# Validate parameters
self.parameters.validate(context=self.context)
# Validate Parameter Groups
parameter_groups = ParameterGroups(self.t)
parameter_groups.validate()
# Check duplicate names between parameters and resources
dup_names = set(self.parameters.keys()) & set(self.keys())
if dup_names:
LOG.debug("Duplicate names %s" % dup_names)
raise StackValidationFailed(message=_("Duplicate names %s") %
dup_names)
for res in self.dependencies:
try:
result = res.validate()
except exception.HeatException as ex:
LOG.info(ex)
raise ex
except Exception as ex:
LOG.exception(ex)
raise StackValidationFailed(
message=encodeutils.safe_decode(six.text_type(ex)))
if result:
raise StackValidationFailed(message=result)
for val in self.outputs.values():
snippet = val.get('Value', '')
try:
function.validate(snippet)
except Exception as ex:
reason = 'Output validation error: %s' % six.text_type(ex)
raise StackValidationFailed(message=reason)
def paginate(qp, seen=0, return_request_id=None):
def filter_owner(owner, image):
# If client side owner 'filter' is specified
# only return images that match 'owner'.
if owner is None:
# Do not filter based on owner
return False
if (not hasattr(image, 'owner')) or image.owner is None:
# ownerless image
return not (owner == '')
else:
return not (image.owner == owner)
owner = qp.pop('owner', None)
for param, value in six.iteritems(qp):
if isinstance(value, six.string_types):
# Note(flaper87) Url encoding should
# be moved inside http utils, at least
# shouldn't be here.
#
# Making sure all params are str before
# trying to encode them
qp[param] = encodeutils.safe_decode(value)
url = '/v1/images/detail?%s' % urlparse.urlencode(qp)
images, resp = self._list(url, "images")
if return_request_id is not None:
return_request_id.append(resp.headers.get(OS_REQ_ID_HDR, None))
for image in images:
if filter_owner(owner, image):
continue
seen += 1
if absolute_limit is not None and seen > absolute_limit:
return
yield image
page_size = qp.get('limit')
if (page_size and len(images) == page_size
and (absolute_limit is None or 0 < seen < absolute_limit)):
qp['marker'] = image.id
for image in paginate(qp, seen, return_request_id):
yield image
def validate(self):
'''
Validates the template.
'''
# TODO(sdake) Should return line number of invalid reference
# validate overall template (top-level structure)
self.t.validate()
# Validate parameters
self.parameters.validate(context=self.context)
# Validate Parameter Groups
parameter_groups = ParameterGroups(self.t)
parameter_groups.validate()
# Check duplicate names between parameters and resources
dup_names = set(self.parameters.keys()) & set(self.keys())
if dup_names:
LOG.debug("Duplicate names %s" % dup_names)
raise StackValidationFailed(message=_("Duplicate names %s") %
dup_names)
for res in self.dependencies:
try:
result = res.validate()
except exception.HeatException as ex:
LOG.info(ex)
raise ex
except Exception as ex:
LOG.exception(ex)
raise StackValidationFailed(message=encodeutils.safe_decode(
six.text_type(ex)))
if result:
raise StackValidationFailed(message=result)
for val in self.outputs.values():
snippet = val.get('Value', '')
try:
function.validate(snippet)
except Exception as ex:
reason = 'Output validation error: %s' % six.text_type(ex)
raise StackValidationFailed(message=reason)
def do_image_delete(gc, args):
"""Delete specified image(s)."""
for args_image in args.images:
image = utils.find_resource(gc.images, args_image)
try:
if args.verbose:
print('Requesting image delete for %s ...' %
encodeutils.safe_decode(args_image),
end=' ')
gc.images.delete(image)
if args.verbose:
print('[Done]')
except exc.HTTPException as e:
if args.verbose:
print('[Fail]')
print('%s: Unable to delete image %s' % (e, args_image))
def exception_to_dict(fault):
"""Converts exceptions to a dict for use in notifications."""
# TODO(johngarbutt) move to nova/exception.py to share with wrap_exception
code = 500
if hasattr(fault, "kwargs"):
code = fault.kwargs.get('code', 500)
# get the message from the exception that was thrown
# if that does not exist, use the name of the exception class itself
try:
message = fault.format_message()
# These exception handlers are broad so we don't fail to log the fault
# just because there is an unexpected error retrieving the message
except Exception:
try:
message = unicode(fault)
except Exception:
message = None
if not message:
message = fault.__class__.__name__
# NOTE(dripton) The message field in the database is limited to 255 chars.
# MySQL silently truncates overly long messages, but PostgreSQL throws an
# error if we don't truncate it.
b_message = encodeutils.safe_encode(message)[:255]
# NOTE(chaochin) UTF-8 character byte size varies from 1 to 6. If
# truncating a long byte string to 255, the last character may be
# cut in the middle, so that UnicodeDecodeError will occur when
# converting it back to unicode.
decode_ok = False
while not decode_ok:
try:
u_message = encodeutils.safe_decode(b_message)
decode_ok = True
except UnicodeDecodeError:
b_message = b_message[:-1]
fault_dict = dict(exception=fault)
fault_dict["message"] = u_message
fault_dict["code"] = code
return fault_dict
def exception_to_dict(fault):
"""Converts exceptions to a dict for use in notifications."""
# TODO(johngarbutt) move to nova/exception.py to share with wrap_exception
code = 500
if hasattr(fault, "kwargs"):
code = fault.kwargs.get('code', 500)
# get the message from the exception that was thrown
# if that does not exist, use the name of the exception class itself
try:
message = fault.format_message()
# These exception handlers are broad so we don't fail to log the fault
# just because there is an unexpected error retrieving the message
except Exception:
try:
message = unicode(fault)
except Exception:
message = None
if not message:
message = fault.__class__.__name__
# NOTE(dripton) The message field in the database is limited to 255 chars.
# MySQL silently truncates overly long messages, but PostgreSQL throws an
# error if we don't truncate it.
b_message = encodeutils.safe_encode(message)[:255]
# NOTE(chaochin) UTF-8 character byte size varies from 1 to 6. If
# truncating a long byte string to 255, the last character may be
# cut in the middle, so that UnicodeDecodeError will occur when
# converting it back to unicode.
decode_ok = False
while not decode_ok:
try:
u_message = encodeutils.safe_decode(b_message)
decode_ok = True
except UnicodeDecodeError:
b_message = b_message[:-1]
fault_dict = dict(exception=fault)
fault_dict["message"] = u_message
fault_dict["code"] = code
return fault_dict
def _build_message(self, message, **kwargs):
"""Builds and returns an exception message.
:raises: KeyError given insufficient kwargs
"""
if not message:
try:
message = self.message_format % kwargs
except UnicodeDecodeError:
try:
kwargs = dict([(k, encodeutils.safe_decode(v)) for k, v in
six.iteritems(kwargs)])
except UnicodeDecodeError:
# NOTE(jamielennox): This is the complete failure case
# at least by showing the template we have some idea
# of where the error is coming from
message = self.message_format
else:
message = self.message_format % kwargs
return message
def _build_message(self, message, **kwargs):
"""Builds and returns an exception message.
:raises: KeyError given insufficient kwargs
"""
if not message:
try:
message = self.message_format % kwargs
except UnicodeDecodeError:
try:
kwargs = dict([(k, encodeutils.safe_decode(v))
for k, v in six.iteritems(kwargs)])
except UnicodeDecodeError:
# NOTE(jamielennox): This is the complete failure case
# at least by showing the template we have some idea
# of where the error is coming from
message = self.message_format
else:
message = self.message_format % kwargs
return message
def to_slug(value, incoming=None, errors="strict"):
"""Normalize string.
Convert to lowercase, remove non-word characters, and convert spaces
to hyphens.
Inspired by Django's `slugify` filter.
:param value: Text to slugify
:param incoming: Text's current encoding
:param errors: Errors handling policy. See here for valid
values http://docs.python.org/2/library/codecs.html
:returns: slugified unicode representation of `value`
:raises TypeError: If text is not an instance of str
"""
value = encodeutils.safe_decode(value, incoming, errors)
# NOTE(aababilov): no need to use safe_(encode|decode) here:
# encodings are always "ascii", error handling is always "ignore"
# and types are always known (first: unicode; second: str)
value = unicodedata.normalize("NFKD", value).encode(
"ascii", "ignore").decode("ascii")
value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
return SLUGIFY_HYPHENATE_RE.sub("-", value)
def print_list(objs, fields, formatters=None, field_settings=None):
formatters = formatters or {}
field_settings = field_settings or {}
pt = prettytable.PrettyTable([f for f in fields], caching=False)
pt.align = 'l'
for o in objs:
row = []
for field in fields:
if field in field_settings:
for setting, value in six.iteritems(field_settings[field]):
setting_dict = getattr(pt, setting)
setting_dict[field] = value
if field in formatters:
row.append(formatters[field](o))
else:
field_name = field.lower().replace(' ', '_')
data = getattr(o, field_name, None) or ''
row.append(data)
pt.add_row(row)
print(encodeutils.safe_decode(pt.get_string()))
def to_slug(value, incoming=None, errors="strict"):
"""Normalize string.
Convert to lowercase, remove non-word characters, and convert spaces
to hyphens.
Inspired by Django's `slugify` filter.
:param value: Text to slugify
:param incoming: Text's current encoding
:param errors: Errors handling policy. See here for valid
values http://docs.python.org/2/library/codecs.html
:returns: slugified unicode representation of `value`
:raises TypeError: If text is not an instance of str
"""
value = encodeutils.safe_decode(value, incoming, errors)
# NOTE(aababilov): no need to use safe_(encode|decode) here:
# encodings are always "ascii", error handling is always "ignore"
# and types are always known (first: unicode; second: str)
value = unicodedata.normalize("NFKD",
value).encode("ascii",
"ignore").decode("ascii")
value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
return SLUGIFY_HYPHENATE_RE.sub("-", value)
def __str__(self):
if self.hidden():
return super(JsonParam, self).__str__()
return encodeutils.safe_decode(json.dumps(self.value()))
def find_resource(manager, name_or_id, **find_args):
"""Look for resource in a given manager.
Used as a helper for the _find_* methods.
Example:
.. code-block:: python
def _find_hypervisor(cs, hypervisor):
#Get a hypervisor by name or ID.
return cliutils.find_resource(cs.hypervisors, hypervisor)
"""
# first try to get entity as integer id
try:
return manager.get(int(name_or_id))
except (TypeError, ValueError, exceptions.NotFound):
pass
# now try to get entity as uuid
try:
if six.PY2:
tmp_id = encodeutils.safe_encode(name_or_id)
else:
tmp_id = encodeutils.safe_decode(name_or_id)
if uuidutils.is_uuid_like(tmp_id):
return manager.get(tmp_id)
except (TypeError, ValueError, exceptions.NotFound):
pass
# for str id which is not uuid
if getattr(manager, 'is_alphanum_id_allowed', False):
try:
return manager.get(name_or_id)
except exceptions.NotFound:
pass
try:
try:
return manager.find(human_id=name_or_id, **find_args)
except exceptions.NotFound:
pass
# finally try to find entity by name
try:
resource = getattr(manager, 'resource_class', None)
name_attr = resource.NAME_ATTR if resource else 'name'
kwargs = {name_attr: name_or_id}
kwargs.update(find_args)
return manager.find(**kwargs)
except exceptions.NotFound:
msg = _("No %(name)s with a name or "
"ID of '%(name_or_id)s' exists.") % \
{
"name": manager.resource_class.__name__.lower(),
"name_or_id": name_or_id
}
raise exceptions.CommandError(msg)
except exceptions.NoUniqueMatch:
msg = _("Multiple %(name)s matches found for "
"'%(name_or_id)s', use an ID to be more specific.") % \
{
"name": manager.resource_class.__name__.lower(),
"name_or_id": name_or_id
}
raise exceptions.CommandError(msg)
def gc(self):
"""Garbage-collect expired message data.
Not all message data can be automatically expired. This method
cleans up the remainder.
:returns: Number of messages removed
"""
client = self._client
num_removed = 0
offset_msgsets = 0
while True:
# NOTE(kgriffs): Iterate across all message sets; there will
# be one set of message IDs per queue.
msgset_keys = client.zrange(MSGSET_INDEX_KEY, offset_msgsets,
offset_msgsets + GC_BATCH_SIZE - 1)
if not msgset_keys:
break
offset_msgsets += len(msgset_keys)
for msgset_key in msgset_keys:
msgset_key = encodeutils.safe_decode(msgset_key)
# NOTE(kgriffs): Drive the claim controller GC from
# here, because we already know the queue and project
# scope.
queue, project = utils.descope_message_ids_set(msgset_key)
self._claim_ctrl._gc(queue, project)
offset_mids = 0
while True:
# NOTE(kgriffs): Look up each message in the message set,
# see if it has expired, and if so, remove it from msgset.
mids = client.zrange(msgset_key, offset_mids,
offset_mids + GC_BATCH_SIZE - 1)
if not mids:
break
offset_mids += len(mids)
# NOTE(kgriffs): If redis expired the message, it will
# not exist, so all we have to do is remove mid from
# the msgset collection.
with client.pipeline() as pipe:
for mid in mids:
pipe.exists(mid)
mid_exists_flags = pipe.execute()
with client.pipeline() as pipe:
for mid, exists in zip(mids, mid_exists_flags):
if not exists:
pipe.zrem(msgset_key, mid)
num_removed += 1
pipe.execute()
return num_removed
def find_resource(manager, name_or_id, **find_args):
"""Look for resource in a given manager.
Used as a helper for the _find_* methods.
Example:
.. code-block:: python
def _find_hypervisor(cs, hypervisor):
#Get a hypervisor by name or ID.
return cliutils.find_resource(cs.hypervisors, hypervisor)
"""
# first try to get entity as integer id
try:
return manager.get(int(name_or_id))
except (TypeError, ValueError, exceptions.NotFound):
pass
# now try to get entity as uuid
try:
if six.PY2:
tmp_id = encodeutils.safe_encode(name_or_id)
else:
tmp_id = encodeutils.safe_decode(name_or_id)
if uuidutils.is_uuid_like(tmp_id):
return manager.get(tmp_id)
except (TypeError, ValueError, exceptions.NotFound):
pass
# for str id which is not uuid
if getattr(manager, 'is_alphanum_id_allowed', False):
try:
return manager.get(name_or_id)
except exceptions.NotFound:
pass
try:
try:
return manager.find(human_id=name_or_id, **find_args)
except exceptions.NotFound:
pass
# finally try to find entity by name
try:
resource = getattr(manager, 'resource_class', None)
name_attr = resource.NAME_ATTR if resource else 'name'
kwargs = {name_attr: name_or_id}
kwargs.update(find_args)
return manager.find(**kwargs)
except exceptions.NotFound:
msg = _("No %(name)s with a name or "
"ID of '%(name_or_id)s' exists.") % \
{
"name": manager.resource_class.__name__.lower(),
"name_or_id": name_or_id
}
raise exceptions.CommandError(msg)
except exceptions.NoUniqueMatch:
msg = _("Multiple %(name)s matches found for "
"'%(name_or_id)s', use an ID to be more specific.") % \
{
"name": manager.resource_class.__name__.lower(),
"name_or_id": name_or_id
}
raise exceptions.CommandError(msg)
def __unicode__(self):
'''Return a human-readable string representation of the node.'''
text = '{%s}' % ', '.join(unicode(n) for n in self)
return encodeutils.safe_decode(text)
def __unicode__(self):
'''Convert the graph to a human-readable string.'''
pairs = ('%s: %s' % (unicode(k), unicode(v))
for k, v in six.iteritems(self))
text = '{%s}' % ', '.join(pairs)
return encodeutils.safe_decode(text)
def gc(self):
"""Garbage-collect expired message data.
Not all message data can be automatically expired. This method
cleans up the remainder.
:returns: Number of messages removed
"""
client = self._client
num_removed = 0
offset_msgsets = 0
while True:
# NOTE(kgriffs): Iterate across all message sets; there will
# be one set of message IDs per queue.
msgset_keys = client.zrange(MSGSET_INDEX_KEY,
offset_msgsets,
offset_msgsets + GC_BATCH_SIZE - 1)
if not msgset_keys:
break
offset_msgsets += len(msgset_keys)
for msgset_key in msgset_keys:
msgset_key = encodeutils.safe_decode(msgset_key)
# NOTE(kgriffs): Drive the claim controller GC from
# here, because we already know the queue and project
# scope.
queue, project = utils.descope_message_ids_set(msgset_key)
self._claim_ctrl._gc(queue, project)
offset_mids = 0
while True:
# NOTE(kgriffs): Look up each message in the message set,
# see if it has expired, and if so, remove it from msgset.
mids = client.zrange(msgset_key, offset_mids,
offset_mids + GC_BATCH_SIZE - 1)
if not mids:
break
offset_mids += len(mids)
# NOTE(kgriffs): If redis expired the message, it will
# not exist, so all we have to do is remove mid from
# the msgset collection.
with client.pipeline() as pipe:
for mid in mids:
pipe.exists(mid)
mid_exists_flags = pipe.execute()
with client.pipeline() as pipe:
for mid, exists in zip(mids, mid_exists_flags):
if not exists:
pipe.zrem(msgset_key, mid)
num_removed += 1
pipe.execute()
return num_removed
def __str__(self):
if self.hidden():
return super(JsonParam, self).__str__()
return encodeutils.safe_decode(json.dumps(self.value()))
def __unicode__(self):
'''Convert the graph to a human-readable string.'''
pairs = ('%s: %s' % (six.text_type(k), six.text_type(v))
for k, v in six.iteritems(self))
text = '{%s}' % ', '.join(pairs)
return encodeutils.safe_decode(text)
def __unicode__(self):
'''Return a human-readable string representation of the node.'''
text = '{%s}' % ', '.join(six.text_type(n) for n in self)
return encodeutils.safe_decode(text)
def __unicode__(self):
"""Return a human-readable string representation of the task."""
text = 'Task %s' % self.name
return encodeutils.safe_decode(text)
def next(self):
curr = next(self.queue_iter)
queue = self.client.hmget(curr, ['c', 'm'])
return self.denormalizer(queue, encodeutils.safe_decode(curr))
