def print_list(objs, fields, formatters={}, sortby_index=None):
if sortby_index is None:
sortby = None
else:
sortby = fields[sortby_index]
mixed_case_fields = ['serverId']
pt = prettytable.PrettyTable([f for f in fields], caching=False)
pt.align = 'l'
for o in objs:
row = []
for field in fields:
if field in formatters:
row.append(formatters[field](o))
else:
if field in mixed_case_fields:
field_name = field.replace(' ', '_')
else:
field_name = field.lower().replace(' ', '_')
data = getattr(o, field_name, '')
if data is None:
data = '-'
row.append(data)
pt.add_row(row)
if sortby is not None:
result = encodeutils.safe_encode(pt.get_string(sortby=sortby))
else:
result = encodeutils.safe_encode(pt.get_string())
if six.PY3:
result = result.decode()
print(result)
def print_dict(dct, dict_property="Property", wrap=0):
"""Print a `dict` as a table of two columns.
:param dct: `dict` to print
:param dict_property: name of the first column
:param wrap: wrapping for the second column
"""
pt = prettytable.PrettyTable([dict_property, 'Value'])
pt.align = 'l'
for k, v in six.iteritems(dct):
# convert dict to str to check length
if isinstance(v, dict):
v = six.text_type(v)
if wrap > 0:
v = textwrap.fill(six.text_type(v), wrap)
# if value has a newline, add in multiple rows
# e.g. fault with stacktrace
if v and isinstance(v, six.string_types) and r'\n' in v:
lines = v.strip().split(r'\n')
col1 = k
for line in lines:
pt.add_row([col1, line])
col1 = ''
else:
pt.add_row([k, v])
if six.PY3:
print(encodeutils.safe_encode(pt.get_string()).decode())
else:
print(encodeutils.safe_encode(pt.get_string()))
def print_dict(dct, dict_property="Property", wrap=0):
"""Print a `dict` as a table of two columns.
:param dct: `dict` to print
:param dict_property: name of the first column
:param wrap: wrapping for the second column
"""
pt = prettytable.PrettyTable([dict_property, 'Value'])
pt.align = 'l'
for k, v in six.iteritems(dct):
# convert dict to str to check length
if isinstance(v, dict):
v = six.text_type(v)
if wrap > 0:
v = textwrap.fill(six.text_type(v), wrap)
# if value has a newline, add in multiple rows
# e.g. fault with stacktrace
if v and isinstance(v, six.string_types) and r'\n' in v:
lines = v.strip().split(r'\n')
col1 = k
for line in lines:
pt.add_row([col1, line])
col1 = ''
else:
pt.add_row([k, v])
if six.PY3:
print(encodeutils.safe_encode(pt.get_string()).decode())
else:
print(encodeutils.safe_encode(pt.get_string()))
def _check_restore_vol_size(self, backup_base, restore_vol, restore_length,
src_pool):
"""Ensure that the restore volume is the correct size.
If the restore volume was bigger than the backup, the diff restore will
shrink it to the size of the original backup so we need to
post-process and resize it back to its expected size.
"""
with rbd_driver.RADOSClient(self, self._ceph_backup_pool) as client:
adjust_size = 0
base_image = self.rbd.Image(client.ioctx,
encodeutils.safe_encode(backup_base),
read_only=True)
try:
if restore_length != base_image.size():
adjust_size = restore_length
finally:
base_image.close()
if adjust_size:
with rbd_driver.RADOSClient(self, src_pool) as client:
restore_vol_encode = encodeutils.safe_encode(restore_vol)
dest_image = self.rbd.Image(client.ioctx, restore_vol_encode)
try:
LOG.debug("Adjusting restore vol size")
dest_image.resize(adjust_size)
finally:
dest_image.close()
def test_do_request_unicode(self):
self.client.format = self.format
self.mox.StubOutWithMock(self.client.httpclient, "request")
unicode_text = u'\u7f51\u7edc'
# url with unicode
action = u'/test'
expected_action = action
# query string with unicode
params = {'test': unicode_text}
expect_query = urlparse.urlencode(utils.safe_encode_dict(params))
# request body with unicode
body = params
expect_body = self.client.serialize(body)
self.client.httpclient.auth_token = encodeutils.safe_encode(
unicode_text)
expected_auth_token = encodeutils.safe_encode(unicode_text)
self.client.httpclient.request(
end_url(expected_action, query=expect_query, format=self.format),
'PUT', body=expect_body,
headers=mox.ContainsKeyValue(
'X-Auth-Token',
expected_auth_token)).AndReturn((MyResp(200), expect_body))
self.mox.ReplayAll()
res_body = self.client.do_request('PUT', action, body=body,
params=params)
self.mox.VerifyAll()
self.mox.UnsetStubs()
# test response with unicode
self.assertEqual(res_body, body)
def print_list(objs, fields, formatters={}, sortby_index=None):
if sortby_index is None:
sortby = None
else:
sortby = fields[sortby_index]
mixed_case_fields = ['serverId']
pt = prettytable.PrettyTable([f for f in fields], caching=False)
pt.align = 'l'
for o in objs:
row = []
for field in fields:
if field in formatters:
row.append(formatters[field](o))
else:
if field in mixed_case_fields:
field_name = field.replace(' ', '_')
else:
field_name = field.lower().replace(' ', '_')
data = getattr(o, field_name, '')
if data is None:
data = '-'
row.append(data)
pt.add_row(row)
if sortby is not None:
result = encodeutils.safe_encode(pt.get_string(sortby=sortby))
else:
result = encodeutils.safe_encode(pt.get_string())
if six.PY3:
result = result.decode()
print(result)
def test_do_request_unicode(self):
self.client.format = self.format
self.mox.StubOutWithMock(self.client.httpclient, "request")
unicode_text = u'\u7f51\u7edc'
# url with unicode
action = u'/test'
expected_action = action
# query string with unicode
params = {'test': unicode_text}
expect_query = urlparse.urlencode(utils.safe_encode_dict(params))
# request body with unicode
body = params
expect_body = self.client.serialize(body)
self.client.httpclient.auth_token = encodeutils.safe_encode(
unicode_text)
expected_auth_token = encodeutils.safe_encode(unicode_text)
self.client.httpclient.request(
end_url(expected_action, query=expect_query, format=self.format),
'PUT', body=expect_body,
headers=mox.ContainsKeyValue(
'X-Auth-Token',
expected_auth_token)).AndReturn((MyResp(200), expect_body))
self.mox.ReplayAll()
res_body = self.client.do_request('PUT', action, body=body,
params=params)
self.mox.VerifyAll()
self.mox.UnsetStubs()
# test response with unicode
self.assertEqual(res_body, body)
def _clone(self, volume, src_pool, src_image, src_snap):
LOG.debug('cloning %(pool)s/%(img)s@%(snap)s to %(dst)s' % dict(
pool=src_pool, img=src_image, snap=src_snap, dst=volume['name']))
with RADOSClient(self, src_pool) as src_client:
with RADOSClient(self) as dest_client:
self.rbd.RBD().clone(src_client.ioctx,
encodeutils.safe_encode(src_image),
encodeutils.safe_encode(src_snap),
dest_client.ioctx,
encodeutils.safe_encode(volume['name']),
features=self.rbd.RBD_FEATURE_LAYERING)
def print_list(objs,
fields,
formatters=None,
sortby_index=0,
mixed_case_fields=None,
field_labels=None):
"""Print a list or objects as a table, one row per object.
:param objs: iterable of :class:`Resource`
:param fields: attributes that correspond to columns, in order
:param formatters: `dict` of callables for field formatting
:param sortby_index: index of the field for sorting table rows
:param mixed_case_fields: fields corresponding to object attributes that
have mixed case names (e.g., 'serverId')
:param field_labels: Labels to use in the heading of the table, default to
fields.
"""
formatters = formatters or {}
mixed_case_fields = mixed_case_fields or []
field_labels = field_labels or fields
if len(field_labels) != len(fields):
raise ValueError(
_("Field labels list %(labels)s has different number "
"of elements than fields list %(fields)s"), {
'labels': field_labels,
'fields': fields
})
if sortby_index is None:
kwargs = {}
else:
kwargs = {'sortby': field_labels[sortby_index]}
pt = prettytable.PrettyTable(field_labels)
pt.align = 'l'
for o in objs:
row = []
for field in fields:
if field in formatters:
row.append(formatters[field](o))
else:
if field in mixed_case_fields:
field_name = field.replace(' ', '_')
else:
field_name = field.lower().replace(' ', '_')
data = getattr(o, field_name, '')
row.append(data)
pt.add_row(row)
if six.PY3:
print(encodeutils.safe_encode(pt.get_string(**kwargs)).decode())
else:
print(encodeutils.safe_encode(pt.get_string(**kwargs)))
def encode_headers(headers):
"""Encodes headers.
Note: This should be used right before
sending anything out.
:param headers: Headers to encode
:returns: Dictionary with encoded headers'
names and values
"""
return dict((encodeutils.safe_encode(h), encodeutils.safe_encode(v))
for h, v in six.iteritems(headers))
def _clone(self, volume, src_pool, src_image, src_snap):
LOG.debug('cloning %(pool)s/%(img)s@%(snap)s to %(dst)s' %
dict(pool=src_pool, img=src_image, snap=src_snap,
dst=volume['name']))
with RADOSClient(self, src_pool) as src_client:
with RADOSClient(self) as dest_client:
self.rbd.RBD().clone(src_client.ioctx,
encodeutils.safe_encode(src_image),
encodeutils.safe_encode(src_snap),
dest_client.ioctx,
encodeutils.safe_encode(volume['name']),
features=self.rbd.RBD_FEATURE_LAYERING)
def test_safe_encode_same_encoding_different_cases(self):
with mock.patch.object(encodeutils, 'safe_decode', mock.Mock()):
utf8 = encodeutils.safe_encode(
six.u('foo\xf1bar'), encoding='utf-8')
self.assertEqual(
encodeutils.safe_encode(utf8, 'UTF-8', 'utf-8'),
encodeutils.safe_encode(utf8, 'utf-8', 'UTF-8'),
)
self.assertEqual(
encodeutils.safe_encode(utf8, 'UTF-8', 'utf-8'),
encodeutils.safe_encode(utf8, 'utf-8', 'utf-8'),
)
encodeutils.safe_decode.assert_has_calls([])
def delete_snapshot(self, snapshot):
"""Deletes an rbd snapshot."""
# NOTE(dosaboy): this was broken by commit cbe1d5f. Ensure names are
# utf-8 otherwise librbd will barf.
volume_name = encodeutils.safe_encode(snapshot['volume_name'])
snap_name = encodeutils.safe_encode(snapshot['name'])
with RBDVolumeProxy(self, volume_name) as volume:
if self._supports_layering():
try:
volume.unprotect_snap(snap_name)
except self.rbd.ImageBusy:
raise exception.SnapshotIsBusy(snapshot_name=snap_name)
volume.remove_snap(snap_name)
def test_safe_encode_same_encoding_different_cases(self):
with mock.patch.object(encodeutils, 'safe_decode', mock.Mock()):
utf8 = encodeutils.safe_encode(six.u('foo\xf1bar'),
encoding='utf-8')
self.assertEqual(
encodeutils.safe_encode(utf8, 'UTF-8', 'utf-8'),
encodeutils.safe_encode(utf8, 'utf-8', 'UTF-8'),
)
self.assertEqual(
encodeutils.safe_encode(utf8, 'UTF-8', 'utf-8'),
encodeutils.safe_encode(utf8, 'utf-8', 'utf-8'),
)
encodeutils.safe_decode.assert_has_calls([])
def delete_snapshot(self, snapshot):
"""Deletes an rbd snapshot."""
# NOTE(dosaboy): this was broken by commit cbe1d5f. Ensure names are
# utf-8 otherwise librbd will barf.
volume_name = encodeutils.safe_encode(snapshot['volume_name'])
snap_name = encodeutils.safe_encode(snapshot['name'])
with RBDVolumeProxy(self, volume_name) as volume:
if self._supports_layering():
try:
volume.unprotect_snap(snap_name)
except self.rbd.ImageBusy:
raise exception.SnapshotIsBusy(snapshot_name=snap_name)
volume.remove_snap(snap_name)
def main():
try:
argv = [encodeutils.safe_decode(a) for a in sys.argv[1:]]
OpenStackComputeShell().main(argv)
except Exception as e:
logger.debug(e, exc_info=1)
details = {'name': encodeutils.safe_encode(e.__class__.__name__),
'msg': encodeutils.safe_encode(six.text_type(e))}
print("ERROR (%(name)s): %(msg)s" % details,
file=sys.stderr)
sys.exit(1)
except KeyboardInterrupt as e:
print("Shutting down novaclient", file=sys.stderr)
sys.exit(1)
def test_exception_to_dict_with_long_message(self):
# Generate Russian byte message whose length is 300
msg = encodeutils.safe_decode(' \xd0\xb2' * 100)
exc = exception.NovaException(message=msg)
fault_dict = compute_utils.exception_to_dict(exc)
byte_message = encodeutils.safe_encode(fault_dict["message"])
self.assertEqual(255, len(byte_message))
def _construct_url(self, action, params=None):
"""
Create a URL object we can use to pass to _do_request().
"""
action = urlparse.quote(action)
path = '/'.join([self.doc_root or '', action.lstrip('/')])
scheme = "https" if self.use_ssl else "http"
netloc = "%s:%d" % (self.host, self.port)
if isinstance(params, dict):
for (key, value) in params.items():
if value is None:
del params[key]
continue
if not isinstance(value, six.string_types):
value = str(value)
params[key] = encodeutils.safe_encode(value)
query = urlparse.urlencode(params)
else:
query = None
url = urlparse.ParseResult(scheme, netloc, path, '', query, '')
log_msg = _("Constructed URL: %s")
LOG.debug(log_msg, url.geturl())
return url
def print_list(objs, fields, formatters={}, order_by=None):
pt = prettytable.PrettyTable([f for f in fields],
caching=False,
print_empty=False)
pt.aligns = ['l' for f in fields]
for o in objs:
row = []
for field in fields:
if field in formatters:
row.append(formatters[field](o))
else:
field_name = field.lower().replace(' ', '_')
data = getattr(o, field_name, '')
if data is None:
data = ''
row.append(data)
pt.add_row(row)
if order_by is None:
order_by = fields[0]
encoded = encodeutils.safe_encode(pt.get_string(sortby=order_by))
if six.PY3:
encoded = encoded.decode()
print(encoded)
def print_dict(d, dict_property="Property", wrap=0):
pt = prettytable.PrettyTable([dict_property, 'Value'], print_empty=False)
pt.align = 'l'
for k, v in sorted(six.iteritems(d)):
# convert dict to str to check length
if isinstance(v, dict):
v = str(v)
# if value has a newline, add in multiple rows
# e.g. fault with stacktrace
if v and isinstance(v, six.string_types) and r'\n' in v:
lines = v.strip().split(r'\n')
col1 = k
for line in lines:
if wrap > 0:
line = textwrap.fill(str(line), wrap)
pt.add_row([col1, line])
col1 = ''
else:
if wrap > 0:
v = textwrap.fill(str(v), wrap)
pt.add_row([k, v])
encoded = encodeutils.safe_encode(pt.get_string())
# FIXME(gordc): https://bugs.launchpad.net/oslo-incubator/+bug/1370710
if six.PY3:
encoded = encoded.decode()
print(encoded)
def test_safe_encode_different_encodings(self):
text = six.u('foo\xc3\xb1bar')
result = encodeutils.safe_encode(text=text,
incoming='utf-8',
encoding='iso-8859-1')
self.assertNotEqual(text, result)
self.assertNotEqual(six.b("foo\xf1bar"), result)
def main():
try:
OpenStackIdentityShell().main(sys.argv[1:])
except Exception as e:
print(encodeutils.safe_encode(six.text_type(e)), file=sys.stderr)
sys.exit(1)
def create_snapshot(self, snapshot):
"""Creates an rbd snapshot."""
with RBDVolumeProxy(self, snapshot['volume_name']) as volume:
snap = encodeutils.safe_encode(snapshot['name'])
volume.create_snap(snap)
if self._supports_layering():
volume.protect_snap(snap)
def create_snapshot(self, snapshot):
"""Creates an rbd snapshot."""
with RBDVolumeProxy(self, snapshot['volume_name']) as volume:
snap = encodeutils.safe_encode(snapshot['name'])
volume.create_snap(snap)
if self._supports_layering():
volume.protect_snap(snap)
def exception_to_dict(fault):
"""Converts exceptions to a dict for use in notifications."""
# TODO(johngarbutt) move to nova/exception.py to share with wrap_exception
code = 500
if hasattr(fault, "kwargs"):
code = fault.kwargs.get('code', 500)
# get the message from the exception that was thrown
# if that does not exist, use the name of the exception class itself
try:
message = fault.format_message()
# These exception handlers are broad so we don't fail to log the fault
# just because there is an unexpected error retrieving the message
except Exception:
try:
message = unicode(fault)
except Exception:
message = None
if not message:
message = fault.__class__.__name__
# NOTE(dripton) The message field in the database is limited to 255 chars.
# MySQL silently truncates overly long messages, but PostgreSQL throws an
# error if we don't truncate it.
u_message = encodeutils.safe_decode(encodeutils.safe_encode(message)[:255])
fault_dict = dict(exception=fault)
fault_dict["message"] = u_message
fault_dict["code"] = code
return fault_dict
def _construct_url(self, action, params=None):
"""
Create a URL object we can use to pass to _do_request().
"""
action = urlparse.quote(action)
path = '/'.join([self.doc_root or '', action.lstrip('/')])
scheme = "https" if self.use_ssl else "http"
netloc = "%s:%d" % (self.host, self.port)
if isinstance(params, dict):
for (key, value) in params.items():
if value is None:
del params[key]
continue
if not isinstance(value, six.string_types):
value = str(value)
params[key] = encodeutils.safe_encode(value)
query = urlparse.urlencode(params)
else:
query = None
url = urlparse.ParseResult(scheme, netloc, path, '', query, '')
log_msg = _("Constructed URL: %s")
LOG.debug(log_msg, url.geturl())
return url
def _get_backup_base_name(self, volume_id, backup_id=None,
diff_format=False):
"""Return name of base image used for backup.
Incremental backups use a new base name so we support old and new style
format.
"""
# Ensure no unicode
if diff_format:
return encodeutils.safe_encode("volume-%s.backup.base" % volume_id)
else:
if backup_id is None:
msg = _("Backup id required")
raise exception.InvalidParameterValue(msg)
return encodeutils.safe_encode("volume-%s.backup.%s" %
(volume_id, backup_id))
def __str__(self):
'''Return a string representation of the parameter'''
value = self.value()
if self.hidden():
return '******'
else:
return encodeutils.safe_encode(six.text_type(value))
def print_dict(d, dict_property="Property", dict_value="Value", wrap=0):
pt = prettytable.PrettyTable([dict_property, dict_value], caching=False)
pt.align = 'l'
for k, v in sorted(d.items()):
# convert dict to str to check length
if isinstance(v, (dict, list)):
v = jsonutils.dumps(v)
if wrap > 0:
v = textwrap.fill(str(v), wrap)
# if value has a newline, add in multiple rows
# e.g. fault with stacktrace
if v and isinstance(v, six.string_types) and r'\n' in v:
lines = v.strip().split(r'\n')
col1 = k
for line in lines:
pt.add_row([col1, line])
col1 = ''
else:
if v is None:
v = '-'
pt.add_row([k, v])
result = encodeutils.safe_encode(pt.get_string())
if six.PY3:
result = result.decode()
print(result)
def print_dict(d, dict_property="Property", wrap=0):
pt = prettytable.PrettyTable([dict_property, 'Value'], print_empty=False)
pt.align = 'l'
for k, v in sorted(six.iteritems(d)):
# convert dict to str to check length
if isinstance(v, dict):
v = jsonutils.dumps(v)
# if value has a newline, add in multiple rows
# e.g. fault with stacktrace
if v and isinstance(v, six.string_types) and r'\n' in v:
lines = v.strip().split(r'\n')
col1 = k
for line in lines:
if wrap > 0:
line = textwrap.fill(six.text_type(line), wrap)
pt.add_row([col1, line])
col1 = ''
else:
if wrap > 0:
v = textwrap.fill(six.text_type(v), wrap)
pt.add_row([k, v])
encoded = encodeutils.safe_encode(pt.get_string())
# FIXME(gordc): https://bugs.launchpad.net/oslo-incubator/+bug/1370710
if six.PY3:
encoded = encoded.decode()
print(encoded)
def exception_to_dict(fault):
"""Converts exceptions to a dict for use in notifications."""
# TODO(johngarbutt) move to nova/exception.py to share with wrap_exception
code = 500
if hasattr(fault, "kwargs"):
code = fault.kwargs.get('code', 500)
# get the message from the exception that was thrown
# if that does not exist, use the name of the exception class itself
try:
message = fault.format_message()
# These exception handlers are broad so we don't fail to log the fault
# just because there is an unexpected error retrieving the message
except Exception:
try:
message = unicode(fault)
except Exception:
message = None
if not message:
message = fault.__class__.__name__
# NOTE(dripton) The message field in the database is limited to 255 chars.
# MySQL silently truncates overly long messages, but PostgreSQL throws an
# error if we don't truncate it.
u_message = encodeutils.safe_decode(encodeutils.safe_encode(message)[:255])
fault_dict = dict(exception=fault)
fault_dict["message"] = u_message
fault_dict["code"] = code
return fault_dict
def main():
try:
DragonShell().main(sys.argv[1:])
except Exception as e:
print(encodeutils.safe_encode(six.text_type(e)), file=sys.stderr)
sys.exit(1)
def print_dict(d, dict_property="Property", dict_value="Value", wrap=0):
pt = prettytable.PrettyTable([dict_property, dict_value], caching=False)
pt.align = 'l'
for k, v in sorted(d.items()):
# convert dict to str to check length
if isinstance(v, (dict, list)):
v = jsonutils.dumps(v)
if wrap > 0:
v = textwrap.fill(str(v), wrap)
# if value has a newline, add in multiple rows
# e.g. fault with stacktrace
if v and isinstance(v, six.string_types) and r'\n' in v:
lines = v.strip().split(r'\n')
col1 = k
for line in lines:
pt.add_row([col1, line])
col1 = ''
else:
if v is None:
v = '-'
pt.add_row([k, v])
result = encodeutils.safe_encode(pt.get_string())
if six.PY3:
result = result.decode()
print(result)
def test_exception_to_dict_with_long_message(self):
# Generate Russian byte message whose length is 300
msg = encodeutils.safe_decode(' \xd0\xb2' * 100)
exc = exception.NovaException(message=msg)
fault_dict = compute_utils.exception_to_dict(exc)
byte_message = encodeutils.safe_encode(fault_dict["message"])
self.assertEqual(255, len(byte_message))
def __str__(self):
'''Return a string representation of the parameter'''
value = self.value()
if self.hidden():
return '******'
else:
return encodeutils.safe_encode(six.text_type(value))
def print_list(objs, fields, formatters=None, sortby_index=0,
mixed_case_fields=None, field_labels=None):
"""Print a list or objects as a table, one row per object.
:param objs: iterable of :class:`Resource`
:param fields: attributes that correspond to columns, in order
:param formatters: `dict` of callables for field formatting
:param sortby_index: index of the field for sorting table rows
:param mixed_case_fields: fields corresponding to object attributes that
have mixed case names (e.g., 'serverId')
:param field_labels: Labels to use in the heading of the table, default to
fields.
"""
formatters = formatters or {}
mixed_case_fields = mixed_case_fields or []
field_labels = field_labels or fields
if len(field_labels) != len(fields):
raise ValueError(_("Field labels list %(labels)s has different number "
"of elements than fields list %(fields)s"),
{'labels': field_labels, 'fields': fields})
if sortby_index is None:
kwargs = {}
else:
kwargs = {'sortby': field_labels[sortby_index]}
pt = prettytable.PrettyTable(field_labels)
pt.align = 'l'
for o in objs:
row = []
for field in fields:
if field in formatters:
row.append(formatters[field](o))
else:
if field in mixed_case_fields:
field_name = field.replace(' ', '_')
else:
field_name = field.lower().replace(' ', '_')
data = getattr(o, field_name, '')
row.append(data)
pt.add_row(row)
if six.PY3:
print(encodeutils.safe_encode(pt.get_string(**kwargs)).decode())
else:
print(encodeutils.safe_encode(pt.get_string(**kwargs)))
def test_exception_to_dict_with_long_message_2_bytes(self):
# Generate Russian byte string whose length is 300. This Russian UTF-8
# character occupies 2 bytes. After truncating, the byte string length
# should be 254.
msg = encodeutils.safe_decode('\xd0\x92' * 150)
exc = exception.NovaException(message=msg)
fault_dict = compute_utils.exception_to_dict(exc)
byte_message = encodeutils.safe_encode(fault_dict["message"])
self.assertEqual(254, len(byte_message))
def test_exception_to_dict_with_long_message_3_bytes(self):
# Generate Chinese byte string whose length is 300. This Chinese UTF-8
# character occupies 3 bytes. After truncating, the byte string length
# should be 255.
msg = encodeutils.safe_decode('\xe8\xb5\xb5' * 100)
exc = exception.NovaException(message=msg)
fault_dict = compute_utils.exception_to_dict(exc)
byte_message = encodeutils.safe_encode(fault_dict["message"])
self.assertEqual(255, len(byte_message))
def manage_existing(self, volume, existing_ref):
"""Manages an existing image.
Renames the image name to match the expected name for the volume.
Error checking done by manage_existing_get_size is not repeated.
:param volume:
volume ref info to be set
:param existing_ref:
existing_ref is a dictionary of the form:
{'source-name': <name of rbd image>}
"""
# Raise an exception if we didn't find a suitable rbd image.
with RADOSClient(self) as client:
rbd_name = existing_ref['source-name']
self.rbd.RBD().rename(client.ioctx,
encodeutils.safe_encode(rbd_name),
encodeutils.safe_encode(volume['name']))
def test_exception_to_dict_with_long_message_2_bytes(self):
# Generate Russian byte string whose length is 300. This Russian UTF-8
# character occupies 2 bytes. After truncating, the byte string length
# should be 254.
msg = encodeutils.safe_decode('\xd0\x92' * 150)
exc = exception.NovaException(message=msg)
fault_dict = compute_utils.exception_to_dict(exc)
byte_message = encodeutils.safe_encode(fault_dict["message"])
self.assertEqual(254, len(byte_message))
def _tenant_path(self):
"""URL-encoded path segment of a URL within a particular tenant.
Returned in the form:
stacks/<stack_name>/<stack_id><path>
"""
return 'stacks/%s%s' % (self.stack_path(),
urlparse.quote(
encodeutils.safe_encode(self.path)))
def _tenant_path(self):
"""URL-encoded path segment of a URL within a particular tenant.
Returned in the form:
stacks/<stack_name>/<stack_id><path>
"""
return 'stacks/%s%s' % (self.stack_path(),
urlparse.quote(encodeutils.safe_encode(
self.path)))
def main():
try:
OpenStackIdentityShell().main(sys.argv[1:])
except KeyboardInterrupt:
print("... terminating keystone client", file=sys.stderr)
sys.exit(130)
except Exception as e:
print(encodeutils.safe_encode(six.text_type(e)), file=sys.stderr)
sys.exit(1)
def test_exception_to_dict_with_long_message_3_bytes(self):
# Generate Chinese byte string whose length is 300. This Chinese UTF-8
# character occupies 3 bytes. After truncating, the byte string length
# should be 255.
msg = encodeutils.safe_decode('\xe8\xb5\xb5' * 100)
exc = exception.NovaException(message=msg)
fault_dict = compute_utils.exception_to_dict(exc)
byte_message = encodeutils.safe_encode(fault_dict["message"])
self.assertEqual(255, len(byte_message))
def main():
try:
OpenStackMagnumShell().main(map(encodeutils.safe_decode, sys.argv[1:]))
except Exception as e:
logger.debug(e, exc_info=1)
print("ERROR: %s" % encodeutils.safe_encode(six.text_type(e)),
file=sys.stderr)
sys.exit(1)
def __init__(self, driver, name, pool=None, snapshot=None,
read_only=False):
client, ioctx = driver._connect_to_rados(pool)
if snapshot is not None:
snapshot = encodeutils.safe_encode(snapshot)
try:
self.volume = driver.rbd.Image(ioctx,
encodeutils.safe_encode(name),
snapshot=snapshot,
read_only=read_only)
except driver.rbd.Error:
LOG.exception(_LE("error opening rbd image %s"), name)
driver._disconnect_from_rados(client, ioctx)
raise
self.driver = driver
self.client = client
self.ioctx = ioctx
def manage_existing(self, volume, existing_ref):
"""Manages an existing image.
Renames the image name to match the expected name for the volume.
Error checking done by manage_existing_get_size is not repeated.
:param volume:
volume ref info to be set
:param existing_ref:
existing_ref is a dictionary of the form:
{'source-name': <name of rbd image>}
"""
# Raise an exception if we didn't find a suitable rbd image.
with RADOSClient(self) as client:
rbd_name = existing_ref['source-name']
self.rbd.RBD().rename(client.ioctx,
encodeutils.safe_encode(rbd_name),
encodeutils.safe_encode(volume['name']))
def exception_to_str(exc):
try:
error = six.text_type(exc)
except UnicodeError:
try:
error = str(exc)
except UnicodeError:
error = ("Caught '%(exception)s' exception." %
{"exception": exc.__class__.__name__})
return encodeutils.safe_encode(error, errors='ignore')
def __init__(self, context, db_driver=None, execute=None):
super(CephBackupDriver, self).__init__(context, db_driver)
self.rbd = rbd
self.rados = rados
self.chunk_size = CONF.backup_ceph_chunk_size
self._execute = execute or utils.execute
if self._supports_stripingv2:
self.rbd_stripe_unit = CONF.backup_ceph_stripe_unit
self.rbd_stripe_count = CONF.backup_ceph_stripe_count
else:
LOG.info(_LI("RBD striping not supported - ignoring configuration "
"settings for rbd striping"))
self.rbd_stripe_count = 0
self.rbd_stripe_unit = 0
self._ceph_backup_user = encodeutils.safe_encode(CONF.backup_ceph_user)
self._ceph_backup_pool = encodeutils.safe_encode(CONF.backup_ceph_pool)
self._ceph_backup_conf = encodeutils.safe_encode(CONF.backup_ceph_conf)
def test_safe_encode_py2(self):
if six.PY2:
# In Python 3, str.encode() doesn't support anymore
# text => text encodings like base64
self.assertEqual(
six.b("dGVzdA==\n"),
encodeutils.safe_encode("test", encoding='base64'),
)
else:
self.skipTest("Requires py2.x")
def test_safe_encode_py2(self):
if six.PY2:
# In Python 3, str.encode() doesn't support anymore
# text => text encodings like base64
self.assertEqual(
six.b("dGVzdA==\n"),
encodeutils.safe_encode("test", encoding='base64'),
)
else:
self.skipTest("Requires py2.x")
def exception_to_str(exc):
try:
error = six.text_type(exc)
except UnicodeError:
try:
error = str(exc)
except UnicodeError:
error = ("Caught '%(exception)s' exception." % {
"exception": exc.__class__.__name__
})
return encodeutils.safe_encode(error, errors='ignore')
def __str__(self):
if self.stack.id:
if self.resource_id:
text = '%s "%s" [%s] %s' % (self.__class__.__name__, self.name,
self.resource_id, str(self.stack))
else:
text = '%s "%s" %s' % (self.__class__.__name__, self.name,
str(self.stack))
else:
text = '%s "%s"' % (self.__class__.__name__, self.name)
return encodeutils.safe_encode(text)
def list(self, **kwargs):
"""Retrieve a listing of Task objects
:param page_size: Number of tasks to request in each paginated request
:returns generator over list of Tasks
"""
def paginate(url):
resp, body = self.http_client.get(url)
for task in body['tasks']:
yield task
try:
next_url = body['next']
except KeyError:
return
else:
for task in paginate(next_url):
yield task
filters = kwargs.get('filters', {})
if not kwargs.get('page_size'):
filters['limit'] = DEFAULT_PAGE_SIZE
else:
filters['limit'] = kwargs['page_size']
if 'marker' in kwargs:
filters['marker'] = kwargs['marker']
sort_key = kwargs.get('sort_key')
if sort_key is not None:
if sort_key in SORT_KEY_VALUES:
filters['sort_key'] = sort_key
else:
raise ValueError('sort_key must be one of the following: %s.'
% ', '.join(SORT_KEY_VALUES))
sort_dir = kwargs.get('sort_dir')
if sort_dir is not None:
if sort_dir in SORT_DIR_VALUES:
filters['sort_dir'] = sort_dir
else:
raise ValueError('sort_dir must be one of the following: %s.'
% ', '.join(SORT_DIR_VALUES))
for param, value in filters.items():
if isinstance(value, six.string_types):
filters[param] = encodeutils.safe_encode(value)
url = '/v2/tasks?%s' % six.moves.urllib.parse.urlencode(filters)
for task in paginate(url):
#NOTE(flwang): remove 'self' for now until we have an elegant
# way to pass it into the model constructor without conflict
task.pop('self', None)
yield self.model(**task)
def find_resource(manager, name_or_id, **find_args):
"""Helper for the _find_* methods."""
# for str id which is not uuid (for Flavor and Keypair search currently)
if getattr(manager, 'is_alphanum_id_allowed', False):
try:
return manager.get(name_or_id)
except exceptions.NotFound:
pass
# try to get entity as integer id
try:
return manager.get(int(name_or_id))
except (TypeError, ValueError, exceptions.NotFound):
pass
# now try to get entity as uuid
try:
tmp_id = encodeutils.safe_encode(name_or_id)
if six.PY3:
tmp_id = tmp_id.decode()
uuid.UUID(tmp_id)
return manager.get(tmp_id)
except (TypeError, ValueError, exceptions.NotFound):
pass
try:
try:
resource = getattr(manager, 'resource_class', None)
name_attr = resource.NAME_ATTR if resource else 'name'
kwargs = {name_attr: name_or_id}
kwargs.update(find_args)
return manager.find(**kwargs)
except exceptions.NotFound:
pass
# finally try to find entity by human_id
try:
return manager.find(human_id=name_or_id, **find_args)
except exceptions.NotFound:
msg = (_("No %(class)s with a name or ID of '%(name)s' exists.") %
{
'class': manager.resource_class.__name__.lower(),
'name': name_or_id
})
raise exceptions.CommandError(msg)
except exceptions.NoUniqueMatch:
msg = (_("Multiple %(class)s matches found for '%(name)s', use an ID "
"to be more specific.") % {
'class': manager.resource_class.__name__.lower(),
'name': name_or_id
})
raise exceptions.CommandError(msg)
def __str__(self):
if self.stack.id:
if self.resource_id:
text = '%s "%s" [%s] %s' % (self.__class__.__name__, self.name,
self.resource_id, str(self.stack))
else:
text = '%s "%s" %s' % (self.__class__.__name__, self.name,
str(self.stack))
else:
text = '%s "%s"' % (self.__class__.__name__, self.name)
return encodeutils.safe_encode(text)
def _rbd_diff_transfer(self, src_name, src_pool, dest_name, dest_pool,
src_user, src_conf, dest_user, dest_conf,
src_snap=None, from_snap=None):
"""Copy only extents changed between two points.
If no snapshot is provided, the diff extents will be all those changed
since the rbd volume/base was created, otherwise it will be those
changed since the snapshot was created.
"""
LOG.debug("Performing differential transfer from '%(src)s' to "
"'%(dest)s'" %
{'src': src_name, 'dest': dest_name})
# NOTE(dosaboy): Need to be tolerant of clusters/clients that do
# not support these operations since at the time of writing they
# were very new.
src_ceph_args = self._ceph_args(src_user, src_conf, pool=src_pool)
dest_ceph_args = self._ceph_args(dest_user, dest_conf, pool=dest_pool)
cmd1 = ['rbd', 'export-diff'] + src_ceph_args
if from_snap is not None:
cmd1.extend(['--from-snap', from_snap])
if src_snap:
path = encodeutils.safe_encode("%s/%s@%s" %
(src_pool, src_name, src_snap))
else:
path = encodeutils.safe_encode("%s/%s" % (src_pool, src_name))
cmd1.extend([path, '-'])
cmd2 = ['rbd', 'import-diff'] + dest_ceph_args
rbd_path = encodeutils.safe_encode("%s/%s" % (dest_pool, dest_name))
cmd2.extend(['-', rbd_path])
ret, stderr = self._piped_execute(cmd1, cmd2)
if ret:
msg = (_("RBD diff op failed - (ret=%(ret)s stderr=%(stderr)s)") %
{'ret': ret, 'stderr': stderr})
LOG.info(msg)
raise exception.BackupRBDOperationFailed(msg)
def __init__(self,
driver,
name,
pool=None,
snapshot=None,
read_only=False):
client, ioctx = driver._connect_to_rados(pool)
if snapshot is not None:
snapshot = encodeutils.safe_encode(snapshot)
try:
self.volume = driver.rbd.Image(ioctx,
encodeutils.safe_encode(name),
snapshot=snapshot,
read_only=read_only)
except driver.rbd.Error:
LOG.exception(_LE("error opening rbd image %s"), name)
driver._disconnect_from_rados(client, ioctx)
raise
self.driver = driver
self.client = client
self.ioctx = ioctx
