def test_get(self):
# Volume won't have anything other than base by default
meta = self.bak_meta_api.get(self.volume_id)
s1 = set(jsonutils.loads(meta).keys())
s2 = ['version', self.bak_meta_api.TYPE_TAG_VOL_BASE_META]
self.assertEqual(s1.symmetric_difference(s2), set())
self._add_metadata(vol_glance_meta=True)
meta = self.bak_meta_api.get(self.volume_id)
s1 = set(jsonutils.loads(meta).keys())
s2 = [
'version', self.bak_meta_api.TYPE_TAG_VOL_BASE_META,
self.bak_meta_api.TYPE_TAG_VOL_GLANCE_META
]
self.assertEqual(s1.symmetric_difference(s2), set())
self._add_metadata(vol_meta=True)
meta = self.bak_meta_api.get(self.volume_id)
s1 = set(jsonutils.loads(meta).keys())
s2 = [
'version', self.bak_meta_api.TYPE_TAG_VOL_BASE_META,
self.bak_meta_api.TYPE_TAG_VOL_GLANCE_META,
self.bak_meta_api.TYPE_TAG_VOL_META
]
self.assertEqual(s1.symmetric_difference(s2), set())
def test_list_extensions_json(self):
app = router.APIRouter()
request = webob.Request.blank("/fake/extensions")
response = request.get_response(app)
self.assertEqual(200, response.status_int)
# Make sure we have all the extensions, extra extensions being OK.
data = jsonutils.loads(response.body)
names = [str(x['name']) for x in data['extensions']
if str(x['name']) in self.ext_list]
names.sort()
self.assertEqual(names, self.ext_list)
# Ensure all the timestamps are valid according to iso8601
for ext in data['extensions']:
iso8601.parse_date(ext['updated'])
# Make sure that at least Fox in Sox is correct.
(fox_ext, ) = [
x for x in data['extensions'] if x['alias'] == 'FOXNSOX']
self.assertEqual(
fox_ext, {'namespace': 'http://www.fox.in.socks/api/ext/pie/v1.0',
'name': 'Fox In Socks',
'updated': '2011-01-22T13:25:27-06:00',
'description': 'The Fox In Socks Extension.',
'alias': 'FOXNSOX',
'links': []}, )
for ext in data['extensions']:
url = '/fake/extensions/%s' % ext['alias']
request = webob.Request.blank(url)
response = request.get_response(app)
output = jsonutils.loads(response.body)
self.assertEqual(output['extension']['alias'], ext['alias'])
def test_list_extensions_json(self):
app = router.APIRouter()
request = webob.Request.blank("/fake/extensions")
response = request.get_response(app)
self.assertEqual(200, response.status_int)
# Make sure we have all the extensions, extra extensions being OK.
data = jsonutils.loads(response.body)
names = [str(x['name']) for x in data['extensions']
if str(x['name']) in self.ext_list]
names.sort()
self.assertEqual(names, self.ext_list)
# Ensure all the timestamps are valid according to iso8601
for ext in data['extensions']:
iso8601.parse_date(ext['updated'])
# Make sure that at least Fox in Sox is correct.
(fox_ext, ) = [
x for x in data['extensions'] if x['alias'] == 'FOXNSOX']
self.assertEqual(
fox_ext, {'namespace': 'http://www.fox.in.socks/api/ext/pie/v1.0',
'name': 'Fox In Socks',
'updated': '2011-01-22T13:25:27-06:00',
'description': 'The Fox In Socks Extension',
'alias': 'FOXNSOX',
'links': []}, )
for ext in data['extensions']:
url = '/fake/extensions/%s' % ext['alias']
request = webob.Request.blank(url)
response = request.get_response(app)
output = jsonutils.loads(response.body)
self.assertEqual(output['extension']['alias'], ext['alias'])
def __call__(self, *args):
data = jsonutils.dumps({'object': self.obj,
'method': self.method,
'params': args})
auth = ('%s:%s' % (self.user, self.password)).encode('base64')[:-1]
headers = {'Content-Type': 'application/json',
'Authorization': 'Basic %s' % (auth,)}
LOG.debug(_('Sending JSON data: %s'), data)
request = urllib2.Request(self.url, data, headers)
response_obj = urllib2.urlopen(request)
if response_obj.info().status == 'EOF in headers':
if self.auto and self.url.startswith('http://'):
LOG.info(_('Auto switching to HTTPS connection to %s'),
self.url)
self.url = 'https' + self.url[4:]
request = urllib2.Request(self.url, data, headers)
response_obj = urllib2.urlopen(request)
else:
LOG.error(_('No headers in server response'))
raise NexentaJSONException(_('Bad response from server'))
response_data = response_obj.read()
LOG.debug(_('Got response: %s'), response_data)
response = jsonutils.loads(response_data)
if response.get('error') is not None:
raise NexentaJSONException(response['error'].get('message', ''))
else:
return response.get('result')
def _decode_json(self, response):
body = response.text
LOG.debug("Decoding JSON: %s" % (body))
if body:
return jsonutils.loads(body)
else:
return ""
def test_empty_index_json(self):
"""Test getting empty limit details in JSON."""
request = self._get_index_request()
response = request.get_response(self.controller)
expected = {"limits": {"rate": [], "absolute": {}}}
body = jsonutils.loads(response.body)
self.assertEqual(expected, body)
def test_export_record(self):
export_string = self.driver.export_record(self.backup)
export_dict = jsonutils.loads(export_string.decode("base64"))
# Make sure we don't lose data when converting to string
for key in _backup_db_fields:
self.assertTrue(key in export_dict)
self.assertEqual(self.backup[key], export_dict[key])
def __call__(self, *args):
data = jsonutils.dumps({
'object': self.obj,
'method': self.method,
'params': args
})
auth = ('%s:%s' % (self.user, self.password)).encode('base64')[:-1]
headers = {
'Content-Type': 'application/json',
'Authorization': 'Basic %s' % auth
}
LOG.debug('Sending JSON data: %s', data)
request = urllib2.Request(self.url, data, headers)
response_obj = urllib2.urlopen(request)
if response_obj.info().status == 'EOF in headers':
if not self.auto or self.scheme != 'http':
LOG.error(_('No headers in server response'))
raise NexentaJSONException(_('Bad response from server'))
LOG.info(_('Auto switching to HTTPS connection to %s'), self.url)
self.scheme = 'https'
request = urllib2.Request(self.url, data, headers)
response_obj = urllib2.urlopen(request)
response_data = response_obj.read()
LOG.debug('Got response: %s', response_data)
response = jsonutils.loads(response_data)
if response.get('error') is not None:
raise NexentaJSONException(response['error'].get('message', ''))
return response.get('result')
def _decode_json(self, response):
body = response.text
LOG.debug(_("Decoding JSON: %s") % (body))
if body:
return jsonutils.loads(body)
else:
return ""
def test_export_record(self):
export_string = self.driver.export_record(self.backup)
export_dict = jsonutils.loads(export_string.decode("base64"))
# Make sure we don't lose data when converting to string
for key in _backup_db_fields:
self.assertTrue(key in export_dict)
self.assertEqual(self.backup[key], export_dict[key])
def __call__(self, request):
"""Handles a call to this application.
Returns 204 if the request is acceptable to the limiter, else a 403
is returned with a relevant header indicating when the request
*will* succeed.
"""
if request.method != "POST":
raise webob.exc.HTTPMethodNotAllowed()
try:
info = dict(jsonutils.loads(request.body))
except ValueError:
raise webob.exc.HTTPBadRequest()
username = request.path_info_pop()
verb = info.get("verb")
path = info.get("path")
delay, error = self._limiter.check_for_delay(verb, path, username)
if delay:
headers = {"X-Wait-Seconds": "%.2f" % delay}
return webob.exc.HTTPForbidden(headers=headers, explanation=error)
else:
return webob.exc.HTTPNoContent()
def _issue_api_request(self, method_name, params):
"""All API requests to SolidFire device go through this method
Simple json-rpc web based API calls.
each call takes a set of paramaters (dict)
and returns results in a dict as well.
"""
host = FLAGS.san_ip
# For now 443 is the only port our server accepts requests on
port = 443
# NOTE(john-griffith): Probably don't need this, but the idea is
# we provide a request_id so we can correlate
# responses with requests
request_id = int(uuid.uuid4()) # just generate a random number
cluster_admin = FLAGS.san_login
cluster_password = FLAGS.san_password
command = {'method': method_name,
'id': request_id}
if params is not None:
command['params'] = params
payload = jsonutils.dumps(command, ensure_ascii=False)
payload.encode('utf-8')
# we use json-rpc, webserver needs to see json-rpc in header
header = {'Content-Type': 'application/json-rpc; charset=utf-8'}
if cluster_password is not None:
# base64.encodestring includes a newline character
# in the result, make sure we strip it off
auth_key = base64.encodestring('%s:%s' % (cluster_admin,
cluster_password))[:-1]
header['Authorization'] = 'Basic %s' % auth_key
LOG.debug(_("Payload for SolidFire API call: %s"), payload)
connection = httplib.HTTPSConnection(host, port)
connection.request('POST', '/json-rpc/1.0', payload, header)
response = connection.getresponse()
data = {}
if response.status != 200:
connection.close()
raise exception.SolidFireAPIException(status=response.status)
else:
data = response.read()
try:
data = jsonutils.loads(data)
except (TypeError, ValueError), exc:
connection.close()
msg = _("Call to json.loads() raised an exception: %s") % exc
raise exception.SfJsonEncodeFailure(msg)
connection.close()
def load_json(cls, data, default_rule=None):
"""Allow loading of JSON rule data."""
# Suck in the JSON data and parse the rules
rules = dict((k, parse_rule(v)) for k, v in
jsonutils.loads(data).items())
return cls(rules, default_rule)
def load_json(cls, data, default_rule=None):
"""Allow loading of JSON rule data."""
# Suck in the JSON data and parse the rules
rules = dict(
(k, parse_rule(v)) for k, v in jsonutils.loads(data).items())
return cls(rules, default_rule)
def test_index_json(self):
"""Test getting limit details in JSON."""
request = self._get_index_request()
request = self._populate_limits(request)
self.absolute_limits = {
'gigabytes': 512,
'volumes': 5,
}
response = request.get_response(self.controller)
expected = {
"limits": {
"rate": [
{
"regex":
".*",
"uri":
"*",
"limit": [
{
"verb": "GET",
"next-available": "1970-01-01T00:00:00Z",
"unit": "MINUTE",
"value": 10,
"remaining": 10,
},
{
"verb": "POST",
"next-available": "1970-01-01T00:00:00Z",
"unit": "HOUR",
"value": 5,
"remaining": 5,
},
],
},
{
"regex":
"changes-since",
"uri":
"changes-since*",
"limit": [
{
"verb": "GET",
"next-available": "1970-01-01T00:00:00Z",
"unit": "MINUTE",
"value": 5,
"remaining": 5,
},
],
},
],
"absolute": {
"maxTotalVolumeGigabytes": 512,
"maxTotalVolumes": 5,
},
},
}
body = jsonutils.loads(response.body)
self.assertEqual(expected, body)
def test_index_json(self):
"""Test getting limit details in JSON."""
request = self._get_index_request()
request = self._populate_limits(request)
self.absolute_limits = {
'gigabytes': 512,
'volumes': 5,
}
response = request.get_response(self.controller)
expected = {
"limits": {
"rate": [
{
"regex": ".*",
"uri": "*",
"limit": [
{
"verb": "GET",
"next-available": "1970-01-01T00:00:00Z",
"unit": "MINUTE",
"value": 10,
"remaining": 10,
},
{
"verb": "POST",
"next-available": "1970-01-01T00:00:00Z",
"unit": "HOUR",
"value": 5,
"remaining": 5,
},
],
},
{
"regex": "changes-since",
"uri": "changes-since*",
"limit": [
{
"verb": "GET",
"next-available": "1970-01-01T00:00:00Z",
"unit": "MINUTE",
"value": 5,
"remaining": 5,
},
],
},
],
"absolute": {
"maxTotalVolumeGigabytes": 512,
"maxTotalVolumes": 5,
},
},
}
body = jsonutils.loads(response.body)
self.assertEqual(expected, body)
def import_record(self, backup_url):
"""Import and verify backup record.
Default backup driver implementation.
De-serialize the backup record into a dictionary, so we can
update the database.
:param backup_url: driver specific backup record string
:returns dictionary object with database updates
"""
return jsonutils.loads(backup_url.decode("base64"))
def import_record(self, backup_url):
"""Import and verify backup record.
Default backup driver implementation.
De-serialize the backup record into a dictionary, so we can
update the database.
:param backup_url: driver specific backup record string
:returns dictionary object with database updates
"""
return jsonutils.loads(backup_url.decode("base64"))
def test_empty_index_json(self):
"""Test getting empty limit details in JSON."""
request = self._get_index_request()
response = request.get_response(self.controller)
expected = {
"limits": {
"rate": [],
"absolute": {},
},
}
body = jsonutils.loads(response.body)
self.assertEqual(expected, body)
def _rpc(self, handle, url_params, data, allow_empty_response):
"""Execute REST RPC using url <esm_url>/handle?url_params.
Send JSON encoded data in body of POST request.
Exceptions:
urllib2.URLError
1. Name or service not found (e.reason is socket.gaierror)
2. Socket blocking operation timeout (e.reason is
socket.timeout)
3. Network IO error (e.reason is socket.error)
urllib2.HTTPError
1. HTTP 404, HTTP 500 etc.
CoraidJsonEncodeFailure - bad REST response
"""
# Handle must be simple path, for example:
# /configure
if '?' in handle or '&' in handle:
raise ValueError(_('Invalid REST handle name. Expected path.'))
# Request url includes base ESM url, handle path and optional
# URL params.
rest_url = urlparse.urljoin(self._esm_url, handle)
encoded_url_params = urllib.urlencode(url_params)
if encoded_url_params:
rest_url += '?' + encoded_url_params
if data is None:
json_request = None
else:
json_request = jsonutils.dumps(data)
request = urllib2.Request(rest_url, json_request)
response = self._url_opener.open(request).read()
try:
if not response and allow_empty_response:
reply = {}
else:
reply = jsonutils.loads(response)
except (TypeError, ValueError) as exc:
msg = (_('Call to json.loads() failed: %(ex)s.'
' Response: %(resp)s') % {
'ex': exc,
'resp': response
})
raise exception.CoraidJsonEncodeFailure(msg)
return reply
def test_get(self):
# Volume won't have anything other than base by default
meta = self.bak_meta_api.get(self.volume_id)
s1 = set(jsonutils.loads(meta).keys())
s2 = ['version', self.bak_meta_api.TYPE_TAG_VOL_BASE_META]
self.assertEqual(s1.symmetric_difference(s2), set())
self._add_metadata(vol_glance_meta=True)
meta = self.bak_meta_api.get(self.volume_id)
s1 = set(jsonutils.loads(meta).keys())
s2 = ['version', self.bak_meta_api.TYPE_TAG_VOL_BASE_META,
self.bak_meta_api.TYPE_TAG_VOL_GLANCE_META]
self.assertEqual(s1.symmetric_difference(s2), set())
self._add_metadata(vol_meta=True)
meta = self.bak_meta_api.get(self.volume_id)
s1 = set(jsonutils.loads(meta).keys())
s2 = ['version', self.bak_meta_api.TYPE_TAG_VOL_BASE_META,
self.bak_meta_api.TYPE_TAG_VOL_GLANCE_META,
self.bak_meta_api.TYPE_TAG_VOL_META]
self.assertEqual(s1.symmetric_difference(s2), set())
def test_get_extension_json(self):
app = volume.APIRouter()
request = webob.Request.blank("/fake/extensions/FOXNSOX")
response = request.get_response(app)
self.assertEqual(200, response.status_int)
data = jsonutils.loads(response.body)
self.assertEqual(data['extension'], {
"namespace": "http://www.fox.in.socks/api/ext/pie/v1.0",
"name": "Fox In Socks",
"updated": "2011-01-22T13:25:27-06:00",
"description": "The Fox In Socks Extension",
"alias": "FOXNSOX",
"links": []})
def _rpc(self, handle, url_params, data, allow_empty_response):
"""Execute REST RPC using url <esm_url>/handle?url_params.
Send JSON encoded data in body of POST request.
Exceptions:
urllib2.URLError
1. Name or service not found (e.reason is socket.gaierror)
2. Socket blocking operation timeout (e.reason is
socket.timeout)
3. Network IO error (e.reason is socket.error)
urllib2.HTTPError
1. HTTP 404, HTTP 500 etc.
CoraidJsonEncodeFailure - bad REST response
"""
# Handle must be simple path, for example:
# /configure
if '?' in handle or '&' in handle:
raise ValueError(_('Invalid REST handle name. Expected path.'))
# Request url includes base ESM url, handle path and optional
# URL params.
rest_url = urlparse.urljoin(self._esm_url, handle)
encoded_url_params = urllib.urlencode(url_params)
if encoded_url_params:
rest_url += '?' + encoded_url_params
if data is None:
json_request = None
else:
json_request = jsonutils.dumps(data)
request = urllib2.Request(rest_url, json_request)
response = self._url_opener.open(request).read()
try:
if not response and allow_empty_response:
reply = {}
else:
reply = jsonutils.loads(response)
except (TypeError, ValueError) as exc:
msg = (_('Call to json.loads() failed: %(ex)s.'
' Response: %(resp)s') %
{'ex': exc, 'resp': response})
raise exception.CoraidJsonEncodeFailure(msg)
return reply
def _unpack_json_msg(self, msg):
"""Load the JSON data in msg if msg.content_type indicates that it
is necessary. Put the loaded data back into msg.content and
update msg.content_type appropriately.
A Qpid Message containing a dict will have a content_type of
'amqp/map', whereas one containing a string that needs to be converted
back from JSON will have a content_type of JSON_CONTENT_TYPE.
:param msg: a Qpid Message object
:returns: None
"""
if msg.content_type == JSON_CONTENT_TYPE:
msg.content = jsonutils.loads(msg.content)
msg.content_type = 'amqp/map'
def _unpack_json_msg(self, msg):
"""Load the JSON data in msg if msg.content_type indicates that it
is necessary. Put the loaded data back into msg.content and
update msg.content_type appropriately.
A Qpid Message containing a dict will have a content_type of
'amqp/map', whereas one containing a string that needs to be converted
back from JSON will have a content_type of JSON_CONTENT_TYPE.
:param msg: a Qpid Message object
:returns: None
"""
if msg.content_type == JSON_CONTENT_TYPE:
msg.content = jsonutils.loads(msg.content)
msg.content_type = 'amqp/map'
def test_get_extension_json(self):
app = router.APIRouter()
request = webob.Request.blank("/fake/extensions/FOXNSOX")
response = request.get_response(app)
self.assertEqual(200, response.status_int)
data = jsonutils.loads(response.body)
self.assertEqual(
data['extension'],
{"namespace": "http://www.fox.in.socks/api/ext/pie/v1.0",
"name": "Fox In Socks",
"updated": "2011-01-22T13:25:27-06:00",
"description": "The Fox In Socks Extension.",
"alias": "FOXNSOX",
"links": []})
def test_initialize_connection(self):
def fake_initialize_connection(*args, **kwargs):
return {}
self.stubs.Set(volume.API, 'initialize_connection',
fake_initialize_connection)
body = {'os-initialize_connection': {'connector': 'fake'}}
req = webob.Request.blank('/v1/fake/volumes/1/action')
req.method = "POST"
req.body = jsonutils.dumps(body)
req.headers["content-type"] = "application/json"
res = req.get_response(fakes.wsgi_app())
output = jsonutils.loads(res.body)
self.assertEqual(res.status_int, 200)
def __call__(self, req):
user_id = req.headers.get('X_USER')
user_id = req.headers.get('X_USER_ID', user_id)
if user_id is None:
LOG.debug("Neither X_USER_ID nor X_USER found in request")
return webob.exc.HTTPUnauthorized()
# get the roles
roles = [r.strip() for r in req.headers.get('X_ROLE', '').split(',')]
if 'X_TENANT_ID' in req.headers:
# This is the new header since Keystone went to ID/Name
project_id = req.headers['X_TENANT_ID']
else:
# This is for legacy compatibility
project_id = req.headers['X_TENANT']
project_name = req.headers.get('X_TENANT_NAME')
req_id = req.environ.get(request_id.ENV_REQUEST_ID)
# Get the auth token
auth_token = req.headers.get('X_AUTH_TOKEN',
req.headers.get('X_STORAGE_TOKEN'))
# Build a context, including the auth_token...
remote_address = req.remote_addr
service_catalog = None
if req.headers.get('X_SERVICE_CATALOG') is not None:
try:
catalog_header = req.headers.get('X_SERVICE_CATALOG')
service_catalog = jsonutils.loads(catalog_header)
except ValueError:
raise webob.exc.HTTPInternalServerError(
_('Invalid service catalog json.'))
if CONF.use_forwarded_for:
remote_address = req.headers.get('X-Forwarded-For', remote_address)
ctx = context.RequestContext(user_id,
project_id,
project_name=project_name,
roles=roles,
auth_token=auth_token,
remote_address=remote_address,
service_catalog=service_catalog,
request_id=req_id)
req.environ['cinder.context'] = ctx
return self.application
def __call__(self, req):
user_id = req.headers.get('X_USER')
user_id = req.headers.get('X_USER_ID', user_id)
if user_id is None:
LOG.debug("Neither X_USER_ID nor X_USER found in request")
return webob.exc.HTTPUnauthorized()
# get the roles
roles = [r.strip() for r in req.headers.get('X_ROLE', '').split(',')]
if 'X_TENANT_ID' in req.headers:
# This is the new header since Keystone went to ID/Name
project_id = req.headers['X_TENANT_ID']
else:
# This is for legacy compatibility
project_id = req.headers['X_TENANT']
project_name = req.headers.get('X_TENANT_NAME')
req_id = req.environ.get(request_id.ENV_REQUEST_ID)
# Get the auth token
auth_token = req.headers.get('X_AUTH_TOKEN',
req.headers.get('X_STORAGE_TOKEN'))
# Build a context, including the auth_token...
remote_address = req.remote_addr
service_catalog = None
if req.headers.get('X_SERVICE_CATALOG') is not None:
try:
catalog_header = req.headers.get('X_SERVICE_CATALOG')
service_catalog = jsonutils.loads(catalog_header)
except ValueError:
raise webob.exc.HTTPInternalServerError(
explanation=_('Invalid service catalog json.'))
if CONF.use_forwarded_for:
remote_address = req.headers.get('X-Forwarded-For', remote_address)
ctx = context.RequestContext(user_id,
project_id,
project_name=project_name,
roles=roles,
auth_token=auth_token,
remote_address=remote_address,
service_catalog=service_catalog,
request_id=req_id)
req.environ['cinder.context'] = ctx
return self.application
def action_peek_json(body):
"""Determine action to invoke."""
try:
decoded = jsonutils.loads(body)
except ValueError:
msg = _("cannot understand JSON")
raise exception.MalformedRequestBody(reason=msg)
# Make sure there's exactly one key...
if len(decoded) != 1:
msg = _("too many body keys")
raise exception.MalformedRequestBody(reason=msg)
# Return the action and the decoded body...
return decoded.keys()[0]
def action_peek_json(body):
"""Determine action to invoke."""
try:
decoded = jsonutils.loads(body)
except ValueError:
msg = _("cannot understand JSON")
raise exception.MalformedRequestBody(reason=msg)
# Make sure there's exactly one key...
if len(decoded) != 1:
msg = _("too many body keys")
raise exception.MalformedRequestBody(reason=msg)
# Return the action and the decoded body...
return decoded.keys()[0]
def test_limited_request_json(self):
"""Test a rate-limited (413) GET request through middleware."""
request = webob.Request.blank("/")
response = request.get_response(self.app)
self.assertEqual(200, response.status_int)
request = webob.Request.blank("/")
response = request.get_response(self.app)
self.assertEqual(response.status_int, 413)
self.assertIn("Retry-After", response.headers)
retry_after = int(response.headers["Retry-After"])
self.assertAlmostEqual(retry_after, 60, 1)
body = jsonutils.loads(response.body)
expected = "Only 1 GET request(s) can be made to * every minute."
value = body["overLimitFault"]["details"].strip()
self.assertEqual(value, expected)
def test_limited_request_json(self):
"""Test a rate-limited (413) GET request through middleware."""
request = webob.Request.blank("/")
response = request.get_response(self.app)
self.assertEqual(200, response.status_int)
request = webob.Request.blank("/")
response = request.get_response(self.app)
self.assertEqual(response.status_int, 413)
self.assertIn('Retry-After', response.headers)
retry_after = int(response.headers['Retry-After'])
self.assertAlmostEqual(retry_after, 60, 1)
body = jsonutils.loads(response.body)
expected = "Only 1 GET request(s) can be made to * every minute."
value = body["overLimitFault"]["details"].strip()
self.assertEqual(value, expected)
def test_index_diff_regex(self):
"""Test getting limit details in JSON."""
request = self._get_index_request()
request = self._populate_limits_diff_regex(request)
response = request.get_response(self.controller)
expected = {
"limits": {
"rate": [
{
"regex":
".*",
"uri":
"*",
"limit": [
{
"verb": "GET",
"next-available": "1970-01-01T00:00:00Z",
"unit": "MINUTE",
"value": 10,
"remaining": 10,
},
],
},
{
"regex":
"*.*",
"uri":
"*",
"limit": [
{
"verb": "GET",
"next-available": "1970-01-01T00:00:00Z",
"unit": "MINUTE",
"value": 10,
"remaining": 10,
},
],
},
],
"absolute": {},
},
}
body = jsonutils.loads(response.body)
self.assertEqual(expected, body)
def deserialize_msg(msg):
# NOTE(russellb): Hang on to your hats, this road is about to
# get a little bumpy.
#
# Robustness Principle:
# "Be strict in what you send, liberal in what you accept."
#
# At this point we have to do a bit of guessing about what it
# is we just received. Here is the set of possibilities:
#
# 1) We received a dict. This could be 2 things:
#
# a) Inspect it to see if it looks like a standard message envelope.
# If so, great!
#
# b) If it doesn't look like a standard message envelope, it could either
# be a notification, or a message from before we added a message
# envelope (referred to as version 1.0).
# Just return the message as-is.
#
# 2) It's any other non-dict type. Just return it and hope for the best.
# This case covers return values from rpc.call() from before message
# envelopes were used. (messages to call a method were always a dict)
if not isinstance(msg, dict):
# See #2 above.
return msg
base_envelope_keys = (_VERSION_KEY, _MESSAGE_KEY)
if not all(map(lambda key: key in msg, base_envelope_keys)):
# See #1.b above.
return msg
# At this point we think we have the message envelope
# format we were expecting. (#1.a above)
if not version_is_compatible(_RPC_ENVELOPE_VERSION, msg[_VERSION_KEY]):
raise UnsupportedRpcEnvelopeVersion(version=msg[_VERSION_KEY])
raw_msg = jsonutils.loads(msg[_MESSAGE_KEY])
return raw_msg
def deserialize_msg(msg):
# NOTE(russellb): Hang on to your hats, this road is about to
# get a little bumpy.
#
# Robustness Principle:
# "Be strict in what you send, liberal in what you accept."
#
# At this point we have to do a bit of guessing about what it
# is we just received. Here is the set of possibilities:
#
# 1) We received a dict. This could be 2 things:
#
# a) Inspect it to see if it looks like a standard message envelope.
# If so, great!
#
# b) If it doesn't look like a standard message envelope, it could either
# be a notification, or a message from before we added a message
# envelope (referred to as version 1.0).
# Just return the message as-is.
#
# 2) It's any other non-dict type. Just return it and hope for the best.
# This case covers return values from rpc.call() from before message
# envelopes were used. (messages to call a method were always a dict)
if not isinstance(msg, dict):
# See #2 above.
return msg
base_envelope_keys = (_VERSION_KEY, _MESSAGE_KEY)
if not all(map(lambda key: key in msg, base_envelope_keys)):
# See #1.b above.
return msg
# At this point we think we have the message envelope
# format we were expecting. (#1.a above)
if not version_is_compatible(_RPC_ENVELOPE_VERSION, msg[_VERSION_KEY]):
raise UnsupportedRpcEnvelopeVersion(version=msg[_VERSION_KEY])
raw_msg = jsonutils.loads(msg[_MESSAGE_KEY])
return raw_msg
def test_index_diff_regex(self):
"""Test getting limit details in JSON."""
request = self._get_index_request()
request = self._populate_limits_diff_regex(request)
response = request.get_response(self.controller)
expected = {
"limits": {
"rate": [
{
"regex": ".*",
"uri": "*",
"limit": [
{
"verb": "GET",
"next-available": "1970-01-01T00:00:00Z",
"unit": "MINUTE",
"value": 10,
"remaining": 10,
},
],
},
{
"regex": "*.*",
"uri": "*",
"limit": [
{
"verb": "GET",
"next-available": "1970-01-01T00:00:00Z",
"unit": "MINUTE",
"value": 10,
"remaining": 10,
},
],
},
],
"absolute": {},
},
}
body = jsonutils.loads(response.body)
self.assertEqual(expected, body)
def deserialize_remote_exception(conf, data):
failure = jsonutils.loads(str(data))
trace = failure.get('tb', [])
message = failure.get('message', "") + "\n" + "\n".join(trace)
name = failure.get('class')
module = failure.get('module')
# NOTE(ameade): We DO NOT want to allow just any module to be imported, in
# order to prevent arbitrary code execution.
if module not in conf.allowed_rpc_exception_modules:
return RemoteError(name, failure.get('message'), trace)
try:
mod = importutils.import_module(module)
klass = getattr(mod, name)
if not issubclass(klass, Exception):
raise TypeError("Can only deserialize Exceptions")
failure = klass(**failure.get('kwargs', {}))
except (AttributeError, TypeError, ImportError):
return RemoteError(name, failure.get('message'), trace)
ex_type = type(failure)
str_override = lambda self: message
new_ex_type = type(ex_type.__name__ + "_Remote", (ex_type, ), {
'__str__': str_override,
'__unicode__': str_override
})
try:
# NOTE(ameade): Dynamically create a new exception type and swap it in
# as the new type for the exception. This only works on user defined
# Exceptions and not core python exceptions. This is important because
# we cannot necessarily change an exception message so we must override
# the __str__ method.
failure.__class__ = new_ex_type
except TypeError:
# NOTE(ameade): If a core exception then just add the traceback to the
# first exception argument.
failure.args = (message, ) + failure.args[1:]
return failure
def test_400_fault_json(self):
"""Test fault serialized to JSON via file-extension and/or header."""
requests = [
webob.Request.blank('/.json'),
webob.Request.blank('/', headers={"Accept": "application/json"}),
]
for request in requests:
fault = wsgi.Fault(webob.exc.HTTPBadRequest(explanation='scram'))
response = request.get_response(fault)
expected = {
"badRequest": {
"message": "scram",
"code": 400,
},
}
actual = jsonutils.loads(response.body)
self.assertEqual(response.content_type, "application/json")
self.assertEqual(expected, actual)
def put(self, volume_id, json_metadata):
"""Restore volume metadata to a volume.
The json container should contain a version that is supported here.
"""
meta_container = jsonutils.loads(json_metadata)
version = meta_container['version']
if version == 1:
factory = self._v1_restore_factory()
else:
msg = (_("Unsupported backup metadata version (%s)") % (version))
raise exception.BackupMetadataUnsupportedVersion(msg)
for type in factory:
func = factory[type][0]
fields = factory[type][1]
if type in meta_container:
func(meta_container[type], volume_id, fields)
else:
msg = "No metadata of type '%s' to restore" % (type)
LOG.debug(msg)
def put(self, volume_id, json_metadata):
"""Restore volume metadata to a volume.
The json container should contain a version that is supported here.
"""
meta_container = jsonutils.loads(json_metadata)
version = meta_container['version']
if version == 1:
factory = self._v1_restore_factory()
else:
msg = (_("Unsupported backup metadata version (%s)") % (version))
raise exception.BackupMetadataUnsupportedVersion(msg)
for type in factory:
func = factory[type][0]
fields = factory[type][1]
if type in meta_container:
func(meta_container[type], volume_id, fields)
else:
msg = _("No metadata of type '%s' to restore") % (type)
LOG.debug(msg)
def deserialize_remote_exception(conf, data):
failure = jsonutils.loads(str(data))
trace = failure.get('tb', [])
message = failure.get('message', "") + "\n" + "\n".join(trace)
name = failure.get('class')
module = failure.get('module')
# NOTE(ameade): We DO NOT want to allow just any module to be imported, in
# order to prevent arbitrary code execution.
if module not in conf.allowed_rpc_exception_modules:
return RemoteError(name, failure.get('message'), trace)
try:
mod = importutils.import_module(module)
klass = getattr(mod, name)
if not issubclass(klass, Exception):
raise TypeError("Can only deserialize Exceptions")
failure = klass(*failure.get('args', []), **failure.get('kwargs', {}))
except (AttributeError, TypeError, ImportError):
return RemoteError(name, failure.get('message'), trace)
ex_type = type(failure)
str_override = lambda self: message
new_ex_type = type(ex_type.__name__ + _REMOTE_POSTFIX, (ex_type,),
{'__str__': str_override, '__unicode__': str_override})
new_ex_type.__module__ = '%s%s' % (module, _REMOTE_POSTFIX)
try:
# NOTE(ameade): Dynamically create a new exception type and swap it in
# as the new type for the exception. This only works on user defined
# Exceptions and not core python exceptions. This is important because
# we cannot necessarily change an exception message so we must override
# the __str__ method.
failure.__class__ = new_ex_type
except TypeError:
# NOTE(ameade): If a core exception then just add the traceback to the
# first exception argument.
failure.args = (message,) + failure.args[1:]
return failure
def _admin_esm_cmd(self, url=False, data=None):
"""
_admin_esm_cmd represent the entry point to send requests to ESM
Appliance. Send the HTTPS call, get response in JSON
convert response into Python Object and return it.
"""
if url:
url = self.url + url
req = urllib2.Request(url, data)
try:
res = self.urlOpener.open(req).read()
except Exception:
raise CoraidRESTException(_('ESM urlOpen error'))
try:
res_json = jsonutils.loads(res)
except Exception:
raise CoraidRESTException(_('JSON Error'))
return res_json
else:
raise CoraidRESTException(_('Request without URL'))
def _admin_esm_cmd(self, url=False, data=None):
"""
_admin_esm_cmd represent the entry point to send requests to ESM
Appliance. Send the HTTPS call, get response in JSON
convert response into Python Object and return it.
"""
if url:
url = self.url + url
req = urllib2.Request(url, data)
try:
res = self.urlOpener.open(req).read()
except Exception:
raise CoraidRESTException(_("ESM urlOpen error"))
try:
res_json = jsonutils.loads(res)
except Exception:
raise CoraidRESTException(_("JSON Error"))
return res_json
else:
raise CoraidRESTException(_("Request without URL"))
def host_passes(self, host_state, filter_properties):
"""Return a list of hosts that can fulfill the requirements
specified in the query.
"""
# TODO(zhiteng) Add description for filter_properties structure
# and scheduler_hints.
try:
query = filter_properties['scheduler_hints']['query']
except KeyError:
query = None
if not query:
return True
# NOTE(comstud): Not checking capabilities or service for
# enabled/disabled so that a provided json filter can decide
result = self._process_filter(jsonutils.loads(query), host_state)
if isinstance(result, list):
# If any succeeded, include the host
result = any(result)
if result:
# Filter it out.
return True
return False
def test_413_fault_json(self):
"""Test fault serialized to JSON via file-extension and/or header."""
requests = [
webob.Request.blank('/.json'),
webob.Request.blank('/', headers={"Accept": "application/json"}),
]
for request in requests:
exc = webob.exc.HTTPRequestEntityTooLarge
fault = wsgi.Fault(exc(explanation='sorry',
headers={'Retry-After': 4}))
response = request.get_response(fault)
expected = {
"overLimit": {
"message": "sorry",
"code": 413,
"retryAfter": 4,
},
}
actual = jsonutils.loads(response.body)
self.assertEqual(response.content_type, "application/json")
self.assertEqual(expected, actual)
def host_passes(self, host_state, filter_properties):
"""Return a list of hosts that can fulfill the requirements
specified in the query.
"""
# TODO(zhiteng) Add description for filter_properties structure
# and scheduler_hints.
try:
query = filter_properties['scheduler_hints']['query']
except KeyError:
query = None
if not query:
return True
# NOTE(comstud): Not checking capabilities or service for
# enabled/disabled so that a provided json filter can decide
result = self._process_filter(jsonutils.loads(query), host_state)
if isinstance(result, list):
# If any succeeded, include the host
result = any(result)
if result:
# Filter it out.
return True
return False
def _from_json(self, datastring):
try:
return jsonutils.loads(datastring)
except ValueError:
msg = _("cannot understand JSON")
raise exception.MalformedRequestBody(reason=msg)
def _json_loads(properties, attr):
prop = properties[attr]
if isinstance(prop, basestring):
properties[attr] = jsonutils.loads(prop)
def _deserialize(data):
"""
Deserialization wrapper
"""
LOG.debug(_("Deserializing: %s"), data)
return jsonutils.loads(data)
def load_json(cls, data, default_rule=None):
"""Init a brain using json instead of a rules dictionary."""
rules_dict = jsonutils.loads(data)
return cls(rules=rules_dict, default_rule=default_rule)
def _get_snapshots(self, body):
return jsonutils.loads(body).get('snapshots')
def _from_json(self, datastring):
try:
return jsonutils.loads(datastring)
except ValueError:
msg = _("cannot understand JSON")
raise exception.MalformedRequestBody(reason=msg)
def _get_snapshots(self, body):
return jsonutils.loads(body).get('snapshots')
def test_get_metadata(self):
json_metadata = self.driver.get_metadata(self.volume_id)
metadata = jsonutils.loads(json_metadata)
self.assertEqual(metadata['version'], 1)
def _json_loads(properties, attr):
prop = properties[attr]
if isinstance(prop, basestring):
properties[attr] = jsonutils.loads(prop)
def _test_index_absolute_limits_json(self, expected):
request = self._get_index_request()
response = request.get_response(self.controller)
body = jsonutils.loads(response.body)
self.assertEqual(expected, body['limits']['absolute'])