def test_extract_complex(self):
"""L{Schema} can cope with complex schemas."""
schema = Schema(
Unicode("GroupName"), RawStr("IpPermissions.n.IpProtocol"),
Integer("IpPermissions.n.FromPort"),
Integer("IpPermissions.n.ToPort"),
Unicode("IpPermissions.n.Groups.m.UserId", optional=True),
Unicode("IpPermissions.n.Groups.m.GroupName", optional=True))
arguments, _ = schema.extract({
"GroupName":
"Foo",
"IpPermissions.1.IpProtocol":
"tcp",
"IpPermissions.1.FromPort":
"1234",
"IpPermissions.1.ToPort":
"5678",
"IpPermissions.1.Groups.1.GroupName":
"Bar",
"IpPermissions.1.Groups.2.GroupName":
"Egg"
})
self.assertEqual(u"Foo", arguments.GroupName)
self.assertEqual(1, len(arguments.IpPermissions))
self.assertEqual(1234, arguments.IpPermissions[0].FromPort)
self.assertEqual(5678, arguments.IpPermissions[0].ToPort)
self.assertEqual(2, len(arguments.IpPermissions[0].Groups))
self.assertEqual("Bar", arguments.IpPermissions[0].Groups[0].GroupName)
self.assertEqual("Egg", arguments.IpPermissions[0].Groups[1].GroupName)
def test_structure(self):
"""
L{Schema}s with L{Structure} parameters can have arguments extracted.
"""
schema = Schema(Structure("foo", {"a": Integer(), "b": Integer()}))
arguments, _ = schema.extract({"foo.a": "1", "foo.b": "2"})
self.assertEqual(1, arguments.foo.a)
self.assertEqual(2, arguments.foo.b)
def test_get_parameters_order_on_parameter_only_construction(self):
"""
L{Schema.get_parameters} returns the original list of L{Parameter}s
even when they are passed as positional arguments to L{Schema}.
"""
schema = Schema(Unicode("name"), List("scores", Integer()),
Integer("index", Integer()))
self.assertEqual(["name", "scores", "index"],
[p.name for p in schema.get_parameters()])
def test_parse_with_negative(self):
"""L{Integer.parse} converts the given raw C{value} to C{int}."""
parameter = Integer("Test")
error = self.assertRaises(APIError, parameter.coerce, "-1")
self.assertEqual(400, error.status)
self.assertEqual("InvalidParameterValue", error.code)
self.assertIn("Value must be at least 0.", error.message)
def test_parameter_doc(self):
"""
All L{Parameter} subclasses accept a 'doc' keyword argument.
"""
parameters = [
Unicode(doc="foo"),
RawStr(doc="foo"),
Integer(doc="foo"),
Bool(doc="foo"),
Enum(mapping={"hey": 1}, doc="foo"),
Date(doc="foo"),
List(item=Integer(), doc="foo"),
Structure(fields={}, doc="foo")
]
for parameter in parameters:
self.assertEqual("foo", parameter.doc)
def test_bundle_with_missing_parameter(self):
"""
L{Schema.bundle} raises an exception one of the given parameters
doesn't exist in the schema.
"""
schema = Schema(Integer("count"))
self.assertRaises(RuntimeError, schema.bundle, name="foo")
def test_bundle_with_arguments(self):
"""L{Schema.bundle} can bundle L{Arguments} too."""
schema = Schema(Unicode("name.n"), Integer("count"))
arguments = Arguments({"name": Arguments({1: "Foo", 7: "Bar"}),
"count": 123})
params = schema.bundle(arguments)
self.assertEqual({"name.1": "Foo", "name.7": "Bar", "count": "123"},
params)
def test_list_of_structures(self):
"""L{List}s of L{Structure}s are extracted properly."""
schema = Schema(
List("foo", Structure(fields={
"a": Integer(),
"b": Integer()
})))
arguments, _ = schema.extract({
"foo.1.a": "1",
"foo.1.b": "2",
"foo.2.a": "3",
"foo.2.b": "4"
})
self.assertEqual(1, arguments.foo[0]['a'])
self.assertEqual(2, arguments.foo[0]['b'])
self.assertEqual(3, arguments.foo[1]['a'])
self.assertEqual(4, arguments.foo[1]['b'])
def test_bundle_with_multiple(self):
"""
L{Schema.bundle} correctly handles multiple arguments.
"""
schema = Schema(Unicode("name.n"), Integer("count"))
params = schema.bundle(name=["Foo", "Bar"], count=123)
self.assertEqual({"name.1": "Foo", "name.2": "Bar", "count": "123"},
params)
def test_extend_maintains_parameter_order(self):
"""
Extending a schema with additional parameters puts the new parameters
at the end.
"""
schema = Schema(parameters=[Unicode("name"), Unicode("value")])
schema2 = schema.extend(parameters=[Integer("foo"), Unicode("index")])
self.assertEqual(["name", "value", "foo", "index"],
[p.name for p in schema2.get_parameters()])
def test_new_parameters(self):
"""
L{Schema} accepts a C{parameters} parameter to specify parameters in a
{name: field} format.
"""
schema = Schema(
parameters=[Structure("foo", fields={"l": List(item=Integer())})])
arguments, _ = schema.extract({"foo.l.1": "1", "foo.l.2": "2"})
self.assertEqual([1, 2], arguments.foo.l)
def test_add_extra_schema_items(self):
"""A list of new Parameters can be added to the Schema."""
schema = Schema(Unicode("name"))
schema = schema.extend(Unicode("computer"), Integer("count"))
arguments, _ = schema.extract({"name": "value", "computer": "testing",
"count": "5"})
self.assertEqual(u"value", arguments.name)
self.assertEqual("testing", arguments.computer)
self.assertEqual(5, arguments.count)
def test_get_parameters(self):
"""
L{Schema.get_parameters} returns the original list of parameters.
"""
schema = Schema(
parameters=[Unicode("name"),
List("scores", Integer())])
parameters = schema.get_parameters()
self.assertEqual("name", parameters[0].name)
self.assertEqual("scores", parameters[1].name)
def test_extract_with_optional_default(self):
"""
The value of C{default} on a parameter is used as the value when it is
not provided as an argument and the parameter is C{optional}.
"""
schema = Schema(Unicode("name"),
Integer("count", optional=True, default=5))
arguments, _ = schema.extract({"name": "value"})
self.assertEqual(u"value", arguments.name)
self.assertEqual(5, arguments.count)
def test_additional_schema_attributes(self):
"""
Additional data can be specified on the Schema class for specifying a
more rich schema.
"""
result = {'id': Integer(), 'name': Unicode(), 'data': RawStr()}
errors = [APIError]
schema = Schema(name="GetStuff",
doc="""Get the stuff.""",
parameters=[Integer("id"),
Unicode("scope")],
result=result,
errors=errors)
self.assertEqual("GetStuff", schema.name)
self.assertEqual("Get the stuff.", schema.doc)
self.assertEqual(result, schema.result)
self.assertEqual(set(errors), schema.errors)
def test_bundle_with_arguments_and_extra(self):
"""
L{Schema.bundle} can bundle L{Arguments} with keyword arguments too.
Keyword arguments take precedence.
"""
schema = Schema(Unicode("name.n"), Integer("count"))
arguments = Arguments({"name": {1: "Foo", 7: "Bar"}, "count": 321})
params = schema.bundle(arguments, count=123)
self.assertEqual({"name.1": "Foo", "name.2": "Bar", "count": "123"},
params)
def test_non_integer_string(self):
"""
The L{Integer} parameter raises an L{APIError} when passed non-int
values (in this case, a string).
"""
garbage = "blah"
parameter = Integer("Test")
error = self.assertRaises(APIError, parameter.coerce, garbage)
self.assertEqual(400, error.status)
self.assertEqual("InvalidParameterValue", error.code)
self.assertIn("Invalid integer value %s" % garbage, error.message)
def test_min_and_max(self):
"""The L{Integer} parameter properly supports ranges."""
parameter = Integer("Test", min=2, max=4)
error = self.assertRaises(APIError, parameter.coerce, "1")
self.assertEqual(400, error.status)
self.assertEqual("InvalidParameterValue", error.code)
self.assertIn("Value must be at least 2.", error.message)
error = self.assertRaises(APIError, parameter.coerce, "5")
self.assertIn("Value exceeds maximum of 4.", error.message)
self.assertEqual(400, error.status)
self.assertEqual("InvalidParameterValue", error.code)
def test_extend_with_additional_schema_attributes(self):
"""
The additional schema attributes can be passed to L{Schema.extend}.
"""
result = {'id': Integer(), 'name': Unicode(), 'data': RawStr()}
errors = [APIError]
schema = Schema(name="GetStuff", parameters=[Integer("id")])
schema2 = schema.extend(name="GetStuff2",
doc="Get stuff 2",
parameters=[Unicode("scope")],
result=result,
errors=errors)
self.assertEqual("GetStuff2", schema2.name)
self.assertEqual("Get stuff 2", schema2.doc)
self.assertEqual(result, schema2.result)
self.assertEqual(set(errors), schema2.errors)
arguments, _ = schema2.extract({'id': '5', 'scope': u'foo'})
self.assertEqual(5, arguments.id)
self.assertEqual(u'foo', arguments.scope)
def test_extend_result(self):
"""
Result fields can also be extended with L{Schema.extend}.
"""
schema = Schema(result={'name': Unicode()})
schema2 = schema.extend(result={'id': Integer()})
result_structure = Structure(fields=schema2.result)
self.assertEqual({
'name': u'foo',
'id': 5
}, result_structure.coerce({
'name': u'foo',
'id': '5'
}))
def test_bundle_with_structure(self):
"""L{Schema.bundle} can bundle L{Structure}s."""
schema = Schema(parameters=[
Structure("struct",
fields={
"field1": Unicode(),
"field2": Integer()
})
])
params = schema.bundle(struct={"field1": "hi", "field2": 59})
self.assertEqual({
"struct.field1": "hi",
"struct.field2": "59"
}, params)
def test_extend_maintains_existing_attributes(self):
"""
If additional schema attributes aren't passed to L{Schema.extend}, they
stay the same.
"""
result = {'id': Integer(), 'name': Unicode(), 'data': RawStr()}
errors = [APIError]
schema = Schema(name="GetStuff",
doc="""Get the stuff.""",
parameters=[Integer("id")],
result=result,
errors=errors)
schema2 = schema.extend(parameters=[Unicode("scope")])
self.assertEqual("GetStuff", schema2.name)
self.assertEqual("Get the stuff.", schema2.doc)
self.assertEqual(result, schema2.result)
self.assertEqual(set(errors), schema2.errors)
arguments, _ = schema2.extract({'id': '5', 'scope': u'foo'})
self.assertEqual(5, arguments.id)
self.assertEqual(u'foo', arguments.scope)
def test_parse(self):
"""L{Integer.parse} converts the given raw C{value} to C{int}."""
parameter = Integer("Test")
self.assertEqual(123, parameter.parse("123"))
def test_format(self):
"""L{Integer.format} converts the given integer to a string."""
parameter = Integer("Test")
self.assertEqual("123", parameter.format(123))
def test_structure_of_list(self):
"""L{Structure}s of L{List}s are extracted properly."""
schema = Schema(Structure("foo", fields={"l": List(item=Integer())}))
arguments, _ = schema.extract({"foo.l.1": "1", "foo.l.2": "2"})
self.assertEqual([1, 2], arguments.foo.l)
class QueryAPI(Resource):
"""Base class for EC2-like query APIs.
@param registry: The L{Registry} to use to look up L{Method}s for handling
the API requests.
@param path: Optionally, the actual resource path the clients are using
when sending HTTP requests to this API, to take into account when
validating the signature. This can differ from the one in the HTTP
request we're processing in case the service sits behind a reverse
proxy, like Apache. For this works to work you have to make sure
that 'path + path_of_the_rewritten_request' equals the resource
path that clients are sending the request to.
The following class variables must be defined by sub-classes:
@ivar signature_versions: A list of allowed values for 'SignatureVersion'.
@cvar content_type: The content type to set the 'Content-Type' header to.
"""
isLeaf = True
time_format = "%Y-%m-%dT%H:%M:%SZ"
schema = Schema(
Unicode("Action"), RawStr("AWSAccessKeyId"),
Date("Timestamp", optional=True), Date("Expires", optional=True),
RawStr("Version", optional=True),
Enum("SignatureMethod", {
"HmacSHA256": "sha256",
"HmacSHA1": "sha1"
},
optional=True,
default="HmacSHA256"), Unicode("Signature"),
Integer("SignatureVersion", optional=True, default=2))
def __init__(self, registry=None, path=None):
Resource.__init__(self)
self.path = path
self.registry = registry
def get_method(self, call, *args, **kwargs):
"""Return the L{Method} instance to invoke for the given L{Call}.
@param args: Positional arguments to pass to the method constructor.
@param kwargs: Keyword arguments to pass to the method constructor.
"""
method_class = self.registry.get(call.action, call.version)
method = method_class(*args, **kwargs)
if not method.is_available():
raise APIError(
400, "InvalidAction", "The action %s is not "
"valid for this web service." % call.action)
else:
return method
def get_principal(self, access_key):
"""Return a principal object by access key.
The returned object must have C{access_key} and C{secret_key}
attributes and if the authentication succeeds, it will be
passed to the created L{Call}.
"""
raise NotImplemented("Must be implemented by subclasses")
def handle(self, request):
"""Handle an HTTP request for executing an API call.
This method authenticates the request checking its signature, and then
calls the C{execute} method, passing it a L{Call} object set with the
principal for the authenticated user and the generic parameters
extracted from the request.
@param request: The L{HTTPRequest} to handle.
"""
request.id = str(uuid4())
deferred = maybeDeferred(self._validate, request)
deferred.addCallback(self.execute)
def write_response(response):
request.setHeader("Content-Length", str(len(response)))
request.setHeader("Content-Type", self.content_type)
request.write(response)
request.finish()
return response
def write_error(failure):
if failure.check(APIError):
status = failure.value.status
# Don't log the stack traces for 4xx responses.
if status < 400 or status >= 500:
log.err(failure)
else:
log.msg("status: %s message: %s" %
(status, safe_str(failure.value)))
bytes = failure.value.response
if bytes is None:
bytes = self.dump_error(failure.value, request)
else:
log.err(failure)
bytes = safe_str(failure.value)
status = 500
request.setResponseCode(status)
request.write(bytes)
request.finish()
deferred.addCallback(write_response)
deferred.addErrback(write_error)
return deferred
def dump_error(self, error, request):
"""Serialize an error generating the response to send to the client.
@param error: The L{APIError} to format.
@param request: The request that generated the error.
"""
raise NotImplementedError("Must be implemented by subclass.")
def dump_result(self, result):
"""Serialize the result of the method invokation.
@param result: The L{Method} result to serialize.
"""
return result
def authorize(self, method, call):
"""Authorize to invoke the given L{Method} with the given L{Call}."""
def execute(self, call):
"""Execute an API L{Call}.
At this point the request has been authenticated and C{call.principal}
is set with the L{Principal} for the L{User} requesting the call.
@return: The response to write in the request for the given L{Call}.
@raises: An L{APIError} in case the execution fails, sporting an error
message the HTTP status code to return.
"""
method = self.get_method(call)
deferred = maybeDeferred(self.authorize, method, call)
deferred.addCallback(lambda _: method.invoke(call))
return deferred.addCallback(self.dump_result)
def get_utc_time(self):
"""Return a C{datetime} object with the current time in UTC."""
return datetime.now(tzutc())
def _validate(self, request):
"""Validate an L{HTTPRequest} before executing it.
The following conditions are checked:
- The request contains all the generic parameters.
- The action specified in the request is a supported one.
- The signature mechanism is a supported one.
- The provided signature matches the one calculated using the locally
stored secret access key for the user.
- The signature hasn't expired.
@return: The validated L{Call}, set with its default arguments and the
the principal of the accessing L{User}.
"""
params = dict((k, v[-1]) for k, v in request.args.iteritems())
args, rest = self.schema.extract(params)
self._validate_generic_parameters(args)
def create_call(principal):
self._validate_principal(principal, args)
self._validate_signature(request, principal, args, params)
return Call(raw_params=rest,
principal=principal,
action=args.Action,
version=args.Version,
id=request.id)
deferred = maybeDeferred(self.get_principal, args.AWSAccessKeyId)
deferred.addCallback(create_call)
return deferred
def _validate_generic_parameters(self, args):
"""Validate the generic request parameters.
@param args: Parsed schema arguments.
@raises APIError: In the following cases:
- Action is not included in C{self.actions}
- SignatureVersion is not included in C{self.signature_versions}
- Expires and Timestamp are present
- Expires is before the current time
- Timestamp is older than 15 minutes.
"""
utc_now = self.get_utc_time()
if getattr(self, "actions", None) is not None:
# Check the deprecated 'actions' attribute
if not args.Action in self.actions:
raise APIError(
400, "InvalidAction", "The action %s is not "
"valid for this web service." % args.Action)
else:
self.registry.check(args.Action, args.Version)
if not args.SignatureVersion in self.signature_versions:
raise APIError(
403, "InvalidSignature", "SignatureVersion '%s' "
"not supported" % args.SignatureVersion)
if args.Expires and args.Timestamp:
raise APIError(
400, "InvalidParameterCombination",
"The parameter Timestamp cannot be used with "
"the parameter Expires")
if args.Expires and args.Expires < utc_now:
raise APIError(
400, "RequestExpired",
"Request has expired. Expires date is %s" %
(args.Expires.strftime(self.time_format)))
if args.Timestamp and args.Timestamp + timedelta(minutes=15) < utc_now:
raise APIError(
400, "RequestExpired",
"Request has expired. Timestamp date is %s" %
(args.Timestamp.strftime(self.time_format)))
def _validate_principal(self, principal, args):
"""Validate the principal."""
if principal is None:
raise APIError(
401, "AuthFailure",
"No user with access key '%s'" % args.AWSAccessKeyId)
def _validate_signature(self, request, principal, args, params):
"""Validate the signature."""
creds = AWSCredentials(principal.access_key, principal.secret_key)
endpoint = AWSServiceEndpoint()
endpoint.set_method(request.method)
endpoint.set_canonical_host(request.getHeader("Host"))
path = request.path
if self.path is not None:
path = "%s/%s" % (self.path.rstrip("/"), path.lstrip("/"))
endpoint.set_path(path)
params.pop("Signature")
signature = Signature(creds, endpoint, params)
if signature.compute() != args.Signature:
raise APIError(
403, "SignatureDoesNotMatch",
"The request signature we calculated does not "
"match the signature you provided. Check your "
"key and signing method.")
def get_status_text(self):
"""Get the text to return when a status check is made."""
return "Query API Service"
def render_GET(self, request):
"""Handle a GET request."""
if not request.args:
request.setHeader("Content-Type", "text/plain")
return self.get_status_text()
else:
self.handle(request)
return NOT_DONE_YET
render_POST = render_GET
def test_list(self):
"""L{List}s can be extracted."""
schema = Schema(List("foo", Integer()))
arguments, _ = schema.extract({"foo.1": "1", "foo.2": "2"})
self.assertEqual([1, 2], arguments.foo)
def test_schema_field_names(self):
structure = Structure(fields={"foo": Integer()})
self.assertEqual("foo", structure.fields["foo"].name)
def test_parse(self):
"""L{Integer.parse} converts the given raw C{value} to C{int}."""
parameter = Integer("Test")
self.assertEqual(123, parameter.parse("123"))
def test_extract_with_optional(self):
"""L{Schema.extract} can handle optional parameters."""
schema = Schema(Unicode("name"), Integer("count", optional=True))
arguments, _ = schema.extract({"name": "value"})
self.assertEqual(u"value", arguments.name)
self.assertEqual(None, arguments.count)
def test_extract_with_many_arguments(self):
"""L{Schema.extract} can handle multiple parameters."""
schema = Schema(Unicode("name"), Integer("count"))
arguments, _ = schema.extract({"name": "value", "count": "123"})
self.assertEqual(u"value", arguments.name)
self.assertEqual(123, arguments.count)
def test_format(self):
"""L{Integer.format} converts the given integer to a string."""
parameter = Integer("Test")
self.assertEqual("123", parameter.format(123))