def test_extract_with_single_numbered(self):
"""
L{Schema.extract} can handle a single parameter with a numbered value.
"""
schema = Schema(Unicode("name.n"))
arguments, _ = schema.extract({"name.0": "Joe"})
self.assertEqual("Joe", arguments.name[0])
def test_extract_complex(self):
"""L{Schema} can cope with complex schemas."""
schema = Schema(
Unicode("GroupName"),
RawStr("IpPermissions.n.IpProtocol"),
Integer("IpPermissions.n.FromPort"),
Integer("IpPermissions.n.ToPort"),
Unicode("IpPermissions.n.Groups.m.UserId", optional=True),
Unicode("IpPermissions.n.Groups.m.GroupName", optional=True))
arguments, _ = schema.extract(
{"GroupName": "Foo",
"IpPermissions.1.IpProtocol": "tcp",
"IpPermissions.1.FromPort": "1234",
"IpPermissions.1.ToPort": "5678",
"IpPermissions.1.Groups.1.GroupName": "Bar",
"IpPermissions.1.Groups.2.GroupName": "Egg"})
self.assertEqual(u"Foo", arguments.GroupName)
self.assertEqual(1, len(arguments.IpPermissions))
self.assertEqual(1234, arguments.IpPermissions[0].FromPort)
self.assertEqual(5678, arguments.IpPermissions[0].ToPort)
self.assertEqual(2, len(arguments.IpPermissions[0].Groups))
self.assertEqual("Bar", arguments.IpPermissions[0].Groups[0].GroupName)
self.assertEqual("Egg", arguments.IpPermissions[0].Groups[1].GroupName)
def test_extend_maintains_existing_attributes(self):
"""
If additional schema attributes aren't passed to L{Schema.extend}, they
stay the same.
"""
result = {'id': Integer(), 'name': Unicode(), 'data': RawStr()}
errors = [APIError]
schema = Schema(
name="GetStuff",
doc="""Get the stuff.""",
parameters=[Integer("id")],
result=result,
errors=errors)
schema2 = schema.extend(parameters=[Unicode("scope")])
self.assertEqual("GetStuff", schema2.name)
self.assertEqual("Get the stuff.", schema2.doc)
self.assertEqual(result, schema2.result)
self.assertEqual(set(errors), schema2.errors)
arguments, _ = schema2.extract({'id': '5', 'scope': u'foo'})
self.assertEqual(5, arguments.id)
self.assertEqual(u'foo', arguments.scope)
def test_bundle_with_numbered_not_supplied(self):
"""
L{Schema.bundle} ignores parameters that are not present.
"""
schema = Schema(Unicode("name.n"))
params = schema.bundle()
self.assertEqual({}, params)
def test_extend_with_additional_schema_attributes(self):
"""
The additional schema attributes can be passed to L{Schema.extend}.
"""
result = {'id': Integer(), 'name': Unicode(), 'data': RawStr()}
errors = [APIError]
schema = Schema(
name="GetStuff",
parameters=[Integer("id")])
schema2 = schema.extend(
name="GetStuff2",
doc="Get stuff 2",
parameters=[Unicode("scope")],
result=result,
errors=errors)
self.assertEqual("GetStuff2", schema2.name)
self.assertEqual("Get stuff 2", schema2.doc)
self.assertEqual(result, schema2.result)
self.assertEqual(set(errors), schema2.errors)
arguments, _ = schema2.extract({'id': '5', 'scope': u'foo'})
self.assertEqual(5, arguments.id)
self.assertEqual(u'foo', arguments.scope)
def test_bundle_with_empty_numbered(self):
"""
L{Schema.bundle} correctly handles an empty numbered arguments list.
"""
schema = Schema(Unicode("name.n"))
params = schema.bundle(name=[])
self.assertEqual({}, params)
def test_add_single_extra_schema_item(self):
"""New Parameters can be added to the Schema."""
schema = Schema(Unicode("name"))
schema = schema.extend(Unicode("computer"))
arguments, _ = schema.extract({"name": "value", "computer": "testing"})
self.assertEqual(u"value", arguments.name)
self.assertEqual("testing", arguments.computer)
def test_bundle_with_numbered(self):
"""
L{Schema.bundle} correctly handles numbered arguments.
"""
schema = Schema(Unicode("name.n"))
params = schema.bundle(name=["foo", "bar"])
self.assertEqual({"name.1": "foo", "name.2": "bar"}, params)
def test_optional_list(self):
"""
The default value of an optional L{List} is C{[]}.
"""
schema = Schema(List("names", Unicode(), optional=True))
arguments, _ = schema.extract({})
self.assertEqual([], arguments.names)
def test_extend_errors(self):
"""
Errors can be extended with L{Schema.extend}.
"""
schema = Schema(parameters=[], errors=[APIError])
schema2 = schema.extend(errors=[ZeroDivisionError])
self.assertEqual(set([APIError, ZeroDivisionError]), schema2.errors)
def test_schema_conversion_optional_list(self):
"""
Backwards-compatibility conversions maintains optional-ness of lists.
"""
schema = Schema(Unicode("foos.N", optional=True))
arguments, _ = schema.extract({})
self.assertEqual([], arguments.foos)
def test_bundle_with_multiple(self):
"""
L{Schema.bundle} correctly handles multiple arguments.
"""
schema = Schema(Unicode("name.n"), Integer("count"))
params = schema.bundle(name=["Foo", "Bar"], count=123)
self.assertEqual({"name.1": "Foo", "name.2": "Bar", "count": "123"},
params)
def test_bundle(self):
"""
L{Schema.bundle} returns a dictionary of raw parameters that
can be used for an EC2-style query.
"""
schema = Schema(Unicode("name"))
params = schema.bundle(name="foo")
self.assertEqual({"name": "foo"}, params)
def test_extract(self):
"""
L{Schema.extract} returns an L{Argument} object whose attributes are
the arguments extracted from the given C{request}, as specified.
"""
schema = Schema(Unicode("name"))
arguments, _ = schema.extract({"name": "value"})
self.assertEqual("value", arguments.name)
def test_extract_with_single_numbered(self):
"""
L{Schema.extract} can handle an un-numbered argument passed in to a
numbered parameter.
"""
schema = Schema(Unicode("name.n"))
arguments, _ = schema.extract({"name": "Joe"})
self.assertEqual("Joe", arguments.name[0])
def test_extract_with_numbered(self):
"""
L{Schema.extract} can handle parameters with numbered values.
"""
schema = Schema(Unicode("name.n"))
arguments, _ = schema.extract({"name.0": "Joe", "name.1": "Tom"})
self.assertEqual("Joe", arguments.name[0])
self.assertEqual("Tom", arguments.name[1])
def test_extract_structure_with_optional(self):
"""L{Schema.extract} can handle optional parameters."""
schema = Schema(
Structure(
"struct",
fields={"name": Unicode(optional=True, default="radix")}))
arguments, _ = schema.extract({"struct": {}})
self.assertEqual(u"radix", arguments.struct.name)
def test_extract_with_rest(self):
"""
L{Schema.extract} stores unknown parameters in the 'rest' return
dictionary.
"""
schema = Schema()
_, rest = schema.extract({"name": "value"})
self.assertEqual(rest, {"name": "value"})
def test_extract_with_non_numbered_template(self):
"""
L{Schema.extract} accepts a single numbered argument even if the
associated template is not numbered.
"""
schema = Schema(Unicode("name"))
arguments, _ = schema.extract({"name.1": "foo"})
self.assertEqual("foo", arguments.name)
def test_extract_with_mixed(self):
"""
L{Schema.extract} stores in the rest result all numbered parameters
given without an index.
"""
schema = Schema(Unicode("name.n"))
_, rest = schema.extract({"name": "foo", "name.1": "bar"})
self.assertEqual(rest, {"name": "foo"})
def test_bundle_with_arguments(self):
"""L{Schema.bundle} can bundle L{Arguments} too."""
schema = Schema(Unicode("name.n"), Integer("count"))
arguments = Arguments({"name": Arguments({1: "Foo", 7: "Bar"}),
"count": 123})
params = schema.bundle(arguments)
self.assertEqual({"name.1": "Foo", "name.7": "Bar", "count": "123"},
params)
def test_structure(self):
"""
L{Schema}s with L{Structure} parameters can have arguments extracted.
"""
schema = Schema(Structure("foo", {"a": Integer(), "b": Integer()}))
arguments, _ = schema.extract({"foo.a": "1", "foo.b": "2"})
self.assertEqual(1, arguments.foo.a)
self.assertEqual(2, arguments.foo.b)
def test_default_list(self):
"""
The default of a L{List} can be specified as a list.
"""
schema = Schema(List("names", Unicode(), optional=True,
default=[u"foo", u"bar"]))
arguments, _ = schema.extract({})
self.assertEqual([u"foo", u"bar"], arguments.names)
def test_bundle_with_two_numbered(self):
"""
L{Schema.bundle} can bundle multiple numbered lists.
"""
schema = Schema(Unicode("names.n"), Unicode("things.n"))
params = schema.bundle(names=["foo", "bar"], things=["baz", "quux"])
self.assertEqual({"names.1": "foo", "names.2": "bar",
"things.1": "baz", "things.2": "quux"},
params)
def test_schema_conversion_list(self):
"""
Backwards-compatibility conversion maintains the name of lists.
"""
schema = Schema(Unicode("foos.N"))
parameters = schema.get_parameters()
self.assertEqual(1, len(parameters))
self.assertTrue(isinstance(parameters[0], List))
self.assertEqual("foos", parameters[0].name)
def test_list_of_list(self):
"""L{List}s can be nested."""
schema = Schema(List("foo", List(item=Unicode())))
arguments, _ = schema.extract(
{"foo.1.1": "first-first", "foo.1.2": "first-second",
"foo.2.1": "second-first", "foo.2.2": "second-second"})
self.assertEqual([["first-first", "first-second"],
["second-first", "second-second"]],
arguments.foo)
def test_extract_with_goofy_numbered(self):
"""
L{Schema.extract} only uses the relative values of indices to determine
the index in the resultant list.
"""
schema = Schema(Unicode("name.n"))
arguments, _ = schema.extract({"name.5": "Joe", "name.10": "Tom"})
self.assertEqual("Joe", arguments.name[0])
self.assertEqual("Tom", arguments.name[1])
def test_add_extra_schema_items(self):
"""A list of new Parameters can be added to the Schema."""
schema = Schema(Unicode("name"))
schema = schema.extend(Unicode("computer"), Integer("count"))
arguments, _ = schema.extract({"name": "value", "computer": "testing",
"count": "5"})
self.assertEqual(u"value", arguments.name)
self.assertEqual("testing", arguments.computer)
self.assertEqual(5, arguments.count)
def test_bundle_with_structure(self):
"""L{Schema.bundle} can bundle L{Structure}s."""
schema = Schema(
parameters=[
Structure("struct", fields={"field1": Unicode(),
"field2": Integer()})])
params = schema.bundle(struct={"field1": "hi", "field2": 59})
self.assertEqual({"struct.field1": "hi", "struct.field2": "59"},
params)
def test_extend_maintains_parameter_order(self):
"""
Extending a schema with additional parameters puts the new parameters
at the end.
"""
schema = Schema(parameters=[Unicode("name"), Unicode("value")])
schema2 = schema.extend(parameters=[Integer("foo"), Unicode("index")])
self.assertEqual(["name", "value", "foo", "index"],
[p.name for p in schema2.get_parameters()])
def test_bundle_with_none(self):
"""L{None} values are discarded in L{Schema.bundle}."""
schema = Schema(Unicode("name.n", optional=True))
params = schema.bundle(name=None)
self.assertEqual({}, params)
def test_extract_with_optional(self):
"""L{Schema.extract} can handle optional parameters."""
schema = Schema(Unicode("name"), Integer("count", optional=True))
arguments, _ = schema.extract({"name": "value"})
self.assertEqual(u"value", arguments.name)
self.assertEqual(None, arguments.count)
def test_extract_with_many_arguments(self):
"""L{Schema.extract} can handle multiple parameters."""
schema = Schema(Unicode("name"), Integer("count"))
arguments, _ = schema.extract({"name": "value", "count": "123"})
self.assertEqual(u"value", arguments.name)
self.assertEqual(123, arguments.count)
def test_extract_with_nested_rest(self):
schema = Schema()
_, rest = schema.extract({"foo.1.bar": "hey", "foo.2.baz": "there"})
self.assertEqual({"foo.1.bar": "hey", "foo.2.baz": "there"}, rest)
class QueryAPI(Resource):
"""Base class for EC2-like query APIs.
@param registry: The L{Registry} to use to look up L{Method}s for handling
the API requests.
@param path: Optionally, the actual resource path the clients are using
when sending HTTP requests to this API, to take into account when
validating the signature. This can differ from the one in the HTTP
request we're processing in case the service sits behind a reverse
proxy, like Apache. For this works to work you have to make sure
that 'path + path_of_the_rewritten_request' equals the resource
path that clients are sending the request to.
The following class variables must be defined by sub-classes:
@ivar signature_versions: A list of allowed values for 'SignatureVersion'.
@cvar content_type: The content type to set the 'Content-Type' header to.
"""
isLeaf = True
time_format = "%Y-%m-%dT%H:%M:%SZ"
schema = Schema(
Unicode("Action"), RawStr("AWSAccessKeyId"),
Date("Timestamp", optional=True), Date("Expires", optional=True),
RawStr("Version", optional=True),
Enum("SignatureMethod", {
"HmacSHA256": "sha256",
"HmacSHA1": "sha1"
},
optional=True,
default="HmacSHA256"), Unicode("Signature"),
Integer("SignatureVersion", optional=True, default=2))
def __init__(self, registry=None, path=None):
Resource.__init__(self)
self.path = path
self.registry = registry
def get_method(self, call, *args, **kwargs):
"""Return the L{Method} instance to invoke for the given L{Call}.
@param args: Positional arguments to pass to the method constructor.
@param kwargs: Keyword arguments to pass to the method constructor.
"""
method_class = self.registry.get(call.action, call.version)
method = method_class(*args, **kwargs)
if not method.is_available():
raise APIError(
400, "InvalidAction", "The action %s is not "
"valid for this web service." % call.action)
else:
return method
def get_principal(self, access_key):
"""Return a principal object by access key.
The returned object must have C{access_key} and C{secret_key}
attributes and if the authentication succeeds, it will be
passed to the created L{Call}.
"""
raise NotImplemented("Must be implemented by subclasses")
def handle(self, request):
"""Handle an HTTP request for executing an API call.
This method authenticates the request checking its signature, and then
calls the C{execute} method, passing it a L{Call} object set with the
principal for the authenticated user and the generic parameters
extracted from the request.
@param request: The L{HTTPRequest} to handle.
"""
request.id = str(uuid4())
deferred = maybeDeferred(self._validate, request)
deferred.addCallback(self.execute)
def write_response(response):
request.setHeader("Content-Length", str(len(response)))
request.setHeader("Content-Type", self.content_type)
request.write(response)
request.finish()
return response
def write_error(failure):
if failure.check(APIError):
status = failure.value.status
# Don't log the stack traces for 4xx responses.
if status < 400 or status >= 500:
log.err(failure)
else:
log.msg("status: %s message: %s" %
(status, safe_str(failure.value)))
bytes = failure.value.response
if bytes is None:
bytes = self.dump_error(failure.value, request)
else:
log.err(failure)
bytes = safe_str(failure.value)
status = 500
request.setResponseCode(status)
request.write(bytes)
request.finish()
deferred.addCallback(write_response)
deferred.addErrback(write_error)
return deferred
def dump_error(self, error, request):
"""Serialize an error generating the response to send to the client.
@param error: The L{APIError} to format.
@param request: The request that generated the error.
"""
raise NotImplementedError("Must be implemented by subclass.")
def dump_result(self, result):
"""Serialize the result of the method invokation.
@param result: The L{Method} result to serialize.
"""
return result
def authorize(self, method, call):
"""Authorize to invoke the given L{Method} with the given L{Call}."""
def execute(self, call):
"""Execute an API L{Call}.
At this point the request has been authenticated and C{call.principal}
is set with the L{Principal} for the L{User} requesting the call.
@return: The response to write in the request for the given L{Call}.
@raises: An L{APIError} in case the execution fails, sporting an error
message the HTTP status code to return.
"""
method = self.get_method(call)
deferred = maybeDeferred(self.authorize, method, call)
deferred.addCallback(lambda _: method.invoke(call))
return deferred.addCallback(self.dump_result)
def get_utc_time(self):
"""Return a C{datetime} object with the current time in UTC."""
return datetime.now(tzutc())
def _validate(self, request):
"""Validate an L{HTTPRequest} before executing it.
The following conditions are checked:
- The request contains all the generic parameters.
- The action specified in the request is a supported one.
- The signature mechanism is a supported one.
- The provided signature matches the one calculated using the locally
stored secret access key for the user.
- The signature hasn't expired.
@return: The validated L{Call}, set with its default arguments and the
the principal of the accessing L{User}.
"""
params = dict((k, v[-1]) for k, v in request.args.iteritems())
args, rest = self.schema.extract(params)
self._validate_generic_parameters(args)
def create_call(principal):
self._validate_principal(principal, args)
self._validate_signature(request, principal, args, params)
return Call(raw_params=rest,
principal=principal,
action=args.Action,
version=args.Version,
id=request.id)
deferred = maybeDeferred(self.get_principal, args.AWSAccessKeyId)
deferred.addCallback(create_call)
return deferred
def _validate_generic_parameters(self, args):
"""Validate the generic request parameters.
@param args: Parsed schema arguments.
@raises APIError: In the following cases:
- Action is not included in C{self.actions}
- SignatureVersion is not included in C{self.signature_versions}
- Expires and Timestamp are present
- Expires is before the current time
- Timestamp is older than 15 minutes.
"""
utc_now = self.get_utc_time()
if getattr(self, "actions", None) is not None:
# Check the deprecated 'actions' attribute
if not args.Action in self.actions:
raise APIError(
400, "InvalidAction", "The action %s is not "
"valid for this web service." % args.Action)
else:
self.registry.check(args.Action, args.Version)
if not args.SignatureVersion in self.signature_versions:
raise APIError(
403, "InvalidSignature", "SignatureVersion '%s' "
"not supported" % args.SignatureVersion)
if args.Expires and args.Timestamp:
raise APIError(
400, "InvalidParameterCombination",
"The parameter Timestamp cannot be used with "
"the parameter Expires")
if args.Expires and args.Expires < utc_now:
raise APIError(
400, "RequestExpired",
"Request has expired. Expires date is %s" %
(args.Expires.strftime(self.time_format)))
if args.Timestamp and args.Timestamp + timedelta(minutes=15) < utc_now:
raise APIError(
400, "RequestExpired",
"Request has expired. Timestamp date is %s" %
(args.Timestamp.strftime(self.time_format)))
def _validate_principal(self, principal, args):
"""Validate the principal."""
if principal is None:
raise APIError(
401, "AuthFailure",
"No user with access key '%s'" % args.AWSAccessKeyId)
def _validate_signature(self, request, principal, args, params):
"""Validate the signature."""
creds = AWSCredentials(principal.access_key, principal.secret_key)
endpoint = AWSServiceEndpoint()
endpoint.set_method(request.method)
endpoint.set_canonical_host(request.getHeader("Host"))
path = request.path
if self.path is not None:
path = "%s/%s" % (self.path.rstrip("/"), path.lstrip("/"))
endpoint.set_path(path)
params.pop("Signature")
signature = Signature(creds, endpoint, params)
if signature.compute() != args.Signature:
raise APIError(
403, "SignatureDoesNotMatch",
"The request signature we calculated does not "
"match the signature you provided. Check your "
"key and signing method.")
def get_status_text(self):
"""Get the text to return when a status check is made."""
return "Query API Service"
def render_GET(self, request):
"""Handle a GET request."""
if not request.args:
request.setHeader("Content-Type", "text/plain")
return self.get_status_text()
else:
self.handle(request)
return NOT_DONE_YET
render_POST = render_GET
def test_bundle_with_list(self):
"""L{Schema.bundle} can bundle L{List}s."""
schema = Schema(parameters=[List("things", item=Unicode())])
params = schema.bundle(things=["foo", "bar"])
self.assertEqual({"things.1": "foo", "things.2": "bar"}, params)
def test_structure_of_list(self):
"""L{Structure}s of L{List}s are extracted properly."""
schema = Schema(Structure("foo", fields={"l": List(item=Integer())}))
arguments, _ = schema.extract({"foo.l.1": "1", "foo.l.2": "2"})
self.assertEqual([1, 2], arguments.foo.l)
def test_bundle_with_list_with_arguments(self):
"""L{Schema.bundle} can bundle L{List}s (specified as L{Arguments})."""
schema = Schema(parameters=[List("things", item=Unicode())])
params = schema.bundle(things=Arguments({1: "foo", 2: "bar"}))
self.assertEqual({"things.1": "foo", "things.2": "bar"}, params)
def test_list(self):
"""L{List}s can be extracted."""
schema = Schema(List("foo", Integer()))
arguments, _ = schema.extract({"foo.1": "1", "foo.2": "2"})
self.assertEqual([1, 2], arguments.foo)
