def exponential_backoff(attempt_num):
"""Returns an exponential backoff value in seconds."""
assert attempt_num >= 0
if random.random() < _PROBABILITY_OF_QUICK_COMEBACK:
# Randomly ask the bot to return quickly.
return 1.0
# Enforces more frequent polls on staging.
max_wait = 3. if utils.is_dev() else 60.
return min(max_wait, math.pow(1.5, min(attempt_num, 10) + 1))
def get_dev_config():
"""Returns an instance of AuthDevConfig (possibly uninitialized).
Asserts that it is used only on dev instance.
"""
assert utils.is_local_dev_server() or utils.is_dev()
k = ndb.Key('AuthDevConfig', 'dev_config')
e = k.get()
if not e:
logging.warning('Initializing AuthDevConfig entity')
e = AuthDevConfig(key=k)
e.put() # there's a race condition here, but we don't care
return e
def dev_oauth_authentication(header, token_info_endpoint, suffix=''):
"""OAuth2 based authentication via URL Fetch to the token info endpoint.
This is slow and ignores client_id whitelist. Must be used only in
a development environment.
Returns:
Identity of the caller in case the request was successfully validated.
Raises:
AuthenticationError in case access token is missing or invalid.
AuthorizationError in case the token is not trusted.
"""
assert utils.is_local_dev_server() or utils.is_dev()
header = header.split(' ', 1)
if len(header) != 2 or header[0] not in ('OAuth', 'Bearer'):
raise AuthenticationError('Invalid authorization header')
# Adapted from endpoints/users_id_tokens.py, _set_bearer_user_vars_local.
logging.info('Using dev token info endpoint %s', token_info_endpoint)
result = urlfetch.fetch(
url='%s?%s' %
(token_info_endpoint, urllib.urlencode({'access_token': header[1]})),
follow_redirects=False,
validate_certificate=True)
if result.status_code != 200:
try:
error = json.loads(result.content)['error_description']
except (KeyError, ValueError):
error = repr(result.content)
raise AuthenticationError('Failed to validate the token: %s' % error)
token_info = json.loads(result.content)
if 'email' not in token_info:
raise AuthorizationError('Token doesn\'t include an email address')
if not token_info.get('verified_email'):
raise AuthorizationError('Token email isn\'t verified')
email = token_info['email'] + suffix
try:
return model.Identity(model.IDENTITY_USER, email)
except ValueError:
raise AuthorizationError('Unsupported user email: %s' % email)
def endpoints_api(name,
version,
auth_level=None,
allowed_client_ids=None,
**kwargs):
"""Same as @endpoints.api but tweaks default auth related properties.
By default API marked with this decorator will use same authentication scheme
as non-endpoints request handlers (i.e. fetch a whitelist of OAuth client_id's
from the datastore, recognize service accounts, etc.), disabling client_id
checks performed by Cloud Endpoints frontend (and doing them on the backend,
see 'initialize_auth' below).
Using service accounts with vanilla Cloud Endpoints auth is somewhat painful:
every service account should be whitelisted in the 'allowed_client_ids' list
in the source code of the application (when calling @endpoints.api). By moving
client_id checks to the backend we can support saner logic.
"""
# 'audiences' is used with id_token auth, it's not supported yet.
assert 'audiences' not in kwargs, 'Not supported'
# On prod, make sure Cloud Endpoints frontend validates OAuth tokens for us.
# On dev instances we will validate them ourselves to support custom token
# validation endpoint.
if auth_level is not None:
if utils.is_local_dev_server() or utils.is_dev():
# AUTH_LEVEL.NONE: Frontend authentication will be skipped. If
# authentication is desired, it will need to be performed by the backend.
auth_level = endpoints.AUTH_LEVEL.NONE
else:
# AUTH_LEVEL.OPTIONAL: Authentication is optional. If authentication
# credentials are supplied they must be valid. Backend will be called if
# the request contains valid authentication credentials or no
# authentication credentials.
auth_level = endpoints.AUTH_LEVEL.OPTIONAL
# We love API Explorer.
if allowed_client_ids is None:
allowed_client_ids = endpoints.SKIP_CLIENT_ID_CHECK
if allowed_client_ids != endpoints.SKIP_CLIENT_ID_CHECK:
allowed_client_ids = sorted(
set(allowed_client_ids) | set([endpoints.API_EXPLORER_CLIENT_ID]))
# Someone was looking for job security here:
# - api() returns _ApiDecorator class instance.
# - One of the following is done:
# - _ApiDecorator.__call__() is called with the remote.Service class as
# argument.
# - api_class() is explicitly called which returns a function, which is then
# called with the remote.Service class as argument.
api_decorator = endpoints.api(name,
version,
auth_level=auth_level,
allowed_client_ids=allowed_client_ids,
**kwargs)
def fn(cls):
if not cls.all_remote_methods():
raise TypeError(
'Service %s must have at least one auth.endpoints_method method'
% name)
for method, func in cls.all_remote_methods().items():
if func and not api.is_decorated(
func.remote._RemoteMethodInfo__method):
raise TypeError(
'Method \'%s\' of \'%s\' is not protected by @require or @public '
'decorator' % (method, name))
return cls
# Monkey patch api_decorator to make 'api_class' to return wrapped decorator.
orig = api_decorator.api_class
def patched_api_class(*args, **kwargs):
wrapper = orig(*args, **kwargs)
return lambda cls: fn(wrapper(cls))
api_decorator.api_class = patched_api_class
return api_decorator
def check_oauth_access_token(headers):
"""Verifies the access token of the current request.
This function uses slightly different strategies for prod, dev and local
environments:
* In prod it always require real OAuth2 tokens, validated by GAE OAuth2 API.
* On local devserver it uses URL Fetch and prod token info endpoint.
* On '-dev' instances or on dev server it can also fallback to a custom
token info endpoint, defined in AuthDevConfig datastore entity. This is
useful to "stub" authentication when running integration or load tests.
In addition to checking the correctness of OAuth token, this function also
verifies that the client_id associated with the token is whitelisted in the
auth config.
The client_id check is skipped on the local devserver or when using custom
token info endpoint (e.g. on '-dev' instances).
Args:
headers: a dict with request headers.
Returns:
Identity of the caller in case the request was successfully validated.
Always 'user:...', never anonymous.
Raises:
AuthenticationError in case the access token is invalid.
AuthorizationError in case the access token is not allowed.
"""
header = headers.get('Authorization')
if not header:
raise AuthenticationError('No "Authorization" header')
# Non-development instances always use real OAuth API.
if not utils.is_local_dev_server() and not utils.is_dev():
return extract_oauth_caller_identity()
# OAuth2 library is mocked on dev server to return some nonsense. Use (slow,
# but real) OAuth2 API endpoint instead to validate access_token. It is also
# what Cloud Endpoints do on a local server.
if utils.is_local_dev_server():
auth_call = lambda: dev_oauth_authentication(header,
TOKEN_INFO_ENDPOINT)
else:
auth_call = extract_oauth_caller_identity
# Do not fallback to custom endpoint if not configured. This call also has a
# side effect of initializing AuthDevConfig entity in the datastore, to make
# it editable in Datastore UI.
cfg = model.get_dev_config()
if not cfg.token_info_endpoint:
return auth_call()
# Try the real call first, then fallback to the custom validation endpoint.
try:
return auth_call()
except AuthenticationError:
ident = dev_oauth_authentication(header, cfg.token_info_endpoint,
'.dev')
logging.warning('Authenticated as dev account: %s', ident.to_bytes())
return ident
# Mask to TaskRequest key ids so they become decreasing numbers.
TASK_REQUEST_KEY_ID_MASK = int(2L**63 - 1)
# The production server must handle up to 1000 task requests per second. The
# number of root entities must be a few orders of magnitude higher. The goal is
# to almost completely get rid of transactions conflicts. This means that the
# probability of two transactions happening on the same shard must be very low.
# This relates to number of transactions per second * seconds per transaction /
# number of shard.
#
# Intentionally starve the canary server by using only 16³=4096 root entities.
# This will cause mild transaction conflicts during load tests. On the
# production server, use 16**6 (~16 million) root entities to reduce the number
# of transaction conflict.
# TODO(maruel): Remove support 2015-02-01.
DEPRECATED_SHARDING_LEVEL = 3 if utils.is_dev() else 6
### Entities relationships.
def request_key_to_result_summary_key(request_key):
"""Returns the TaskResultSummary ndb.Key for this TaskRequest.key."""
assert request_key.kind() == 'TaskRequest', request_key
assert request_key.integer_id(), request_key
return ndb.Key('TaskResultSummary', 1, parent=request_key)
def result_summary_key_to_request_key(result_summary_key):
"""Returns the TaskRequest ndb.Key for this TaskResultSummmary key."""
assert result_summary_key.kind() == 'TaskResultSummary', result_summary_key
return result_summary_key.parent()
def cfg_path():
"""Returns a path of the project config file to read."""
if utils.is_local_dev_server() or utils.is_dev():
return 'realms-dev.cfg'
return 'realms.cfg'