def get_system_account_token(system_service_account, scopes):
"""Returns an access token to use on bots when calling internal services.
"Internal services" are (loosely) everything that is needed for correct
functioning of the internal guts of the bot (at least Isolate and CIPD).
Assumes authorization checks have been made already. If the bot is not
configured to use service account returns ('none', None). If the bot is
configured to use whatever it is using when calling Swarming itself, returns
('bot', None).
Otherwise returns (<email>, AccessToken with valid token for <email>).
Args:
system_service_account: whatever is specified in bots.cfg for the bot.
scopes: list of requested OAuth scopes.
Returns:
(<service account email> or 'bot' or 'none', AccessToken or None).
Raises:
PermissionError if the usage of the account is forbidden.
MisconfigurationError if the service account is misconfigured.
InternalError if the RPC fails unexpectedly.
"""
if not system_service_account:
return 'none', None # no auth is configured
if system_service_account == 'bot':
return 'bot', None # the bot should use tokens provided by local hooks
# Attempt to mint a token (or grab an existing one from cache).
try:
blob, expiry = auth.get_access_token(
scopes=scopes,
act_as=system_service_account,
min_lifetime_sec=MIN_TOKEN_LIFETIME_SEC)
except auth.AccessTokenError as exc:
if exc.transient:
raise InternalError(exc.message)
raise MisconfigurationError(exc.message)
assert isinstance(blob, basestring)
assert isinstance(expiry, (int, long, float))
token = AccessToken(blob, int(expiry))
_check_and_log_token('bot associated', system_service_account, token)
return system_service_account, token
def patched_get_token_async(self, refresh=False):
fut = ndb.Future()
fut.set_result(
auth.get_access_token(self.scopes, service_account_key)[0])
return fut
def patched_get_token_async(self, refresh=False): fut = ndb.Future() fut.set_result(auth.get_access_token(self.scopes, service_account_key)[0]) return fut
def request_async(url,
method='GET',
payload=None,
params=None,
headers=None,
scopes=None,
service_account_key=None,
delegation_token=None,
deadline=None,
max_attempts=None):
"""Sends a REST API request, returns raw unparsed response.
Retries the request on transient errors for up to |max_attempts| times.
Args:
url: url to send the request to.
method: HTTP method to use, e.g. GET, POST, PUT.
payload: raw data to put in the request body.
params: dict with query GET parameters (i.e. ?key=value&key=value).
headers: additional request headers.
scopes: OAuth2 scopes for the access token (ok skip auth if None).
service_account_key: auth.ServiceAccountKey with credentials.
delegation_token: delegation token returned by auth.delegate.
deadline: deadline for a single attempt (10 sec by default).
max_attempts: how many times to retry on errors (4 times by default).
Returns:
Buffer with raw response.
Raises:
NotFoundError on 404 response.
AuthError on 401 or 403 response.
Error on any other non-transient error.
"""
deadline = 10 if deadline is None else deadline
max_attempts = 4 if max_attempts is None else max_attempts
if utils.is_local_dev_server():
protocols = ('http://', 'https://')
else:
protocols = ('https://', )
assert url.startswith(protocols) and '?' not in url, url
if params:
url += '?' + urllib.urlencode(params)
if scopes:
access_token = auth.get_access_token(scopes, service_account_key)[0]
headers = (headers or {}).copy()
headers['Authorization'] = 'Bearer %s' % access_token
if delegation_token:
if isinstance(delegation_token, auth.DelegationToken):
delegation_token = delegation_token.token
assert isinstance(delegation_token, basestring)
headers[delegation.HTTP_HEADER] = delegation_token
if payload is not None:
assert isinstance(payload, str), type(payload)
assert method in ('CREATE', 'POST', 'PUT'), method
attempt = 0
response = None
last_status_code = None
while attempt < max_attempts:
if attempt:
logging.info('Retrying...')
attempt += 1
logging.info('%s %s', method, url)
try:
response = yield urlfetch_async(url=url,
payload=payload,
method=method,
headers=headers or {},
follow_redirects=False,
deadline=deadline,
validate_certificate=True)
except (apiproxy_errors.DeadlineExceededError, urlfetch.Error) as e:
# Transient network error or URL fetch service RPC deadline.
logging.warning('%s %s failed: %s', method, url, e)
continue
last_status_code = response.status_code
# Transient error on the other side.
if is_transient_error(response, url):
logging.warning('%s %s failed with HTTP %d: %r', method, url,
response.status_code, response.content)
continue
# Non-transient error.
if 300 <= response.status_code < 500:
logging.warning('%s %s failed with HTTP %d: %r', method, url,
response.status_code, response.content)
raise _error_class_for_status(response.status_code)(
'Failed to call %s: HTTP %d' % (url, response.status_code),
response.status_code, response.content)
# Success. Beware of large responses.
if len(response.content) > 1024 * 1024:
logging.warning('Response size: %.1f KiB',
len(response.content) / 1024.0)
raise ndb.Return(response.content)
raise _error_class_for_status(last_status_code)(
'Failed to call %s after %d attempts' % (url, max_attempts),
response.status_code if response else None,
response.content if response else None)
def request_async(
url,
method='GET',
payload=None,
params=None,
headers=None,
scopes=None,
service_account_key=None,
deadline=None,
max_attempts=None):
"""Sends a REST API request, returns raw unparsed response.
Retries the request on transient errors for up to |max_attempts| times.
Args:
url: url to send the request to.
method: HTTP method to use, e.g. GET, POST, PUT.
payload: raw data to put in the request body.
params: dict with query GET parameters (i.e. ?key=value&key=value).
headers: additional request headers.
scopes: OAuth2 scopes for the access token (ok skip auth if None).
service_account_key: auth.ServiceAccountKey with credentials.
deadline: deadline for a single attempt (10 sec by default).
max_attempts: how many times to retry on errors (4 times by default).
Returns:
Buffer with raw response.
Raises:
NotFoundError on 404 response.
AuthError on 401 or 403 response.
Error on any other non-transient error.
"""
deadline = 10 if deadline is None else deadline
max_attempts = 4 if max_attempts is None else max_attempts
if utils.is_local_dev_server():
protocols = ('http://', 'https://')
else:
protocols = ('https://',)
assert url.startswith(protocols) and '?' not in url, url
if params:
url += '?' + urllib.urlencode(params)
if scopes:
access_token = auth.get_access_token(scopes, service_account_key)[0]
headers = (headers or {}).copy()
headers['Authorization'] = 'Bearer %s' % access_token
if payload is not None:
assert isinstance(payload, str), type(payload)
assert method in ('CREATE', 'POST', 'PUT'), method
attempt = 0
while attempt < max_attempts:
if attempt:
logging.info('Retrying...')
attempt += 1
logging.info('%s %s', method, url)
try:
response = yield urlfetch_async(
url=url,
payload=payload,
method=method,
headers=headers,
follow_redirects=False,
deadline=deadline,
validate_certificate=True)
except urlfetch.Error as e:
# Transient network error.
logging.warning('%s %s failed: %s', method, url, e)
continue
# Transient error on the other side.
if response.status_code >= 500 or response.status_code == 408:
logging.warning(
'%s %s failed with HTTP %s: %s',
method, url, response.status_code, response.content)
continue
# Non-transient error.
if 300 <= response.status_code < 500:
logging.warning(
'%s %s failed with HTTP %s: %s',
method, url, response.status_code, response.content)
cls = Error
if response.status_code == 404:
cls = NotFoundError
elif response.status_code in (401, 403):
cls = AuthError
raise cls(
'Failed to call %s: HTTP %d' % (url, response.status_code),
response.status_code, response.content)
# Success. Beware of large responses.
if len(response.content) > 1024 * 1024:
logging.warning('Response size: %.1f KiB', len(response.content) / 1024.0)
raise ndb.Return(response.content)
raise Error(
'Failed to call %s after %d attempts' % (url, max_attempts),
response.status_code, response.content)
