def _load_local_auth():
"""Returns a LocalAuthParameters tuple from LUCI_CONTEXT.
Returns:
LocalAuthParameters for connecting to a local auth server.
Raises:
BadLuciContextParameters if file is missing or not valid.
"""
data = luci_context.read('local_auth')
if data is None:
raise BadLuciContextParameters('Missing "local_auth" in LUCI_CONTEXT')
try:
return LocalAuthParameters(
rpc_port=int(data['rpc_port']),
secret=str(data['secret']),
accounts=[
LocalAuthAccount(id=acc['id'])
for acc in data.get('accounts', [])
],
default_account_id=data.get('default_account_id'))
except (ValueError, KeyError):
data[
'secret'] = '...' # note: 'data' is a copy, it's fine to mutate it
raise BadLuciContextParameters(
'Invalid "local_auth" section in LUCI_CONTEXT: %r' % (data, ))
def test_validation(self):
def token_gen(_account_id, _scopes):
self.fail('must not be called')
with local_auth_server(token_gen, 'acc_1'):
ctx = luci_context.read('local_auth')
def must_fail(body, err, code):
r = requests.post(
url='http://127.0.0.1:%d/rpc/LuciLocalAuthService.GetOAuthToken' %
ctx['rpc_port'],
data=json.dumps(body),
headers={'Content-Type': 'application/json'})
self.assertEqual(code, r.status_code)
self.assertIn(err, r.text)
cases = [
# account_id
({}, '"account_id" is required', 400),
({'account_id': 123}, '"account_id" must be a string', 400),
# scopes
({'account_id': 'acc_1'}, '"scopes" is required', 400),
({'account_id': 'acc_1', 'scopes': []}, '"scopes" is required', 400),
(
{'account_id': 'acc_1', 'scopes': 'abc'},
'"scopes" must be a list of strings',
400,
),
(
{'account_id': 'acc_1', 'scopes': [1]},
'"scopes" must be a list of strings',
400,
),
# secret
({'account_id': 'acc_1', 'scopes': ['a']}, '"secret" is required', 400),
(
{'account_id': 'acc_1', 'scopes': ['a'], 'secret': 123},
'"secret" must be a string',
400,
),
(
{'account_id': 'acc_1', 'scopes': ['a'], 'secret': 'abc'},
'Invalid "secret"',
403,
),
# The account is known.
(
{'account_id': 'zzz', 'scopes': ['a'], 'secret': ctx['secret']},
'Unrecognized account ID',
404,
),
]
for body, err, code in cases:
must_fail(body, err, code)
def must_fail(err, body, code=400):
ctx = luci_context.read('local_auth')
r = requests.post(
url=
'http://127.0.0.1:%d/rpc/LuciLocalAuthService.GetOAuthToken'
% ctx['rpc_port'],
data=json.dumps(body),
headers={'Content-Type': 'application/json'})
self.assertEqual(code, r.status_code)
self.assertIn(err, r.text)
def call_rpc(scopes):
ctx = luci_context.read('local_auth')
r = requests.post(
url='http://127.0.0.1:%d/rpc/LuciLocalAuthService.GetOAuthToken' %
ctx['rpc_port'],
data=json.dumps({
'scopes': scopes,
'secret': ctx['secret'],
}),
headers={'Content-Type': 'application/json'})
return r.json()
def test_http_level_errors(self):
def token_gen(_account_id, _scopes):
self.fail('must not be called')
with local_auth_server(token_gen, 'acc_1'):
# Wrong URL.
ctx = luci_context.read('local_auth')
r = requests.post(
url=
'http://127.0.0.1:%d/blah/LuciLocalAuthService.GetOAuthToken' %
ctx['rpc_port'],
data=json.dumps({
'account_id': 'acc_1',
'scopes': ['A', 'B', 'C'],
'secret': ctx['secret'],
}),
headers={'Content-Type': 'application/json'})
self.assertEqual(404, r.status_code)
# Wrong HTTP method.
r = requests.get(
url='http://127.0.0.1:%d/rpc/LuciLocalAuthService.GetOAuthToken'
% ctx['rpc_port'],
data=json.dumps({
'account_id': 'acc_1',
'scopes': ['A', 'B', 'C'],
'secret': ctx['secret'],
}),
headers={'Content-Type': 'application/json'})
self.assertEqual(501, r.status_code)
# Wrong content type.
r = requests.post(
url='http://127.0.0.1:%d/rpc/LuciLocalAuthService.GetOAuthToken'
% ctx['rpc_port'],
data=json.dumps({
'account_id': 'acc_1',
'scopes': ['A', 'B', 'C'],
'secret': ctx['secret'],
}),
headers={'Content-Type': 'application/xml'})
self.assertEqual(400, r.status_code)
# Bad JSON.
r = requests.post(
url='http://127.0.0.1:%d/rpc/LuciLocalAuthService.GetOAuthToken'
% ctx['rpc_port'],
data='not a json',
headers={'Content-Type': 'application/json'})
self.assertEqual(400, r.status_code)
def test_works(self):
calls = []
def token_gen(account_id, scopes):
calls.append((account_id, scopes))
return auth_server.AccessToken('tok_%s' % account_id, time.time() + 300)
with local_auth_server(token_gen, 'acc_1'):
# Accounts are set correctly.
ctx = luci_context.read('local_auth')
ctx.pop('rpc_port')
ctx.pop('secret')
self.assertEqual({
'accounts': [
{'email': '*****@*****.**', 'id': 'acc_1'},
{'email': '*****@*****.**', 'id': 'acc_2'},
{'email': '*****@*****.**', 'id': 'acc_3'},
],
'default_account_id': 'acc_1',
}, ctx)
# Grab initial token.
resp = call_rpc('acc_1', ['B', 'B', 'A', 'C'])
self.assertEqual(
{u'access_token': u'tok_acc_1', u'expiry': self.epoch + 300}, resp)
self.assertEqual([('acc_1', ('A', 'B', 'C'))], calls)
del calls[:]
# Reuses cached token until it is close to expiration.
self.mock_time(60)
resp = call_rpc('acc_1', ['B', 'A', 'C'])
self.assertEqual(
{u'access_token': u'tok_acc_1', u'expiry': self.epoch + 300}, resp)
self.assertFalse(calls)
# Asking for different account gives another token.
resp = call_rpc('acc_2', ['B', 'B', 'A', 'C'])
self.assertEqual(
{u'access_token': u'tok_acc_2', u'expiry': self.epoch + 360}, resp)
self.assertEqual([('acc_2', ('A', 'B', 'C'))], calls)
del calls[:]
# First token has expired. Generated new one.
self.mock_time(300)
resp = call_rpc('acc_1', ['A', 'B', 'C'])
self.assertEqual(
{u'access_token': u'tok_acc_1', u'expiry': self.epoch + 600}, resp)
self.assertEqual([('acc_1', ('A', 'B', 'C'))], calls)
def set_luci_context_account(account, tmp_dir):
"""Sets LUCI_CONTEXT account to be used by the task.
If 'account' is None or '', does nothing at all. This happens when
run_isolated.py is called without '--switch-to-account' flag. In this case,
if run_isolated.py is running in some LUCI_CONTEXT environment, the task will
just inherit whatever account is already set. This may happen is users invoke
run_isolated.py explicitly from their code.
If the requested account is not defined in the context, switches to
non-authenticated access. This happens for Swarming tasks that don't use
'task' service accounts.
If not using LUCI_CONTEXT-based auth, does nothing.
If already running as requested account, does nothing.
"""
if not account:
# Not actually switching.
yield
return
local_auth = luci_context.read('local_auth')
if not local_auth:
# Not using LUCI_CONTEXT auth at all.
yield
return
# See LUCI_CONTEXT.md for the format of 'local_auth'.
if local_auth.get('default_account_id') == account:
# Already set, no need to switch.
yield
return
available = {a['id'] for a in local_auth.get('accounts') or []}
if account in available:
logging.info('Switching default LUCI_CONTEXT account to %r', account)
local_auth['default_account_id'] = account
else:
logging.warning(
'Requested LUCI_CONTEXT account %r is not available (have only %r), '
'disabling authentication', account, sorted(available))
local_auth.pop('default_account_id', None)
with luci_context.write(_tmpdir=tmp_dir, local_auth=local_auth):
yield
def _load_local_auth():
"""Returns a LocalAuthParameters tuple from LUCI_CONTEXT.
Returns:
LocalAuthParameters for connecting to a local auth server.
Raises:
BadLuciContextParameters if file is missing or not valid.
"""
data = luci_context.read('local_auth')
if data is None:
raise BadLuciContextParameters('Missing "local_auth" in LUCI_CONTEXT')
try:
return LocalAuthParameters(rpc_port=int(data['rpc_port']),
secret=str(data['secret']))
except ValueError as e:
raise BadLuciContextParameters(
'Invalid "local_auth" section in LUCI_CONTEXT: %s' % (e, ))
def run_command(
remote, task_details, work_dir,
cost_usd_hour, start, min_free_space, bot_file):
self.assertTrue(remote.uses_auth) # mainly to avoid unused arg warning
self.assertTrue(isinstance(task_details, task_runner.TaskDetails))
# Necessary for OSX.
self.assertEqual(
os.path.realpath(self.work_dir), os.path.realpath(work_dir))
self.assertEqual(3600., cost_usd_hour)
self.assertEqual(time.time(), start)
self.assertEqual(1, min_free_space)
self.assertEqual('/path/to/bot-file', bot_file)
self.assertIsNone(luci_context.read('local_auth'))
return {
u'exit_code': 0,
u'hard_timeout': False,
u'io_timeout': False,
u'must_signal_internal_failure': None,
u'version': task_runner.OUT_VERSION,
}
def has_local_auth():
"""Checks LUCI_CONTEXT to see if we should enable ambient authentication."""
if not luci_context.read('local_auth'):
return False
try:
params = _load_local_auth()
except BadLuciContextParameters as exc:
logging.error('LUCI_CONTEXT["local_auth"] is broken, ignoring it: %s', exc)
return False
# Old protocol doesn't specify 'accounts' at all. It has only one account that
# is always enabled.
#
# TODO(vadimsh): Get rid of support of old protocol when it isn't deployed
# anywhere anymore.
if not params.accounts:
return True
# In the new protocol (when 'accounts' are always specified), use ambient
# authentication only if it is explicitly enabled in LUCI_CONTEXT by non-None
# 'default_account_id'.
return bool(params.default_account_id)
def load_and_run(
in_file, swarming_server, is_grpc, cost_usd_hour, start, out_file,
run_isolated_flags, bot_file, auth_params_file):
"""Loads the task's metadata, prepares auth environment and executes the task.
This may throw all sorts of exceptions in case of failure. It's up to the
caller to trap them. These shall be considered 'internal_failure' instead of
'failure' from a TaskRunResult standpoint.
"""
auth_system = None
local_auth_context = None
task_result = None
work_dir = os.path.dirname(out_file)
def handler(sig, _):
logging.info('Got signal %s', sig)
raise ExitSignal(sig)
try:
with subprocess42.set_signal_handler([SIG_BREAK_OR_TERM], handler):
# The work directory is guaranteed to exist since it was created by
# bot_main.py and contains the manifest. Temporary files will be
# downloaded there. It's bot_main.py that will delete the directory
# afterward. Tests are not run from there.
if not os.path.isdir(work_dir):
raise InternalError('%s expected to exist' % work_dir)
# Raises InternalError on errors.
task_details = TaskDetails.load(in_file)
# This will start a thread that occasionally reads bot authentication
# headers from 'auth_params_file'. It will also optionally launch local
# HTTP server that serves OAuth tokens to the task processes. We put
# location of this service into a file referenced by LUCI_CONTEXT env var
# below.
if auth_params_file:
try:
auth_system = bot_auth.AuthSystem(auth_params_file)
local_auth_context = auth_system.start()
except bot_auth.AuthSystemError as e:
raise InternalError('Failed to init auth: %s' % e)
# Override LUCI_CONTEXT['local_auth']. If the task is not using auth,
# do NOT inherit existing local_auth (if its there). Kick it out by
# passing None.
context_edits = {
'local_auth': local_auth_context
}
# Extend existing LUCI_CONTEXT['swarming'], if any.
if task_details.secret_bytes is not None:
swarming = luci_context.read('swarming') or {}
swarming['secret_bytes'] = task_details.secret_bytes
context_edits['swarming'] = swarming
# Returns bot authentication headers dict or raises InternalError.
def headers_cb():
try:
if auth_system:
return auth_system.get_bot_headers()
return (None, None) # A timeout of "None" means "don't use auth"
except bot_auth.AuthSystemError as e:
raise InternalError('Failed to grab bot auth headers: %s' % e)
# Make a client that can send request to Swarming using bot auth headers.
grpc_proxy = ''
if is_grpc:
grpc_proxy = swarming_server
swarming_server = ''
# The hostname and work dir provided here don't really matter, since the
# task runner is always called with a specific versioned URL.
remote = remote_client.createRemoteClient(
swarming_server, headers_cb, os_utilities.get_hostname_short(),
work_dir, grpc_proxy)
remote.initialize()
# Let AuthSystem know it can now send RPCs to Swarming (to grab OAuth
# tokens). There's a circular dependency here! AuthSystem will be
# indirectly relying on its own 'get_bot_headers' method to authenticate
# RPCs it sends through the provided client.
if auth_system:
auth_system.set_remote_client(remote)
# Auth environment is up, start the command. task_result is dumped to
# disk in 'finally' block.
with luci_context.stage(_tmpdir=work_dir, **context_edits) as ctx_file:
task_result = run_command(
remote, task_details, work_dir, cost_usd_hour,
start, run_isolated_flags, bot_file, ctx_file)
except (ExitSignal, InternalError, remote_client.InternalError) as e:
# This normally means run_command() didn't get the chance to run, as it
# itself traps exceptions and will report accordingly. In this case, we want
# the parent process to send the message instead.
if not task_result:
task_result = {
u'exit_code': -1,
u'hard_timeout': False,
u'io_timeout': False,
u'must_signal_internal_failure': str(e.message or 'unknown error'),
u'version': OUT_VERSION,
}
finally:
# We've found tests to delete the working directory work_dir when quitting,
# causing an exception here. Try to recreate the directory if necessary.
if not os.path.isdir(work_dir):
os.mkdir(work_dir)
if auth_system:
auth_system.stop()
with open(out_file, 'wb') as f:
json.dump(task_result, f)
def has_local_auth():
"""Checks LUCI_CONTEXT to see if local_auth parameters are defined."""
return luci_context.read('local_auth') is not None
