def testing_main():
"""Launches a local HTTP auth service and waits for Ctrl+C.
Useful during development and manual testing.
"""
# Don't mess with sys.path outside of adhoc testing.
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, ROOT_DIR)
from libs import luci_context
logging.basicConfig(level=logging.DEBUG)
class DumbProvider(object):
def generate_token(self, account_id, scopes):
logging.info('generate_token(%r, %r) called', account_id, scopes)
return AccessToken('fake_tok_for_%s' % account_id,
time.time() + 80)
server = LocalAuthServer()
ctx = server.start(token_provider=DumbProvider(),
accounts=['a', 'b', 'c'],
default_account_id='a',
port=11111)
try:
with luci_context.write(local_auth=ctx):
print 'Copy-paste this into another shell:'
print 'export LUCI_CONTEXT=%s' % os.getenv('LUCI_CONTEXT')
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
finally:
server.stop()
def test_main_naked_with_account_pop(self):
self.capture_luci_ctx = True
self.mock_popen_with_oserr()
cmd = [
'--no-log',
'--cache',
os.path.join(self.tempdir, 'isolated_cache'),
'--named-cache-root',
os.path.join(self.tempdir, 'named_cache'),
'--switch-to-account',
'task',
'--raw-cmd',
'--',
'/bin/echo',
'hello',
'world',
]
root_ctx = {
'accounts': [{
'id': 'bot'
}], # only 'bot', there's no 'task'
'default_account_id': 'bot',
'secret': 'sekret',
'rpc_port': 12345,
}
with luci_context.write(local_auth=root_ctx):
run_isolated.main(cmd)
# Unset default account, since 'task' account is not defined.
task_ctx = root_ctx.copy()
task_ctx.pop('default_account_id')
self.assertEqual(task_ctx,
self.popen_calls[0][1]['luci_ctx']['local_auth'])
def test_main_naked_with_account_switch(self):
self.capture_luci_ctx = True
self.mock_popen_with_oserr()
cmd = [
'--no-log',
'--cache', self.tempdir,
'--named-cache-root', os.path.join(self.tempdir, 'c'),
'--switch-to-account', 'task',
'--',
'/bin/echo',
'hello',
'world',
]
root_ctx = {
'accounts': [{'id': 'bot'}, {'id': 'task'}],
'default_account_id' : 'bot',
'secret': 'sekret',
'rpc_port': 12345,
}
with luci_context.write(local_auth=root_ctx):
run_isolated.main(cmd)
# Switched default account to task.
task_ctx = root_ctx.copy()
task_ctx['default_account_id'] = 'task'
self.assertEqual(task_ctx, self.popen_calls[0][1]['luci_ctx']['local_auth'])
def local_auth_server(token_cb, **overrides):
class MockedProvider(object):
def generate_token(self, scopes):
return token_cb(scopes)
s = auth_server.LocalAuthServer()
try:
local_auth = s.start(MockedProvider())
local_auth.update(overrides)
with luci_context.write(local_auth=local_auth):
yield
finally:
s.stop()
def local_auth_server(token_cb, default_account_id, **overrides):
class MockedProvider(object):
def generate_token(self, account_id, scopes):
return token_cb(account_id, scopes)
s = auth_server.LocalAuthServer()
try:
local_auth = s.start(token_provider=MockedProvider(),
accounts=('acc_1', 'acc_2', 'acc_3'),
default_account_id=default_account_id)
local_auth.update(overrides)
with luci_context.write(local_auth=local_auth):
yield
finally:
s.stop()
def set_luci_context_account(account, tmp_dir):
"""Sets LUCI_CONTEXT account to be used by the task.
If 'account' is None or '', does nothing at all. This happens when
run_isolated.py is called without '--switch-to-account' flag. In this case,
if run_isolated.py is running in some LUCI_CONTEXT environment, the task will
just inherit whatever account is already set. This may happen is users invoke
run_isolated.py explicitly from their code.
If the requested account is not defined in the context, switches to
non-authenticated access. This happens for Swarming tasks that don't use
'task' service accounts.
If not using LUCI_CONTEXT-based auth, does nothing.
If already running as requested account, does nothing.
"""
if not account:
# Not actually switching.
yield
return
local_auth = luci_context.read('local_auth')
if not local_auth:
# Not using LUCI_CONTEXT auth at all.
yield
return
# See LUCI_CONTEXT.md for the format of 'local_auth'.
if local_auth.get('default_account_id') == account:
# Already set, no need to switch.
yield
return
available = {a['id'] for a in local_auth.get('accounts') or []}
if account in available:
logging.info('Switching default LUCI_CONTEXT account to %r', account)
local_auth['default_account_id'] = account
else:
logging.warning(
'Requested LUCI_CONTEXT account %r is not available (have only %r), '
'disabling authentication', account, sorted(available))
local_auth.pop('default_account_id', None)
with luci_context.write(_tmpdir=tmp_dir, local_auth=local_auth):
yield
def local_auth_server(token_cb, default_account_id, **overrides):
class MockedProvider(object):
def generate_token(self, account_id, scopes):
return token_cb(account_id, scopes)
acc = lambda aid: auth_server.Account(id=aid, email=aid + '@example.com')
s = auth_server.LocalAuthServer()
try:
local_auth = s.start(token_provider=MockedProvider(),
accounts=(acc('acc_1'), acc('acc_2'),
acc('acc_3')),
default_account_id=default_account_id)
local_auth.update(overrides)
with luci_context.write(local_auth=local_auth):
yield
finally:
s.stop()
def load_and_run(in_file, swarming_server, cost_usd_hour, start, out_file,
min_free_space, bot_file, auth_params_file):
"""Loads the task's metadata, prepares auth environment and executes the task.
This may throw all sorts of exceptions in case of failure. It's up to the
caller to trap them. These shall be considered 'internal_failure' instead of
'failure' from a TaskRunResult standpoint.
"""
auth_system = None
task_result = None
work_dir = os.path.dirname(out_file)
def handler(sig, _):
logging.info('Got signal %s', sig)
raise ExitSignal(sig)
try:
with subprocess42.set_signal_handler([SIG_BREAK_OR_TERM], handler):
# The work directory is guaranteed to exist since it was created by
# bot_main.py and contains the manifest. Temporary files will be
# downloaded there. It's bot_main.py that will delete the directory
# afterward. Tests are not run from there.
if not os.path.isdir(work_dir):
raise InternalError('%s expected to exist' % work_dir)
# Raises InternalError on errors.
task_details = TaskDetails.load(in_file)
# This will start a thread that occasionally reads bot authentication
# headers from 'auth_params_file'. It will also optionally launch local
# HTTP server that serves OAuth tokens to the task processes. We put
# location of this service into a file referenced by LUCI_CONTEXT env var
# below.
if auth_params_file:
try:
auth_system = bot_auth.AuthSystem(auth_params_file)
auth_system.start()
except bot_auth.AuthSystemError as e:
raise InternalError('Failed to init auth: %s' % e)
context_edits = {}
# If the task is using service accounts, add local_auth details to
# LUCI_CONTEXT.
if auth_system and auth_system.local_auth_context:
context_edits['local_auth'] = auth_system.local_auth_context
# Returns bot authentication headers dict or raises InternalError.
def headers_cb():
try:
if auth_system:
# The second parameter is the time until which the remote client
# should cache the headers. Since auth_system is doing the
# caching, we're just sending "0", which is to say the Epoch
# (Jan 1 1970), which effectively means "never cache."
return (auth_system.bot_headers, 0)
return (None, None
) # A timeout of "None" means "don't use auth"
except bot_auth.AuthSystemError as e:
raise InternalError('Failed to grab bot auth headers: %s' %
e)
# Auth environment is up, start the command. task_result is dumped to
# disk in 'finally' block.
remote = remote_client.createRemoteClient(swarming_server,
headers_cb)
with luci_context.write(_tmpdir=work_dir, **context_edits):
task_result = run_command(remote, task_details, work_dir,
cost_usd_hour, start, min_free_space,
bot_file)
except (ExitSignal, InternalError) as e:
# This normally means run_command() didn't get the chance to run, as it
# itself traps exceptions and will report accordingly. In this case, we want
# the parent process to send the message instead.
if not task_result:
task_result = {
u'exit_code': -1,
u'hard_timeout': False,
u'io_timeout': False,
u'must_signal_internal_failure': str(e.message
or 'unknown error'),
u'version': OUT_VERSION,
}
finally:
# We've found tests to delete the working directory work_dir when quitting,
# causing an exception here. Try to recreate the directory if necessary.
if not os.path.isdir(work_dir):
os.mkdir(work_dir)
if auth_system:
auth_system.stop()
with open(out_file, 'wb') as f:
json.dump(task_result, f)
