def test_two_default_groups(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(auth=DEFAULT_AUTH_CFG),
bots_pb2.BotGroup(auth=DEFAULT_AUTH_CFG),
])
self.validator_test(
cfg, [u'bot_group #1: group #0 is already set as default'])
def test_bad_owners(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(
bot_id=['blah'], auth=DEFAULT_AUTH_CFG, owners=['bad email']),
])
self.validator_test(cfg,
['bot_group #0: invalid owner email "bad email"'])
def test_bad_auth_cfg_no_ip_whitelist(self):
cfg = bots_pb2.BotsCfg(
bot_group=[bots_pb2.BotGroup(auth=[bots_pb2.BotAuth()])])
self.validator_test(cfg, [
'bot_group #0: if all auth requirements are unset, '
'ip_whitelist must be set'
])
def test_bad_require_gce_vm_token_no_proj(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(auth=[
bots_pb2.BotAuth(require_gce_vm_token=bots_pb2.BotAuth.GCE()),
])
])
self.validator_test(
cfg, ['bot_group #0: missing project in require_gce_vm_token'])
def test_bad_ip_whitelist_name(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(auth=[
bots_pb2.BotAuth(ip_whitelist='bad ## name'),
])
])
self.validator_test(
cfg, ['bot_group #0: invalid ip_whitelist name "bad ## name"'])
def test_duplicate_prefixes(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(bot_id_prefix=['abc-'], auth=DEFAULT_AUTH_CFG),
bots_pb2.BotGroup(bot_id_prefix=['abc-'], auth=DEFAULT_AUTH_CFG),
])
self.validator_test(cfg, [
'bot_group #1: bot_id_prefix "abc-" is already specified in group #0'
])
def test_bot_id_duplication(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(bot_id=['b{0..5}'], auth=DEFAULT_AUTH_CFG),
bots_pb2.BotGroup(bot_id=['b5'], auth=DEFAULT_AUTH_CFG),
])
self.validator_test(
cfg,
['bot_group #1: bot_id "b5" was already mentioned in group #0'])
def test_bad_bot_id(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(bot_id=['blah{}'], auth=DEFAULT_AUTH_CFG),
])
self.validator_test(cfg, [
'bot_group #0: bad bot_id expression "blah{}" - Invalid set "", '
'not a list and not a range'
])
def test_bad_dimension_not_kv(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(bot_id=['blah'],
auth=DEFAULT_AUTH_CFG,
dimensions=['not_kv_pair']),
])
self.validator_test(cfg,
[u'bot_group #0: bad dimension u\'not_kv_pair\''])
def test_bad_dimension_bad_dim_key(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(bot_id=['blah'],
auth=DEFAULT_AUTH_CFG,
dimensions=['blah####key:value:value']),
])
self.validator_test(cfg, [
u'bot_group #0: bad dimension u\'blah####key:value:value\'',
])
def test_system_service_account_bad_email(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(bot_id=['blah'],
auth=DEFAULT_AUTH_CFG,
system_service_account='bad email'),
])
self.validator_test(
cfg,
['bot_group #0: invalid system service account email "bad email"'])
def test_bad_required_service_account(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(auth=[
bots_pb2.BotAuth(require_service_account=['not-an-email']),
])
])
self.validator_test(
cfg,
['bot_group #0: invalid service account email "not-an-email"'])
def _get_expanded_bots_cfg(known_digest=None):
"""Fetches expanded bots.cfg from the datastore cache.
If the cache is not there (may happen right after deploying the service or
after changing _BOT_CFG_CACHE_VER), falls back to fetching the config directly
right here. This situation is rare.
Args:
known_digest: digest of ExpandedBotsCfg already known to the caller, to skip
fetching it from the cache if nothing has changed.
Returns:
(True, ExpandedBotsCfg) if fetched some new version from the cache.
(True, None) if there's no bots.cfg config at all.
(False, None) if the cached version has digest matching 'known_digest'.
Raises:
BadConfigError if there's no cached config and the current config at HEAD is
not passing validation.
"""
head = _bots_cfg_head_key().get()
if not head:
# This branch is hit when we deploy the service the first time, before
# the fetch cron runs, or after changing _BOT_CFG_CACHE_VER. We manually
# refresh the cache in this case, not waiting for the cron.
logging.warning('No bots.cfg cached for code v%d, forcing the refresh',
_BOT_CFG_CACHE_VER)
expanded = refetch_from_config_service(
) # raises BadConfigError on errors
if expanded and known_digest and expanded.digest == known_digest:
return False, None
return True, expanded
if known_digest and head.digest == known_digest:
return False, None
if head.empty:
return True, None
# At this point we know there's something newer stored in the cache. Grab it.
# Since this happens outside of a transaction, we may fetch a version that is
# ever newer than pointed to by 'head'. This is fine.
body = _bots_cfg_body_key().get()
if not body:
raise AssertionError(
'BotsCfgBody is missing, this should not be possible')
if known_digest and body.digest == known_digest:
return False, None # the body was sneakily reverted back just now
if body.empty:
return True, None
logging.debug('Retrieved bots.cfg. %s', body.bots_cfg)
bots = bots_pb2.BotsCfg()
bots.ParseFromString(body.bots_cfg)
logging.debug('Parsed bots.cfg. %s', bots)
return True, ExpandedBotsCfg(bots, body.bots_cfg_rev, body.digest)
def test_system_service_account_bot_on_non_oauth_machine(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(bot_id=['blah'],
auth=[bots_pb2.BotAuth(ip_whitelist='bots')],
system_service_account='bot'),
])
self.validator_test(cfg, [
'bot_group #0: system_service_account "bot" requires '
'auth.require_service_account to be used'
])
def test_expands_bot_config_scripts_ok(self):
good_script = "# coding=utf-8\nprint('Hello')\n"
calls = self.mock_config({
'bots.cfg':
('rev1',
bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(
bot_id=['bot1'],
auth=[bots_pb2.BotAuth(require_luci_machine_token=True)],
bot_config_script='script.py',
),
bots_pb2.BotGroup(
bot_id=['bot2'],
auth=[bots_pb2.BotAuth(require_luci_machine_token=True)],
bot_config_script='script.py',
),
], )),
'scripts/script.py': ('rev2', good_script),
})
# Has 'bot_config_script_content' populated.
cfg = bot_groups_config.refetch_from_config_service()
self.assertEqual(
bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(
bot_id=['bot1'],
auth=[bots_pb2.BotAuth(require_luci_machine_token=True)],
bot_config_script='script.py',
bot_config_script_rev='rev2',
bot_config_script_content=good_script,
),
bots_pb2.BotGroup(
bot_id=['bot2'],
auth=[bots_pb2.BotAuth(require_luci_machine_token=True)],
bot_config_script='script.py',
bot_config_script_rev='rev2',
bot_config_script_content=good_script,
),
], ), cfg.bots)
# The script was fetched only once.
self.assertEqual({'bots.cfg': 1, u'scripts/script.py': 1}, calls)
def test_system_service_account_bot_on_oauth_machine(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(
bot_id=['blah'],
auth=[
bots_pb2.BotAuth(
require_service_account=['*****@*****.**'])
],
system_service_account='bot'),
])
self.validator_test(cfg, [])
def test_bad_auth_cfg_two_methods(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(auth=[
bots_pb2.BotAuth(require_luci_machine_token=True,
require_service_account=['*****@*****.**']),
])
])
self.validator_test(cfg, [
'bot_group #0: require_luci_machine_token and require_service_account '
'can\'t be used at the same time'
])
def test_intersecting_prefixes(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(bot_id_prefix=['abc-'], auth=DEFAULT_AUTH_CFG),
bots_pb2.BotGroup(bot_id_prefix=['abc-def-'],
auth=DEFAULT_AUTH_CFG),
bots_pb2.BotGroup(bot_id_prefix=['xyz-def-'],
auth=DEFAULT_AUTH_CFG),
bots_pb2.BotGroup(bot_id_prefix=['xyz-'], auth=DEFAULT_AUTH_CFG),
])
self.validator_test(cfg, [
(u'bot_group #1: bot_id_prefix "abc-def-" contains prefix "abc-", '
'defined in group #0, making group assigned for bots with prefix '
'"abc-" ambigious'),
(u'bot_group #3: bot_id_prefix "xyz-" is subprefix of "xyz-def-", '
'defined in group #2, making group assigned for bots with prefix '
'"xyz-" ambigious'),
])
def test_expands_bot_config_scripts_fail(self):
self.mock_config({
'bots.cfg':
('rev1',
bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(
auth=[bots_pb2.BotAuth(require_luci_machine_token=True)],
bot_config_script='script.py',
),
], )),
'scripts/script.py': ('rev2', '!!not python!!'),
})
ctx = ValidationCtx()
with self.assertRaises(bot_groups_config.BadConfigError):
bot_groups_config.refetch_from_config_service(ctx)
ctx.assert_errors(self, [
'bot_group #0: invalid bot config script "script.py": invalid syntax'
' (<unknown>, line 1)',
])
def test_duplicate_bot_and_prefix_ids(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(bot_id=['abc', 'ok'],
bot_id_prefix=['xyz'],
auth=DEFAULT_AUTH_CFG,
dimensions=['g:first']),
bots_pb2.BotGroup(bot_id=['xyz'],
bot_id_prefix=['abc', 'ok-'],
auth=DEFAULT_AUTH_CFG,
dimensions=['g:second']),
bots_pb2.BotGroup(bot_id=['foo'],
bot_id_prefix=['foo'],
auth=DEFAULT_AUTH_CFG,
dimensions=['g:third']),
bots_pb2.BotGroup(auth=DEFAULT_AUTH_CFG, dimensions=['g:default']),
])
self.validator_test(cfg, [
(u'bot_group #1: bot_id "xyz" was already mentioned as bot_id_prefix '
'in group #0'),
(u'bot_group #1: bot_id_prefix "abc" is already specified as bot_id '
'in group #0'),
(u'bot_group #2: bot_id_prefix "foo" is already specified as bot_id '
'in group #2'),
])
from proto.config import bots_pb2
from server import bot_groups_config
TEST_CONFIG = bots_pb2.BotsCfg(
trusted_dimensions=['pool'],
bot_group=[
bots_pb2.BotGroup(
bot_id=['bot1', 'bot{2..3}'],
auth=[
bots_pb2.BotAuth(require_luci_machine_token=True),
bots_pb2.BotAuth(require_service_account=['*****@*****.**']),
bots_pb2.BotAuth(require_gce_vm_token=bots_pb2.BotAuth.GCE(
project='proj'), ),
],
owners=['*****@*****.**'],
dimensions=['pool:A', 'pool:B', 'other:D'],
),
# This group includes an injected bot_config and system_service_account.
bots_pb2.BotGroup(
bot_id=['other_bot'],
bot_id_prefix=['bot'],
auth=[bots_pb2.BotAuth(require_service_account=['*****@*****.**'])],
bot_config_script='foo.py',
system_service_account='bot'),
bots_pb2.BotGroup(auth=[bots_pb2.BotAuth(ip_whitelist='bots')],
dimensions=['pool:default']),
],
)
EXPECTED_GROUP_1 = bot_groups_config._make_bot_group_config(
owners=(u'*****@*****.**', ),
def test_empty_config_is_valid(self):
self.validator_test(bots_pb2.BotsCfg(), [])
def test_trusted_dimensions_valid(self):
cfg = bots_pb2.BotsCfg(trusted_dimensions=['pool', 'project'])
self.validator_test(cfg, [])
def test_trusted_dimensions_invalid(self):
cfg = bots_pb2.BotsCfg(trusted_dimensions=['pool:blah'])
self.validator_test(
cfg, [u'trusted_dimensions: invalid dimension key u\'pool:blah\''])
def test_empty_prefix(self):
cfg = bots_pb2.BotsCfg(bot_group=[
bots_pb2.BotGroup(bot_id_prefix=[''], auth=DEFAULT_AUTH_CFG)
])
self.validator_test(
cfg, ['bot_group #0: empty bot_id_prefix is not allowed'])
def test_bad_auth_completely_missing(self):
cfg = bots_pb2.BotsCfg(bot_group=[bots_pb2.BotGroup()])
self.validator_test(cfg, ['bot_group #0: an "auth" entry is required'])
TEST_CONFIG = bots_pb2.BotsCfg(
trusted_dimensions=['pool'],
bot_group=[
bots_pb2.BotGroup(
bot_id=['bot_with_token'],
auth=[bots_pb2.BotAuth(require_luci_machine_token=True)],
dimensions=['pool:with_token']),
bots_pb2.BotGroup(bot_id=['bot_with_service_account'],
auth=[
bots_pb2.BotAuth(require_service_account=[
'*****@*****.**',
'*****@*****.**',
]),
],
dimensions=['pool:with_service_account']),
bots_pb2.BotGroup(bot_id=['bot_with_ip_whitelist'],
auth=[bots_pb2.BotAuth(ip_whitelist='ip_whitelist')],
dimensions=['pool:with_ip_whitelist']),
bots_pb2.BotGroup(bot_id=['bot_with_gce_token'],
auth=[
bots_pb2.BotAuth(
require_gce_vm_token=bots_pb2.BotAuth.GCE(
project='expected_proj'), ),
],
dimensions=['pool:bot_with_gce_token']),
bots_pb2.BotGroup(
bot_id=['bot_with_service_account_and_ip_whitelist'],
auth=[
bots_pb2.BotAuth(
require_service_account=['*****@*****.**'],
ip_whitelist='ip_whitelist',
),
],
dimensions=['pool:with_service_account_and_ip_whitelist']),
bots_pb2.BotGroup(bot_id=['bot_with_token_and_ip_whitelist'],
auth=[
bots_pb2.BotAuth(require_luci_machine_token=True,
ip_whitelist='ip_whitelist'),
],
dimensions=['pool:with_token_and_ip_whitelist']),
bots_pb2.BotGroup(bot_id=['bot_with_fallback_to_ip_wl'],
auth=[
bots_pb2.BotAuth(require_luci_machine_token=True,
log_if_failed=True),
bots_pb2.BotAuth(ip_whitelist='ip_whitelist'),
],
dimensions=['pool:with_fallback_to_ip_wl']),
bots_pb2.BotGroup(
bot_id=['bot_host--container1'],
auth=[bots_pb2.BotAuth(require_luci_machine_token=True)],
dimensions=['pool:container1']),
bots_pb2.BotGroup(
bot_id=['bot_host--container2'],
auth=[bots_pb2.BotAuth(require_luci_machine_token=True)],
dimensions=['pool:container2']),
bots_pb2.BotGroup(
bot_id=['bot_host--container{3..4}'],
auth=[bots_pb2.BotAuth(require_luci_machine_token=True)],
dimensions=['pool:container_range']),
bots_pb2.BotGroup(
bot_id=['bot_host'],
auth=[bots_pb2.BotAuth(require_luci_machine_token=True)],
dimensions=['pool:bot_host']),
bots_pb2.BotGroup(
auth=[bots_pb2.BotAuth(require_luci_machine_token=True)],
dimensions=['pool:unassigned']),
],
)
