def test_two_default_groups(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(auth=DEFAULT_AUTH_CFG), bots_pb2.BotGroup(auth=DEFAULT_AUTH_CFG), ]) self.validator_test( cfg, [u'bot_group #1: group #0 is already set as default'])
def test_bad_owners(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup( bot_id=['blah'], auth=DEFAULT_AUTH_CFG, owners=['bad email']), ]) self.validator_test(cfg, ['bot_group #0: invalid owner email "bad email"'])
def test_bad_auth_cfg_no_ip_whitelist(self): cfg = bots_pb2.BotsCfg( bot_group=[bots_pb2.BotGroup(auth=[bots_pb2.BotAuth()])]) self.validator_test(cfg, [ 'bot_group #0: if all auth requirements are unset, ' 'ip_whitelist must be set' ])
def test_bad_require_gce_vm_token_no_proj(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(auth=[ bots_pb2.BotAuth(require_gce_vm_token=bots_pb2.BotAuth.GCE()), ]) ]) self.validator_test( cfg, ['bot_group #0: missing project in require_gce_vm_token'])
def test_bad_ip_whitelist_name(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(auth=[ bots_pb2.BotAuth(ip_whitelist='bad ## name'), ]) ]) self.validator_test( cfg, ['bot_group #0: invalid ip_whitelist name "bad ## name"'])
def test_duplicate_prefixes(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(bot_id_prefix=['abc-'], auth=DEFAULT_AUTH_CFG), bots_pb2.BotGroup(bot_id_prefix=['abc-'], auth=DEFAULT_AUTH_CFG), ]) self.validator_test(cfg, [ 'bot_group #1: bot_id_prefix "abc-" is already specified in group #0' ])
def test_bot_id_duplication(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(bot_id=['b{0..5}'], auth=DEFAULT_AUTH_CFG), bots_pb2.BotGroup(bot_id=['b5'], auth=DEFAULT_AUTH_CFG), ]) self.validator_test( cfg, ['bot_group #1: bot_id "b5" was already mentioned in group #0'])
def test_bad_bot_id(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(bot_id=['blah{}'], auth=DEFAULT_AUTH_CFG), ]) self.validator_test(cfg, [ 'bot_group #0: bad bot_id expression "blah{}" - Invalid set "", ' 'not a list and not a range' ])
def test_bad_dimension_not_kv(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(bot_id=['blah'], auth=DEFAULT_AUTH_CFG, dimensions=['not_kv_pair']), ]) self.validator_test(cfg, [u'bot_group #0: bad dimension u\'not_kv_pair\''])
def test_bad_dimension_bad_dim_key(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(bot_id=['blah'], auth=DEFAULT_AUTH_CFG, dimensions=['blah####key:value:value']), ]) self.validator_test(cfg, [ u'bot_group #0: bad dimension u\'blah####key:value:value\'', ])
def test_system_service_account_bad_email(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(bot_id=['blah'], auth=DEFAULT_AUTH_CFG, system_service_account='bad email'), ]) self.validator_test( cfg, ['bot_group #0: invalid system service account email "bad email"'])
def test_bad_required_service_account(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(auth=[ bots_pb2.BotAuth(require_service_account=['not-an-email']), ]) ]) self.validator_test( cfg, ['bot_group #0: invalid service account email "not-an-email"'])
def _get_expanded_bots_cfg(known_digest=None): """Fetches expanded bots.cfg from the datastore cache. If the cache is not there (may happen right after deploying the service or after changing _BOT_CFG_CACHE_VER), falls back to fetching the config directly right here. This situation is rare. Args: known_digest: digest of ExpandedBotsCfg already known to the caller, to skip fetching it from the cache if nothing has changed. Returns: (True, ExpandedBotsCfg) if fetched some new version from the cache. (True, None) if there's no bots.cfg config at all. (False, None) if the cached version has digest matching 'known_digest'. Raises: BadConfigError if there's no cached config and the current config at HEAD is not passing validation. """ head = _bots_cfg_head_key().get() if not head: # This branch is hit when we deploy the service the first time, before # the fetch cron runs, or after changing _BOT_CFG_CACHE_VER. We manually # refresh the cache in this case, not waiting for the cron. logging.warning('No bots.cfg cached for code v%d, forcing the refresh', _BOT_CFG_CACHE_VER) expanded = refetch_from_config_service( ) # raises BadConfigError on errors if expanded and known_digest and expanded.digest == known_digest: return False, None return True, expanded if known_digest and head.digest == known_digest: return False, None if head.empty: return True, None # At this point we know there's something newer stored in the cache. Grab it. # Since this happens outside of a transaction, we may fetch a version that is # ever newer than pointed to by 'head'. This is fine. body = _bots_cfg_body_key().get() if not body: raise AssertionError( 'BotsCfgBody is missing, this should not be possible') if known_digest and body.digest == known_digest: return False, None # the body was sneakily reverted back just now if body.empty: return True, None logging.debug('Retrieved bots.cfg. %s', body.bots_cfg) bots = bots_pb2.BotsCfg() bots.ParseFromString(body.bots_cfg) logging.debug('Parsed bots.cfg. %s', bots) return True, ExpandedBotsCfg(bots, body.bots_cfg_rev, body.digest)
def test_system_service_account_bot_on_non_oauth_machine(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(bot_id=['blah'], auth=[bots_pb2.BotAuth(ip_whitelist='bots')], system_service_account='bot'), ]) self.validator_test(cfg, [ 'bot_group #0: system_service_account "bot" requires ' 'auth.require_service_account to be used' ])
def test_expands_bot_config_scripts_ok(self): good_script = "# coding=utf-8\nprint('Hello')\n" calls = self.mock_config({ 'bots.cfg': ('rev1', bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup( bot_id=['bot1'], auth=[bots_pb2.BotAuth(require_luci_machine_token=True)], bot_config_script='script.py', ), bots_pb2.BotGroup( bot_id=['bot2'], auth=[bots_pb2.BotAuth(require_luci_machine_token=True)], bot_config_script='script.py', ), ], )), 'scripts/script.py': ('rev2', good_script), }) # Has 'bot_config_script_content' populated. cfg = bot_groups_config.refetch_from_config_service() self.assertEqual( bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup( bot_id=['bot1'], auth=[bots_pb2.BotAuth(require_luci_machine_token=True)], bot_config_script='script.py', bot_config_script_rev='rev2', bot_config_script_content=good_script, ), bots_pb2.BotGroup( bot_id=['bot2'], auth=[bots_pb2.BotAuth(require_luci_machine_token=True)], bot_config_script='script.py', bot_config_script_rev='rev2', bot_config_script_content=good_script, ), ], ), cfg.bots) # The script was fetched only once. self.assertEqual({'bots.cfg': 1, u'scripts/script.py': 1}, calls)
def test_system_service_account_bot_on_oauth_machine(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup( bot_id=['blah'], auth=[ bots_pb2.BotAuth( require_service_account=['*****@*****.**']) ], system_service_account='bot'), ]) self.validator_test(cfg, [])
def test_bad_auth_cfg_two_methods(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(auth=[ bots_pb2.BotAuth(require_luci_machine_token=True, require_service_account=['*****@*****.**']), ]) ]) self.validator_test(cfg, [ 'bot_group #0: require_luci_machine_token and require_service_account ' 'can\'t be used at the same time' ])
def test_intersecting_prefixes(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(bot_id_prefix=['abc-'], auth=DEFAULT_AUTH_CFG), bots_pb2.BotGroup(bot_id_prefix=['abc-def-'], auth=DEFAULT_AUTH_CFG), bots_pb2.BotGroup(bot_id_prefix=['xyz-def-'], auth=DEFAULT_AUTH_CFG), bots_pb2.BotGroup(bot_id_prefix=['xyz-'], auth=DEFAULT_AUTH_CFG), ]) self.validator_test(cfg, [ (u'bot_group #1: bot_id_prefix "abc-def-" contains prefix "abc-", ' 'defined in group #0, making group assigned for bots with prefix ' '"abc-" ambigious'), (u'bot_group #3: bot_id_prefix "xyz-" is subprefix of "xyz-def-", ' 'defined in group #2, making group assigned for bots with prefix ' '"xyz-" ambigious'), ])
def test_expands_bot_config_scripts_fail(self): self.mock_config({ 'bots.cfg': ('rev1', bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup( auth=[bots_pb2.BotAuth(require_luci_machine_token=True)], bot_config_script='script.py', ), ], )), 'scripts/script.py': ('rev2', '!!not python!!'), }) ctx = ValidationCtx() with self.assertRaises(bot_groups_config.BadConfigError): bot_groups_config.refetch_from_config_service(ctx) ctx.assert_errors(self, [ 'bot_group #0: invalid bot config script "script.py": invalid syntax' ' (<unknown>, line 1)', ])
def test_duplicate_bot_and_prefix_ids(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(bot_id=['abc', 'ok'], bot_id_prefix=['xyz'], auth=DEFAULT_AUTH_CFG, dimensions=['g:first']), bots_pb2.BotGroup(bot_id=['xyz'], bot_id_prefix=['abc', 'ok-'], auth=DEFAULT_AUTH_CFG, dimensions=['g:second']), bots_pb2.BotGroup(bot_id=['foo'], bot_id_prefix=['foo'], auth=DEFAULT_AUTH_CFG, dimensions=['g:third']), bots_pb2.BotGroup(auth=DEFAULT_AUTH_CFG, dimensions=['g:default']), ]) self.validator_test(cfg, [ (u'bot_group #1: bot_id "xyz" was already mentioned as bot_id_prefix ' 'in group #0'), (u'bot_group #1: bot_id_prefix "abc" is already specified as bot_id ' 'in group #0'), (u'bot_group #2: bot_id_prefix "foo" is already specified as bot_id ' 'in group #2'), ])
from proto.config import bots_pb2 from server import bot_groups_config TEST_CONFIG = bots_pb2.BotsCfg( trusted_dimensions=['pool'], bot_group=[ bots_pb2.BotGroup( bot_id=['bot1', 'bot{2..3}'], auth=[ bots_pb2.BotAuth(require_luci_machine_token=True), bots_pb2.BotAuth(require_service_account=['*****@*****.**']), bots_pb2.BotAuth(require_gce_vm_token=bots_pb2.BotAuth.GCE( project='proj'), ), ], owners=['*****@*****.**'], dimensions=['pool:A', 'pool:B', 'other:D'], ), # This group includes an injected bot_config and system_service_account. bots_pb2.BotGroup( bot_id=['other_bot'], bot_id_prefix=['bot'], auth=[bots_pb2.BotAuth(require_service_account=['*****@*****.**'])], bot_config_script='foo.py', system_service_account='bot'), bots_pb2.BotGroup(auth=[bots_pb2.BotAuth(ip_whitelist='bots')], dimensions=['pool:default']), ], ) EXPECTED_GROUP_1 = bot_groups_config._make_bot_group_config( owners=(u'*****@*****.**', ),
def test_empty_config_is_valid(self): self.validator_test(bots_pb2.BotsCfg(), [])
def test_trusted_dimensions_valid(self): cfg = bots_pb2.BotsCfg(trusted_dimensions=['pool', 'project']) self.validator_test(cfg, [])
def test_trusted_dimensions_invalid(self): cfg = bots_pb2.BotsCfg(trusted_dimensions=['pool:blah']) self.validator_test( cfg, [u'trusted_dimensions: invalid dimension key u\'pool:blah\''])
def test_empty_prefix(self): cfg = bots_pb2.BotsCfg(bot_group=[ bots_pb2.BotGroup(bot_id_prefix=[''], auth=DEFAULT_AUTH_CFG) ]) self.validator_test( cfg, ['bot_group #0: empty bot_id_prefix is not allowed'])
def test_bad_auth_completely_missing(self): cfg = bots_pb2.BotsCfg(bot_group=[bots_pb2.BotGroup()]) self.validator_test(cfg, ['bot_group #0: an "auth" entry is required'])
TEST_CONFIG = bots_pb2.BotsCfg( trusted_dimensions=['pool'], bot_group=[ bots_pb2.BotGroup( bot_id=['bot_with_token'], auth=[bots_pb2.BotAuth(require_luci_machine_token=True)], dimensions=['pool:with_token']), bots_pb2.BotGroup(bot_id=['bot_with_service_account'], auth=[ bots_pb2.BotAuth(require_service_account=[ '*****@*****.**', '*****@*****.**', ]), ], dimensions=['pool:with_service_account']), bots_pb2.BotGroup(bot_id=['bot_with_ip_whitelist'], auth=[bots_pb2.BotAuth(ip_whitelist='ip_whitelist')], dimensions=['pool:with_ip_whitelist']), bots_pb2.BotGroup(bot_id=['bot_with_gce_token'], auth=[ bots_pb2.BotAuth( require_gce_vm_token=bots_pb2.BotAuth.GCE( project='expected_proj'), ), ], dimensions=['pool:bot_with_gce_token']), bots_pb2.BotGroup( bot_id=['bot_with_service_account_and_ip_whitelist'], auth=[ bots_pb2.BotAuth( require_service_account=['*****@*****.**'], ip_whitelist='ip_whitelist', ), ], dimensions=['pool:with_service_account_and_ip_whitelist']), bots_pb2.BotGroup(bot_id=['bot_with_token_and_ip_whitelist'], auth=[ bots_pb2.BotAuth(require_luci_machine_token=True, ip_whitelist='ip_whitelist'), ], dimensions=['pool:with_token_and_ip_whitelist']), bots_pb2.BotGroup(bot_id=['bot_with_fallback_to_ip_wl'], auth=[ bots_pb2.BotAuth(require_luci_machine_token=True, log_if_failed=True), bots_pb2.BotAuth(ip_whitelist='ip_whitelist'), ], dimensions=['pool:with_fallback_to_ip_wl']), bots_pb2.BotGroup( bot_id=['bot_host--container1'], auth=[bots_pb2.BotAuth(require_luci_machine_token=True)], dimensions=['pool:container1']), bots_pb2.BotGroup( bot_id=['bot_host--container2'], auth=[bots_pb2.BotAuth(require_luci_machine_token=True)], dimensions=['pool:container2']), bots_pb2.BotGroup( bot_id=['bot_host--container{3..4}'], auth=[bots_pb2.BotAuth(require_luci_machine_token=True)], dimensions=['pool:container_range']), bots_pb2.BotGroup( bot_id=['bot_host'], auth=[bots_pb2.BotAuth(require_luci_machine_token=True)], dimensions=['pool:bot_host']), bots_pb2.BotGroup( auth=[bots_pb2.BotAuth(require_luci_machine_token=True)], dimensions=['pool:unassigned']), ], )