def test_fetch_auth_db(self): # Create AuthGlobalConfig. global_config = model.AuthGlobalConfig(key=model.root_key()) global_config.oauth_client_id = '1' global_config.oauth_client_secret = 'secret' global_config.oauth_additional_client_ids = ['2', '3'] global_config.put() # Create a bunch of (empty) groups. groups = [ model.AuthGroup(key=model.group_key('Group A')), model.AuthGroup(key=model.group_key('Group B')), ] for group in groups: group.put() # And a bunch of secrets (local and global). local_secrets = [ model.AuthSecret.bootstrap('local%d' % i, 'local') for i in (0, 1, 2) ] global_secrets = [ model.AuthSecret.bootstrap('global%d' % i, 'global') for i in (0, 1, 2) ] # And IP whitelist. ip_whitelist_assignments = model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), assignments=[ model.AuthIPWhitelistAssignments.Assignment( identity=model.Anonymous, ip_whitelist='some ip whitelist', ), ]) ip_whitelist_assignments.put() some_ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('some ip whitelist'), subnets=['127.0.0.1/32']) bots_ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('bots'), subnets=['127.0.0.1/32']) some_ip_whitelist.put() bots_ip_whitelist.put() # This all stuff should be fetched into AuthDB. auth_db = api.fetch_auth_db() self.assertEqual(global_config, auth_db.global_config) self.assertEqual(set(g.key.id() for g in groups), set(auth_db.groups)) self.assertEqual(set(s.key.id() for s in local_secrets), set(auth_db.secrets['local'])) self.assertEqual(set(s.key.id() for s in global_secrets), set(auth_db.secrets['global'])) self.assertEqual(ip_whitelist_assignments, auth_db.ip_whitelist_assignments) self.assertEqual( { 'bots': bots_ip_whitelist, 'some ip whitelist': some_ip_whitelist }, auth_db.ip_whitelists)
def make_ip_whitelist(name, comment, **kwargs): wl = model.AuthIPWhitelist(key=model.ip_whitelist_key(name), **kwargs) wl.record_revision( modified_by=ident('*****@*****.**'), modified_ts=utils.utcnow(), comment=comment) wl.put()
def make_auth_db(): model.AuthGlobalConfig(key=model.root_key()).put() model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key()).put() model.AuthGroup(key=model.group_key('A group')).put() model.AuthIPWhitelist(key=model.ip_whitelist_key('A whitelist')).put() model.replicate_auth_db()
def test_is_ip_whitelisted(self): ent = model.AuthIPWhitelist(subnets=['127.0.0.1', '192.168.0.0/24']) test = lambda ip: ent.is_ip_whitelisted(ipaddr.ip_from_string(ip)) self.assertTrue(test('127.0.0.1')) self.assertTrue(test('192.168.0.0')) self.assertTrue(test('192.168.0.9')) self.assertTrue(test('192.168.0.255')) self.assertFalse(test('192.168.1.0')) self.assertFalse(test('192.1.0.0'))
def make_auth_db_with_ip_whitelist(): """AuthDB with [email protected] assigned IP whitelist '127.0.0.1/32'.""" return api.AuthDB( ip_whitelists=[ model.AuthIPWhitelist( key=model.ip_whitelist_key('some ip whitelist'), subnets=['127.0.0.1/32'], ), model.AuthIPWhitelist( key=model.ip_whitelist_key('bots'), subnets=['192.168.1.1/32', '::1/32'], ), ], ip_whitelist_assignments=model.AuthIPWhitelistAssignments( assignments=[ model.AuthIPWhitelistAssignments.Assignment( identity=model.Identity(model.IDENTITY_USER, '*****@*****.**'), ip_whitelist='some ip whitelist',) ], ), )
def test_ip_whitelists_serialization(self): """Serializing snapshot with non-trivial IP whitelist.""" ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('bots'), subnets=['127.0.0.1/32'], description='Blah blah blah', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]'), ) snapshot = make_snapshot_obj(ip_whitelists=[ip_whitelist]) self.assert_serialization_works(snapshot)
def modify(name, **kwargs): k = model.ip_whitelist_key(name) e = k.get() if not e: e = model.AuthIPWhitelist( key=k, created_by=model.Identity.from_bytes('user:[email protected]'), created_ts=utils.utcnow()) e.record_revision( modified_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), comment='Comment') e.populate(**kwargs) e.put() model.replicate_auth_db()
def test_ip_whitelists_serialization(self): """Serializing snapshot with non-trivial IP whitelist.""" ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('bots'), subnets=['127.0.0.1/32'], description='Blah blah blah', created_ts=datetime.datetime(2020, 1, 1, 1, 1, 1), created_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=datetime.datetime(2020, 2, 2, 2, 2, 2), modified_by=model.Identity.from_bytes('user:[email protected]'), ) auth_db = make_auth_db_proto(ip_whitelists=[ip_whitelist]) self.assertEqual(list(auth_db.ip_whitelists), [ replication_pb2.AuthIPWhitelist( name='bots', subnets=['127.0.0.1/32'], description='Blah blah blah', created_ts=1577840461000000, created_by='user:[email protected]', modified_ts=1580608922000000, modified_by='user:[email protected]', ) ])
def test_non_empty(self): self.mock_now(datetime.datetime(2014, 1, 1, 1, 1, 1)) state = model.AuthReplicationState( key=model.replication_state_key(), primary_id='blah', primary_url='https://blah', auth_db_rev=123) state.put() global_config = model.AuthGlobalConfig( key=model.root_key(), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]'), oauth_client_id='oauth_client_id', oauth_client_secret='oauth_client_secret', oauth_additional_client_ids=['a', 'b']) global_config.put() group = model.AuthGroup( key=model.group_key('Some group'), members=[model.Identity.from_bytes('user:[email protected]')], globs=[model.IdentityGlob.from_bytes('user:*@example.com')], nested=[], description='Some description', owners='owning-group', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) group.put() another = model.AuthGroup( key=model.group_key('Another group'), nested=['Some group']) another.put() global_secret = model.AuthSecret( id='global_secret', parent=model.secret_scope_key('global'), values=['1234', '5678'], modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) global_secret.put() # Local secret should not appear in a snapshot. local_secret = model.AuthSecret( id='local_secret', parent=model.secret_scope_key('local'), values=['1234', '5678'], modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) local_secret.put() ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('bots'), subnets=['127.0.0.1/32'], description='Some description', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) ip_whitelist.put() ip_whitelist_assignments = model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]'), assignments=[ model.AuthIPWhitelistAssignments.Assignment( identity=model.Identity.from_bytes('user:[email protected]'), ip_whitelist='bots', comment='some comment', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes('user:[email protected]')), ]) ip_whitelist_assignments.put() captured_state, snapshot = replication.new_auth_db_snapshot() expected_state = { 'auth_db_rev': 123, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'primary_id': u'blah', 'primary_url': u'https://blah', } self.assertEqual(expected_state, captured_state.to_dict()) expected_snapshot = { 'global_config': { '__id__': 'root', '__parent__': None, 'auth_db_rev': None, 'auth_db_prev_rev': None, 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'oauth_additional_client_ids': [u'a', u'b'], 'oauth_client_id': u'oauth_client_id', 'oauth_client_secret': u'oauth_client_secret', }, 'groups': [ { '__id__': 'Another group', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': None, 'description': u'', 'globs': [], 'members': [], 'modified_by': None, 'modified_ts': None, 'nested': [u'Some group'], 'owners': u'administrators', }, { '__id__': 'Some group', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'Some description', 'globs': [model.IdentityGlob(kind='user', pattern='*@example.com')], 'members': [model.Identity(kind='user', name='*****@*****.**')], 'modified_by': model.Identity( kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'nested': [], 'owners': u'owning-group', }, ], 'secrets': [ { '__id__': 'global_secret', '__parent__': ndb.Key( 'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'), 'modified_by': model.Identity( kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'values': ['1234', '5678'], }, ], 'ip_whitelists': [ { '__id__': 'bots', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'Some description', 'modified_by': model.Identity( kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'subnets': [u'127.0.0.1/32'], }, ], 'ip_whitelist_assignments': { '__id__': 'default', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'assignments': [ { 'comment': u'some comment', 'created_by': model.Identity( kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'identity': model.Identity( kind='user', name='*****@*****.**'), 'ip_whitelist': u'bots', }, ], 'auth_db_rev': None, 'auth_db_prev_rev': None, 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), }, } self.assertEqual(expected_snapshot, snapshot_to_dict(snapshot))
def ip_whitelist(name, **kwargs): return model.AuthIPWhitelist( key=model.ip_whitelist_key(name), created_ts=utils.utcnow(), modified_ts=utils.utcnow(), **kwargs)
def _update_ip_whitelist_config(rev, conf): assert ndb.in_transaction(), 'Must be called in AuthDB transaction' now = utils.utcnow() # Existing whitelist entities. existing_ip_whitelists = { e.key.id(): e for e in model.AuthIPWhitelist.query(ancestor=model.root_key()) } # Whitelists being imported (name => IPWhitelist proto msg). imported_ip_whitelists = {msg.name: msg for msg in conf.ip_whitelists} to_put = [] to_delete = [] # New or modified IP whitelists. for wl_proto in imported_ip_whitelists.itervalues(): # Convert proto magic list to a regular list. subnets = list(wl_proto.subnets) # Existing whitelist and it hasn't changed? wl = existing_ip_whitelists.get(wl_proto.name) if wl and wl.subnets == subnets: continue # Update existing (to preserve auth_db_prev_rev) or create a new one. if not wl: wl = model.AuthIPWhitelist( key=model.ip_whitelist_key(wl_proto.name), created_ts=now, created_by=model.get_service_self_identity()) wl.subnets = subnets wl.description = 'Imported from ip_whitelist.cfg at rev %s' % rev.revision to_put.append(wl) # Removed IP whitelists. for wl in existing_ip_whitelists.itervalues(): if wl.key.id() not in imported_ip_whitelists: to_delete.append(wl) # Update assignments. Don't touch created_ts and created_by for existing ones. ip_whitelist_assignments = (model.ip_whitelist_assignments_key().get() or model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key())) existing = {(a.identity.to_bytes(), a.ip_whitelist): a for a in ip_whitelist_assignments.assignments} updated = [] for a in conf.assignments: key = (a.identity, a.ip_whitelist_name) if key in existing: updated.append(existing[key]) else: new_one = model.AuthIPWhitelistAssignments.Assignment( identity=model.Identity.from_bytes(a.identity), ip_whitelist=a.ip_whitelist_name, comment='Imported from ip_whitelist.cfg at rev %s' % rev.revision, created_ts=now, created_by=model.get_service_self_identity()) updated.append(new_one) # Something has changed? updated_keys = [(a.identity.to_bytes(), a.ip_whitelist) for a in updated] if set(updated_keys) != set(existing): ip_whitelist_assignments.assignments = updated to_put.append(ip_whitelist_assignments) if not to_put and not to_delete: return False comment = 'Importing ip_whitelist.cfg at rev %s' % rev.revision for e in to_put: e.record_revision(modified_by=model.get_service_self_identity(), modified_ts=now, comment=comment) for e in to_delete: e.record_deletion(modified_by=model.get_service_self_identity(), modified_ts=now, comment=comment) futures = [] futures.extend(ndb.put_multi_async(to_put)) futures.extend(ndb.delete_multi_async(e.key for e in to_delete)) for f in futures: f.check_success() return True
def store_whitelist(name): model.AuthIPWhitelist(key=model.ip_whitelist_key(name)).put()
def ip_whitelist(name, **kwargs): return model.AuthIPWhitelist(key=model.ip_whitelist_key(name), **kwargs)
def test_non_empty(self): self.mock_now(datetime.datetime(2014, 1, 1, 1, 1, 1)) state = model.AuthReplicationState(key=model.replication_state_key(), primary_id='blah', primary_url='https://blah', auth_db_rev=123) state.put() global_config = model.AuthGlobalConfig( key=model.root_key(), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]'), oauth_client_id='oauth_client_id', oauth_client_secret='oauth_client_secret', oauth_additional_client_ids=['a', 'b'], token_server_url='https://token-server', security_config='security config blob') global_config.put() group = model.AuthGroup( key=model.group_key('Some group'), members=[model.Identity.from_bytes('user:[email protected]')], globs=[model.IdentityGlob.from_bytes('user:*@example.com')], nested=[], description='Some description', owners='owning-group', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) group.put() another = model.AuthGroup(key=model.group_key('Another group'), nested=['Some group']) another.put() ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('bots'), subnets=['127.0.0.1/32'], description='Some description', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) ip_whitelist.put() ip_whitelist_assignments = model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]'), assignments=[ model.AuthIPWhitelistAssignments.Assignment( identity=model.Identity.from_bytes( 'user:[email protected]'), ip_whitelist='bots', comment='some comment', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes( 'user:[email protected]')), ]) ip_whitelist_assignments.put() realms_globals = model.AuthRealmsGlobals( key=model.realms_globals_key(), permissions=[ realms_pb2.Permission(name='luci.dev.p1'), realms_pb2.Permission(name='luci.dev.p2'), ]) realms_globals.put() model.AuthProjectRealms(key=model.project_realms_key('proj_id1'), realms=realms_pb2.Realms(api_version=1234), config_rev='rev1', perms_rev='rev1').put() model.AuthProjectRealms(key=model.project_realms_key('proj_id2'), realms=realms_pb2.Realms(api_version=1234), config_rev='rev2', perms_rev='rev2').put() captured_state, snapshot = replication.new_auth_db_snapshot() expected_state = { 'auth_db_rev': 123, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'primary_id': u'blah', 'primary_url': u'https://blah', 'shard_ids': [], } self.assertEqual(expected_state, captured_state.to_dict()) expected_snapshot = { 'global_config': { '__id__': 'root', '__parent__': None, 'auth_db_rev': None, 'auth_db_prev_rev': None, 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'oauth_additional_client_ids': [u'a', u'b'], 'oauth_client_id': u'oauth_client_id', 'oauth_client_secret': u'oauth_client_secret', 'security_config': 'security config blob', 'token_server_url': u'https://token-server', }, 'groups': [ { '__id__': 'Another group', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': None, 'description': u'', 'globs': [], 'members': [], 'modified_by': None, 'modified_ts': None, 'nested': [u'Some group'], 'owners': u'administrators', }, { '__id__': 'Some group', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'Some description', 'globs': [model.IdentityGlob(kind='user', pattern='*@example.com')], 'members': [model.Identity(kind='user', name='*****@*****.**')], 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'nested': [], 'owners': u'owning-group', }, ], 'ip_whitelists': [ { '__id__': 'bots', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'Some description', 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'subnets': [u'127.0.0.1/32'], }, ], 'ip_whitelist_assignments': { '__id__': 'default', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'assignments': [ { 'comment': u'some comment', 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'bots', }, ], 'auth_db_rev': None, 'auth_db_prev_rev': None, 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), }, 'realms_globals': { '__id__': 'globals', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_prev_rev': None, 'auth_db_rev': None, 'modified_by': None, 'modified_ts': None, 'permissions': [ realms_pb2.Permission(name='luci.dev.p1'), realms_pb2.Permission(name='luci.dev.p2'), ], }, 'project_realms': [{ '__id__': 'proj_id1', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_prev_rev': None, 'auth_db_rev': None, 'config_rev': u'rev1', 'perms_rev': u'rev1', 'modified_by': None, 'modified_ts': None, 'realms': realms_pb2.Realms(api_version=1234), }, { '__id__': 'proj_id2', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_prev_rev': None, 'auth_db_rev': None, 'config_rev': u'rev2', 'perms_rev': u'rev2', 'modified_by': None, 'modified_ts': None, 'realms': realms_pb2.Realms(api_version=1234), }], } self.assertEqual(expected_snapshot, snapshot_to_dict(snapshot))
def test_fetch_auth_db(self): # Client IDs callback. Disable config.ensure_configured() since it overrides # _additional_client_ids_cb after we mock it. self.mock(config, 'ensure_configured', lambda: None) self.mock(api, '_additional_client_ids_cb', lambda: ['', 'cb_client_id']) self.mock(api, 'get_web_client_id', lambda: 'web_client_id') # Create AuthGlobalConfig. global_config = model.AuthGlobalConfig(key=model.root_key()) global_config.oauth_client_id = '1' global_config.oauth_client_secret = 'secret' global_config.oauth_additional_client_ids = ['2', '3'] global_config.put() # Create a bunch of (empty) groups. groups = [ model.AuthGroup(key=model.group_key('Group A')), model.AuthGroup(key=model.group_key('Group B')), ] for group in groups: group.put() # And a bunch of secrets. secrets = [ model.AuthSecret.bootstrap('local%d' % i) for i in (0, 1, 2) ] # And IP whitelist. ip_whitelist_assignments = model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), assignments=[ model.AuthIPWhitelistAssignments.Assignment( identity=model.Anonymous, ip_whitelist='some ip whitelist', ), ]) ip_whitelist_assignments.put() some_ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('some ip whitelist'), subnets=['127.0.0.1/32']) bots_ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('bots'), subnets=['127.0.0.1/32']) some_ip_whitelist.put() bots_ip_whitelist.put() # This all stuff should be fetched into AuthDB. auth_db = api.fetch_auth_db() self.assertEqual(global_config, auth_db.global_config) self.assertEqual(set(g.key.id() for g in groups), set(auth_db.groups)) self.assertEqual(set(s.key.id() for s in secrets), set(auth_db.secrets)) self.assertEqual(ip_whitelist_assignments, auth_db.ip_whitelist_assignments) self.assertEqual( { 'bots': bots_ip_whitelist, 'some ip whitelist': some_ip_whitelist }, auth_db.ip_whitelists) self.assertTrue(auth_db.is_allowed_oauth_client_id('1')) self.assertTrue(auth_db.is_allowed_oauth_client_id('cb_client_id')) self.assertTrue(auth_db.is_allowed_oauth_client_id('web_client_id')) self.assertFalse(auth_db.is_allowed_oauth_client_id(''))