def update(): existing = ndb.get_multi( model.project_realms_key(rev.project_id) for rev, _ in expanded ) updated = [] metas = [] for (rev, realms), ent in zip(expanded, existing): logging.info('Visiting project "%s"...', rev.project_id) if not ent: logging.info('New realms config in project "%s"', rev.project_id) ent = model.AuthProjectRealms( key=model.project_realms_key(rev.project_id), realms=realms, config_rev=rev.config_rev, perms_rev=db.revision) ent.record_revision( modified_by=model.get_service_self_identity(), comment='New realms config') updated.append(ent) elif ent.realms != realms: logging.info('Updated realms config in project "%s"', rev.project_id) ent.realms = realms ent.config_rev = rev.config_rev ent.perms_rev = db.revision ent.record_revision( modified_by=model.get_service_self_identity(), comment=comment) updated.append(ent) else: logging.info('Realms config in project "%s" are fresh', rev.project_id) # Always update AuthProjectRealmsMeta to match the state we just checked. metas.append(AuthProjectRealmsMeta( key=project_realms_meta_key(rev.project_id), config_rev=rev.config_rev, perms_rev=db.revision, config_digest=rev.config_digest, modified_ts=utils.utcnow(), )) logging.info('Persisting changes...') ndb.put_multi(updated + metas) if updated: model.replicate_auth_db()
def delete_realms(project_id): """Performs an AuthDB transaction that deletes all realms of some project. Args: project_id: ID of the project being deleted. """ realms = model.project_realms_key(project_id).get() if not realms: return # already gone realms.record_deletion( modified_by=model.get_service_self_identity(), comment='No longer in the configs') realms.key.delete() project_realms_meta_key(project_id).delete() model.replicate_auth_db()
def test_update_many_projects(self): self.assertEqual(model.get_auth_db_revision(), 0) cfg_rev = lambda proj, realm, rev_sfx: config.RealmsCfgRev( project_id=proj, config_rev='cfg-rev-' + rev_sfx, config_digest='digest-' + rev_sfx, config_body='realms{ name: "%s" }' % realm, perms_rev=None) # Create a bunch of project configs at once. config.update_realms(fake_db('db-rev1'), [ cfg_rev('proj1', 'realm1', 'p1s1'), cfg_rev('proj2', 'realm1', 'p2s1'), ], 'New config') # Produced a single revision. self.assertEqual(model.get_auth_db_revision(), 1) # Present now. revs = config.get_stored_revs_async().get_result() self.assertEqual(revs, [ config.RealmsCfgRev( project_id='proj1', config_rev=u'cfg-rev-p1s1', config_digest=u'digest-p1s1', config_body=None, perms_rev=u'db-rev1', ), config.RealmsCfgRev( project_id='proj2', config_rev=u'cfg-rev-p2s1', config_digest=u'digest-p2s1', config_body=None, perms_rev=u'db-rev1', ), ]) self.assertEqual( model.project_realms_key('proj1').get().config_rev, 'cfg-rev-p1s1') self.assertEqual( model.project_realms_key('proj2').get().config_rev, 'cfg-rev-p2s1') # One is modified significantly, another not. config.update_realms( fake_db('db-rev1'), [ cfg_rev('proj1', 'realm1', 'p1s2'), # noop change cfg_rev('proj2', 'realm2', 'p2s2'), # significant change ], 'New config') revs = config.get_stored_revs_async().get_result() self.assertEqual( model.project_realms_key('proj1').get().config_rev, 'cfg-rev-p1s1') self.assertEqual( model.project_realms_key('proj2').get().config_rev, 'cfg-rev-p2s2') # One config is broken. config.update_realms(fake_db('db-rev1'), [ cfg_rev('proj1', 'realm3', 'p1s3'), cfg_rev('proj2', '@@@@@@', 'p2s3'), ], 'New config') revs = config.get_stored_revs_async().get_result() self.assertEqual( model.project_realms_key('proj1').get().config_rev, 'cfg-rev-p1s3') self.assertEqual( model.project_realms_key('proj2').get().config_rev, 'cfg-rev-p2s2')
def test_realms_config_lifecycle(self, project_id): self.assertEqual(model.get_auth_db_revision(), 0) # A new config appears. rev = config.RealmsCfgRev(project_id=project_id, config_rev='cfg_rev1', config_digest='digest1', config_body='realms{ name: "realm1" }', perms_rev=None) config.update_realms(fake_db('db-rev1'), [rev], 'New config') # Generated new AuthDB revisions. self.assertEqual(model.get_auth_db_revision(), 1) # Stored now in the expanded form. ent = model.project_realms_key(project_id).get() self.assertEqual([r.name for r in ent.realms.realms], ['%s:@root' % project_id, '%s:realm1' % project_id]) self.assertEqual(ent.config_rev, 'cfg_rev1') self.assertEqual(ent.perms_rev, 'db-rev1') # Permissions DB changes in a way that doesn't affect the expanded form. config.update_realms(fake_db('db-rev2'), [rev], 'Reeval') # Seeing the same AuthDB version. self.assertEqual(model.get_auth_db_revision(), 1) # The config body changes in a way that doesn't affect the expanded form. rev = config.RealmsCfgRev( project_id=project_id, config_rev='cfg_rev2', config_digest='digest2', config_body='realms{ name: "realm1" } # blah blah', perms_rev=None) config.update_realms(fake_db('db-rev2'), [rev], 'Updated config') # Still the same AuthDB version. self.assertEqual(model.get_auth_db_revision(), 1) # The config change significantly now. rev = config.RealmsCfgRev(project_id=project_id, config_rev='cfg_rev3', config_digest='digest3', config_body='realms{ name: "realm2" }', perms_rev=None) config.update_realms(fake_db('db-rev2'), [rev], 'Updated config') # New revision. self.assertEqual(model.get_auth_db_revision(), 2) # And new body. ent = model.project_realms_key(project_id).get() self.assertEqual([r.name for r in ent.realms.realms], ['%s:@root' % project_id, '%s:realm2' % project_id]) self.assertEqual(ent.config_rev, 'cfg_rev3') self.assertEqual(ent.perms_rev, 'db-rev2') # The config is gone. config.delete_realms(project_id) # This generated a new revision. self.assertEqual(model.get_auth_db_revision(), 3) # And it is indeed gone. ent = model.project_realms_key(project_id).get() self.assertIsNone(ent) # The second deletion is noop. config.delete_realms(project_id) self.assertEqual(model.get_auth_db_revision(), 3)
def project_realms_meta_key(project_id): """An ndb.Key for an AuthProjectRealmsMeta entity.""" return ndb.Key( AuthProjectRealmsMeta, 'meta', parent=model.project_realms_key(project_id))
def test_realms_serialization(self): """Serializing snapshot with non-trivial realms configs.""" realms_globals = model.AuthRealmsGlobals( key=model.realms_globals_key(), permissions=[ realms_pb2.Permission(name='luci.dev.p1'), realms_pb2.Permission(name='luci.dev.p2'), ], ) p1 = model.AuthProjectRealms( key=model.project_realms_key('proj1'), realms=realms_pb2.Realms( permissions=[{ 'name': 'luci.dev.p2' }], realms=[{ 'name': 'proj1:@root', 'bindings': [ { 'permissions': [0], 'principals': ['group:gr1'], }, ], }], ), ) p2 = model.AuthProjectRealms( key=model.project_realms_key('proj2'), realms=realms_pb2.Realms( permissions=[{ 'name': 'luci.dev.p1' }], realms=[{ 'name': 'proj2:@root', 'bindings': [ { 'permissions': [0], 'principals': ['group:gr2'], }, ], }], ), ) auth_db = make_auth_db_proto(realms_globals=realms_globals, project_realms=[p1, p2]) self.assertEqual( auth_db.realms, realms_pb2.Realms( api_version=realms.API_VERSION, permissions=[{ 'name': 'luci.dev.p1' }, { 'name': 'luci.dev.p2' }], realms=[ { 'name': 'proj1:@root', 'bindings': [ { 'permissions': [1], 'principals': ['group:gr1'], }, ], }, { 'name': 'proj2:@root', 'bindings': [ { 'permissions': [0], 'principals': ['group:gr2'], }, ], }, ], ))
def test_non_empty(self): self.mock_now(datetime.datetime(2014, 1, 1, 1, 1, 1)) state = model.AuthReplicationState(key=model.replication_state_key(), primary_id='blah', primary_url='https://blah', auth_db_rev=123) state.put() global_config = model.AuthGlobalConfig( key=model.root_key(), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]'), oauth_client_id='oauth_client_id', oauth_client_secret='oauth_client_secret', oauth_additional_client_ids=['a', 'b'], token_server_url='https://token-server', security_config='security config blob') global_config.put() group = model.AuthGroup( key=model.group_key('Some group'), members=[model.Identity.from_bytes('user:[email protected]')], globs=[model.IdentityGlob.from_bytes('user:*@example.com')], nested=[], description='Some description', owners='owning-group', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) group.put() another = model.AuthGroup(key=model.group_key('Another group'), nested=['Some group']) another.put() ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('bots'), subnets=['127.0.0.1/32'], description='Some description', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) ip_whitelist.put() ip_whitelist_assignments = model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]'), assignments=[ model.AuthIPWhitelistAssignments.Assignment( identity=model.Identity.from_bytes( 'user:[email protected]'), ip_whitelist='bots', comment='some comment', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes( 'user:[email protected]')), ]) ip_whitelist_assignments.put() realms_globals = model.AuthRealmsGlobals( key=model.realms_globals_key(), permissions=[ realms_pb2.Permission(name='luci.dev.p1'), realms_pb2.Permission(name='luci.dev.p2'), ]) realms_globals.put() model.AuthProjectRealms(key=model.project_realms_key('proj_id1'), realms=realms_pb2.Realms(api_version=1234), config_rev='rev1', perms_rev='rev1').put() model.AuthProjectRealms(key=model.project_realms_key('proj_id2'), realms=realms_pb2.Realms(api_version=1234), config_rev='rev2', perms_rev='rev2').put() captured_state, snapshot = replication.new_auth_db_snapshot() expected_state = { 'auth_db_rev': 123, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'primary_id': u'blah', 'primary_url': u'https://blah', 'shard_ids': [], } self.assertEqual(expected_state, captured_state.to_dict()) expected_snapshot = { 'global_config': { '__id__': 'root', '__parent__': None, 'auth_db_rev': None, 'auth_db_prev_rev': None, 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'oauth_additional_client_ids': [u'a', u'b'], 'oauth_client_id': u'oauth_client_id', 'oauth_client_secret': u'oauth_client_secret', 'security_config': 'security config blob', 'token_server_url': u'https://token-server', }, 'groups': [ { '__id__': 'Another group', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': None, 'description': u'', 'globs': [], 'members': [], 'modified_by': None, 'modified_ts': None, 'nested': [u'Some group'], 'owners': u'administrators', }, { '__id__': 'Some group', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'Some description', 'globs': [model.IdentityGlob(kind='user', pattern='*@example.com')], 'members': [model.Identity(kind='user', name='*****@*****.**')], 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'nested': [], 'owners': u'owning-group', }, ], 'ip_whitelists': [ { '__id__': 'bots', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'Some description', 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'subnets': [u'127.0.0.1/32'], }, ], 'ip_whitelist_assignments': { '__id__': 'default', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'assignments': [ { 'comment': u'some comment', 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'bots', }, ], 'auth_db_rev': None, 'auth_db_prev_rev': None, 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), }, 'realms_globals': { '__id__': 'globals', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_prev_rev': None, 'auth_db_rev': None, 'modified_by': None, 'modified_ts': None, 'permissions': [ realms_pb2.Permission(name='luci.dev.p1'), realms_pb2.Permission(name='luci.dev.p2'), ], }, 'project_realms': [{ '__id__': 'proj_id1', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_prev_rev': None, 'auth_db_rev': None, 'config_rev': u'rev1', 'perms_rev': u'rev1', 'modified_by': None, 'modified_ts': None, 'realms': realms_pb2.Realms(api_version=1234), }, { '__id__': 'proj_id2', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_prev_rev': None, 'auth_db_rev': None, 'config_rev': u'rev2', 'perms_rev': u'rev2', 'modified_by': None, 'modified_ts': None, 'realms': realms_pb2.Realms(api_version=1234), }], } self.assertEqual(expected_snapshot, snapshot_to_dict(snapshot))