def update():
existing = ndb.get_multi(
model.project_realms_key(rev.project_id)
for rev, _ in expanded
)
updated = []
metas = []
for (rev, realms), ent in zip(expanded, existing):
logging.info('Visiting project "%s"...', rev.project_id)
if not ent:
logging.info('New realms config in project "%s"', rev.project_id)
ent = model.AuthProjectRealms(
key=model.project_realms_key(rev.project_id),
realms=realms,
config_rev=rev.config_rev,
perms_rev=db.revision)
ent.record_revision(
modified_by=model.get_service_self_identity(),
comment='New realms config')
updated.append(ent)
elif ent.realms != realms:
logging.info('Updated realms config in project "%s"', rev.project_id)
ent.realms = realms
ent.config_rev = rev.config_rev
ent.perms_rev = db.revision
ent.record_revision(
modified_by=model.get_service_self_identity(),
comment=comment)
updated.append(ent)
else:
logging.info('Realms config in project "%s" are fresh', rev.project_id)
# Always update AuthProjectRealmsMeta to match the state we just checked.
metas.append(AuthProjectRealmsMeta(
key=project_realms_meta_key(rev.project_id),
config_rev=rev.config_rev,
perms_rev=db.revision,
config_digest=rev.config_digest,
modified_ts=utils.utcnow(),
))
logging.info('Persisting changes...')
ndb.put_multi(updated + metas)
if updated:
model.replicate_auth_db()
def delete_realms(project_id):
"""Performs an AuthDB transaction that deletes all realms of some project.
Args:
project_id: ID of the project being deleted.
"""
realms = model.project_realms_key(project_id).get()
if not realms:
return # already gone
realms.record_deletion(
modified_by=model.get_service_self_identity(),
comment='No longer in the configs')
realms.key.delete()
project_realms_meta_key(project_id).delete()
model.replicate_auth_db()
def test_update_many_projects(self):
self.assertEqual(model.get_auth_db_revision(), 0)
cfg_rev = lambda proj, realm, rev_sfx: config.RealmsCfgRev(
project_id=proj,
config_rev='cfg-rev-' + rev_sfx,
config_digest='digest-' + rev_sfx,
config_body='realms{ name: "%s" }' % realm,
perms_rev=None)
# Create a bunch of project configs at once.
config.update_realms(fake_db('db-rev1'), [
cfg_rev('proj1', 'realm1', 'p1s1'),
cfg_rev('proj2', 'realm1', 'p2s1'),
], 'New config')
# Produced a single revision.
self.assertEqual(model.get_auth_db_revision(), 1)
# Present now.
revs = config.get_stored_revs_async().get_result()
self.assertEqual(revs, [
config.RealmsCfgRev(
project_id='proj1',
config_rev=u'cfg-rev-p1s1',
config_digest=u'digest-p1s1',
config_body=None,
perms_rev=u'db-rev1',
),
config.RealmsCfgRev(
project_id='proj2',
config_rev=u'cfg-rev-p2s1',
config_digest=u'digest-p2s1',
config_body=None,
perms_rev=u'db-rev1',
),
])
self.assertEqual(
model.project_realms_key('proj1').get().config_rev, 'cfg-rev-p1s1')
self.assertEqual(
model.project_realms_key('proj2').get().config_rev, 'cfg-rev-p2s1')
# One is modified significantly, another not.
config.update_realms(
fake_db('db-rev1'),
[
cfg_rev('proj1', 'realm1', 'p1s2'), # noop change
cfg_rev('proj2', 'realm2', 'p2s2'), # significant change
],
'New config')
revs = config.get_stored_revs_async().get_result()
self.assertEqual(
model.project_realms_key('proj1').get().config_rev, 'cfg-rev-p1s1')
self.assertEqual(
model.project_realms_key('proj2').get().config_rev, 'cfg-rev-p2s2')
# One config is broken.
config.update_realms(fake_db('db-rev1'), [
cfg_rev('proj1', 'realm3', 'p1s3'),
cfg_rev('proj2', '@@@@@@', 'p2s3'),
], 'New config')
revs = config.get_stored_revs_async().get_result()
self.assertEqual(
model.project_realms_key('proj1').get().config_rev, 'cfg-rev-p1s3')
self.assertEqual(
model.project_realms_key('proj2').get().config_rev, 'cfg-rev-p2s2')
def test_realms_config_lifecycle(self, project_id):
self.assertEqual(model.get_auth_db_revision(), 0)
# A new config appears.
rev = config.RealmsCfgRev(project_id=project_id,
config_rev='cfg_rev1',
config_digest='digest1',
config_body='realms{ name: "realm1" }',
perms_rev=None)
config.update_realms(fake_db('db-rev1'), [rev], 'New config')
# Generated new AuthDB revisions.
self.assertEqual(model.get_auth_db_revision(), 1)
# Stored now in the expanded form.
ent = model.project_realms_key(project_id).get()
self.assertEqual([r.name for r in ent.realms.realms],
['%s:@root' % project_id,
'%s:realm1' % project_id])
self.assertEqual(ent.config_rev, 'cfg_rev1')
self.assertEqual(ent.perms_rev, 'db-rev1')
# Permissions DB changes in a way that doesn't affect the expanded form.
config.update_realms(fake_db('db-rev2'), [rev], 'Reeval')
# Seeing the same AuthDB version.
self.assertEqual(model.get_auth_db_revision(), 1)
# The config body changes in a way that doesn't affect the expanded form.
rev = config.RealmsCfgRev(
project_id=project_id,
config_rev='cfg_rev2',
config_digest='digest2',
config_body='realms{ name: "realm1" } # blah blah',
perms_rev=None)
config.update_realms(fake_db('db-rev2'), [rev], 'Updated config')
# Still the same AuthDB version.
self.assertEqual(model.get_auth_db_revision(), 1)
# The config change significantly now.
rev = config.RealmsCfgRev(project_id=project_id,
config_rev='cfg_rev3',
config_digest='digest3',
config_body='realms{ name: "realm2" }',
perms_rev=None)
config.update_realms(fake_db('db-rev2'), [rev], 'Updated config')
# New revision.
self.assertEqual(model.get_auth_db_revision(), 2)
# And new body.
ent = model.project_realms_key(project_id).get()
self.assertEqual([r.name for r in ent.realms.realms],
['%s:@root' % project_id,
'%s:realm2' % project_id])
self.assertEqual(ent.config_rev, 'cfg_rev3')
self.assertEqual(ent.perms_rev, 'db-rev2')
# The config is gone.
config.delete_realms(project_id)
# This generated a new revision.
self.assertEqual(model.get_auth_db_revision(), 3)
# And it is indeed gone.
ent = model.project_realms_key(project_id).get()
self.assertIsNone(ent)
# The second deletion is noop.
config.delete_realms(project_id)
self.assertEqual(model.get_auth_db_revision(), 3)
def project_realms_meta_key(project_id):
"""An ndb.Key for an AuthProjectRealmsMeta entity."""
return ndb.Key(
AuthProjectRealmsMeta, 'meta',
parent=model.project_realms_key(project_id))
def test_realms_serialization(self):
"""Serializing snapshot with non-trivial realms configs."""
realms_globals = model.AuthRealmsGlobals(
key=model.realms_globals_key(),
permissions=[
realms_pb2.Permission(name='luci.dev.p1'),
realms_pb2.Permission(name='luci.dev.p2'),
],
)
p1 = model.AuthProjectRealms(
key=model.project_realms_key('proj1'),
realms=realms_pb2.Realms(
permissions=[{
'name': 'luci.dev.p2'
}],
realms=[{
'name':
'proj1:@root',
'bindings': [
{
'permissions': [0],
'principals': ['group:gr1'],
},
],
}],
),
)
p2 = model.AuthProjectRealms(
key=model.project_realms_key('proj2'),
realms=realms_pb2.Realms(
permissions=[{
'name': 'luci.dev.p1'
}],
realms=[{
'name':
'proj2:@root',
'bindings': [
{
'permissions': [0],
'principals': ['group:gr2'],
},
],
}],
),
)
auth_db = make_auth_db_proto(realms_globals=realms_globals,
project_realms=[p1, p2])
self.assertEqual(
auth_db.realms,
realms_pb2.Realms(
api_version=realms.API_VERSION,
permissions=[{
'name': 'luci.dev.p1'
}, {
'name': 'luci.dev.p2'
}],
realms=[
{
'name':
'proj1:@root',
'bindings': [
{
'permissions': [1],
'principals': ['group:gr1'],
},
],
},
{
'name':
'proj2:@root',
'bindings': [
{
'permissions': [0],
'principals': ['group:gr2'],
},
],
},
],
))
def test_non_empty(self):
self.mock_now(datetime.datetime(2014, 1, 1, 1, 1, 1))
state = model.AuthReplicationState(key=model.replication_state_key(),
primary_id='blah',
primary_url='https://blah',
auth_db_rev=123)
state.put()
global_config = model.AuthGlobalConfig(
key=model.root_key(),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'),
oauth_client_id='oauth_client_id',
oauth_client_secret='oauth_client_secret',
oauth_additional_client_ids=['a', 'b'],
token_server_url='https://token-server',
security_config='security config blob')
global_config.put()
group = model.AuthGroup(
key=model.group_key('Some group'),
members=[model.Identity.from_bytes('user:[email protected]')],
globs=[model.IdentityGlob.from_bytes('user:*@example.com')],
nested=[],
description='Some description',
owners='owning-group',
created_ts=utils.utcnow(),
created_by=model.Identity.from_bytes('user:[email protected]'),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'))
group.put()
another = model.AuthGroup(key=model.group_key('Another group'),
nested=['Some group'])
another.put()
ip_whitelist = model.AuthIPWhitelist(
key=model.ip_whitelist_key('bots'),
subnets=['127.0.0.1/32'],
description='Some description',
created_ts=utils.utcnow(),
created_by=model.Identity.from_bytes('user:[email protected]'),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'))
ip_whitelist.put()
ip_whitelist_assignments = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=model.Identity.from_bytes(
'user:[email protected]'),
ip_whitelist='bots',
comment='some comment',
created_ts=utils.utcnow(),
created_by=model.Identity.from_bytes(
'user:[email protected]')),
])
ip_whitelist_assignments.put()
realms_globals = model.AuthRealmsGlobals(
key=model.realms_globals_key(),
permissions=[
realms_pb2.Permission(name='luci.dev.p1'),
realms_pb2.Permission(name='luci.dev.p2'),
])
realms_globals.put()
model.AuthProjectRealms(key=model.project_realms_key('proj_id1'),
realms=realms_pb2.Realms(api_version=1234),
config_rev='rev1',
perms_rev='rev1').put()
model.AuthProjectRealms(key=model.project_realms_key('proj_id2'),
realms=realms_pb2.Realms(api_version=1234),
config_rev='rev2',
perms_rev='rev2').put()
captured_state, snapshot = replication.new_auth_db_snapshot()
expected_state = {
'auth_db_rev': 123,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'primary_id': u'blah',
'primary_url': u'https://blah',
'shard_ids': [],
}
self.assertEqual(expected_state, captured_state.to_dict())
expected_snapshot = {
'global_config': {
'__id__':
'root',
'__parent__':
None,
'auth_db_rev':
None,
'auth_db_prev_rev':
None,
'modified_by':
model.Identity(kind='user', name='*****@*****.**'),
'modified_ts':
datetime.datetime(2014, 1, 1, 1, 1, 1),
'oauth_additional_client_ids': [u'a', u'b'],
'oauth_client_id':
u'oauth_client_id',
'oauth_client_secret':
u'oauth_client_secret',
'security_config':
'security config blob',
'token_server_url':
u'https://token-server',
},
'groups': [
{
'__id__': 'Another group',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': None,
'description': u'',
'globs': [],
'members': [],
'modified_by': None,
'modified_ts': None,
'nested': [u'Some group'],
'owners': u'administrators',
},
{
'__id__':
'Some group',
'__parent__':
ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev':
None,
'auth_db_prev_rev':
None,
'created_by':
model.Identity(kind='user', name='*****@*****.**'),
'created_ts':
datetime.datetime(2014, 1, 1, 1, 1, 1),
'description':
u'Some description',
'globs':
[model.IdentityGlob(kind='user', pattern='*@example.com')],
'members':
[model.Identity(kind='user', name='*****@*****.**')],
'modified_by':
model.Identity(kind='user', name='*****@*****.**'),
'modified_ts':
datetime.datetime(2014, 1, 1, 1, 1, 1),
'nested': [],
'owners':
u'owning-group',
},
],
'ip_whitelists': [
{
'__id__':
'bots',
'__parent__':
ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev':
None,
'auth_db_prev_rev':
None,
'created_by':
model.Identity(kind='user', name='*****@*****.**'),
'created_ts':
datetime.datetime(2014, 1, 1, 1, 1, 1),
'description':
u'Some description',
'modified_by':
model.Identity(kind='user', name='*****@*****.**'),
'modified_ts':
datetime.datetime(2014, 1, 1, 1, 1, 1),
'subnets': [u'127.0.0.1/32'],
},
],
'ip_whitelist_assignments': {
'__id__':
'default',
'__parent__':
ndb.Key('AuthGlobalConfig', 'root'),
'assignments': [
{
'comment':
u'some comment',
'created_by':
model.Identity(kind='user',
name='*****@*****.**'),
'created_ts':
datetime.datetime(2014, 1, 1, 1, 1, 1),
'identity':
model.Identity(kind='user',
name='*****@*****.**'),
'ip_whitelist':
u'bots',
},
],
'auth_db_rev':
None,
'auth_db_prev_rev':
None,
'modified_by':
model.Identity(kind='user', name='*****@*****.**'),
'modified_ts':
datetime.datetime(2014, 1, 1, 1, 1, 1),
},
'realms_globals': {
'__id__':
'globals',
'__parent__':
ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_prev_rev':
None,
'auth_db_rev':
None,
'modified_by':
None,
'modified_ts':
None,
'permissions': [
realms_pb2.Permission(name='luci.dev.p1'),
realms_pb2.Permission(name='luci.dev.p2'),
],
},
'project_realms': [{
'__id__':
'proj_id1',
'__parent__':
ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_prev_rev':
None,
'auth_db_rev':
None,
'config_rev':
u'rev1',
'perms_rev':
u'rev1',
'modified_by':
None,
'modified_ts':
None,
'realms':
realms_pb2.Realms(api_version=1234),
}, {
'__id__':
'proj_id2',
'__parent__':
ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_prev_rev':
None,
'auth_db_rev':
None,
'config_rev':
u'rev2',
'perms_rev':
u'rev2',
'modified_by':
None,
'modified_ts':
None,
'realms':
realms_pb2.Realms(api_version=1234),
}],
}
self.assertEqual(expected_snapshot, snapshot_to_dict(snapshot))