def test_works(self):
# Initial entities. Don't call 'record_revision' to imitate "old"
# application without history related code.
@ndb.transactional
def make_auth_db():
model.AuthGlobalConfig(key=model.root_key()).put()
model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key()).put()
model.AuthGroup(key=model.group_key('A group')).put()
model.AuthIPWhitelist(
key=model.ip_whitelist_key('A whitelist')).put()
model.replicate_auth_db()
make_auth_db()
# Bump auth_db once more to avoid hitting trivial case of "processing first
# revision ever".
auth_db_rev = ndb.transaction(model.replicate_auth_db)
self.assertEqual(2, auth_db_rev)
# Now do the work.
change_log.ensure_initial_snapshot(auth_db_rev)
# Generated new AuthDB rev with updated entities.
self.assertEqual(3, model.get_auth_db_revision())
# Check all *History entitites exist now.
p = model.historical_revision_key(3)
self.assertIsNotNone(
ndb.Key('AuthGlobalConfigHistory', 'root', parent=p).get())
self.assertIsNotNone(
ndb.Key('AuthIPWhitelistAssignmentsHistory', 'default',
parent=p).get())
self.assertIsNotNone(
ndb.Key('AuthGroupHistory', 'A group', parent=p).get())
self.assertIsNotNone(
ndb.Key('AuthIPWhitelistHistory', 'A whitelist', parent=p).get())
# Call again, should be noop (marker is set).
change_log.ensure_initial_snapshot(3)
self.assertEqual(3, model.get_auth_db_revision())
def test_works(self):
# Initial entities. Don't call 'record_revision' to imitate "old"
# application without history related code.
@ndb.transactional
def make_auth_db():
model.AuthGlobalConfig(key=model.root_key()).put()
model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key()).put()
model.AuthGroup(key=model.group_key('A group')).put()
model.AuthIPWhitelist(key=model.ip_whitelist_key('A whitelist')).put()
model.replicate_auth_db()
make_auth_db()
# Bump auth_db once more to avoid hitting trivial case of "processing first
# revision ever".
auth_db_rev = ndb.transaction(model.replicate_auth_db)
self.assertEqual(2, auth_db_rev)
# Now do the work.
change_log.ensure_initial_snapshot(auth_db_rev)
# Generated new AuthDB rev with updated entities.
self.assertEqual(3, model.get_auth_db_revision())
# Check all *History entitites exist now.
p = model.historical_revision_key(3)
self.assertIsNotNone(
ndb.Key('AuthGlobalConfigHistory', 'root', parent=p).get())
self.assertIsNotNone(
ndb.Key(
'AuthIPWhitelistAssignmentsHistory', 'default', parent=p).get())
self.assertIsNotNone(ndb.Key('AuthGroupHistory', 'A group', parent=p).get())
self.assertIsNotNone(
ndb.Key('AuthIPWhitelistHistory', 'A whitelist', parent=p).get())
# Call again, should be noop (marker is set).
change_log.ensure_initial_snapshot(3)
self.assertEqual(3, model.get_auth_db_revision())
def test_works(self):
def perms_from_authdb():
e = model.realms_globals_key().get()
return [p.name for p in e.permissions] if e else []
# The initial state.
self.assertEqual(model.get_auth_db_revision(), 0)
self.assertEqual(perms_from_authdb(), [])
# Create the initial copy of AuthRealmsGlobals.
self.call(fake_db('rev1', ['luci.dev.p1', 'luci.dev.p2']))
self.assertEqual(model.get_auth_db_revision(), 1)
self.assertEqual(perms_from_authdb(), ['luci.dev.p1', 'luci.dev.p2'])
# Noop change.
self.call(fake_db('rev1', ['luci.dev.p1', 'luci.dev.p2']))
self.assertEqual(model.get_auth_db_revision(), 1)
self.assertEqual(perms_from_authdb(), ['luci.dev.p1', 'luci.dev.p2'])
# Real change.
self.call(fake_db('rev2', ['luci.dev.p3']))
self.assertEqual(model.get_auth_db_revision(), 2)
self.assertEqual(perms_from_authdb(), ['luci.dev.p3'])
def test_import_external_groups(self):
self.mock_now(datetime.datetime(2010, 1, 2, 3, 4, 5, 6))
importer.write_config("""
tarball {
domain: "example.com"
groups: "ldap/new"
oauth_scopes: "scope"
systems: "ldap"
url: "https://fake_tarball"
}
plainlist {
group: "external_1"
oauth_scopes: "scope"
url: "https://fake_external_1"
}
plainlist {
domain: "example.com"
group: "external_2"
oauth_scopes: "scope"
url: "https://fake_external_2"
}
""")
self.mock_urlfetch({
'https://fake_tarball':
build_tar_gz({
'ldap/new': 'a\nb',
}),
'https://fake_external_1':
'[email protected]\[email protected]\n',
'https://fake_external_2':
'123\n456',
})
# Should be deleted during import, since not in a imported bundle.
group('ldap/deleted', []).put()
# Should be updated.
group('external/external_1', ['x', 'y']).put()
# Should be removed, since not in list of external groups.
group('external/deleted', []).put()
# Run the import.
initial_auth_db_rev = model.get_auth_db_revision()
importer.import_external_groups()
self.assertEqual(initial_auth_db_rev + 1, model.get_auth_db_revision())
# Verify final state.
expected_groups = {
'ldap/new': {
'auth_db_rev': 1,
'auth_db_prev_rev': None,
'created_by': model.get_service_self_identity(),
'created_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'description': u'',
'globs': [],
'members': [ident('a'), ident('b')],
'modified_by': model.get_service_self_identity(),
'modified_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'nested': [],
'owners': u'administrators',
},
'external/external_1': {
'auth_db_rev': 1,
'auth_db_prev_rev': None,
'created_by': ident('admin'),
'created_ts': datetime.datetime(1999, 1, 2, 3, 4, 5, 6),
'description': u'',
'globs': [],
'members': [ident('*****@*****.**'),
ident('*****@*****.**')],
'modified_by': model.get_service_self_identity(),
'modified_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'nested': [],
'owners': u'administrators',
},
'external/external_2': {
'auth_db_rev': 1,
'auth_db_prev_rev': None,
'created_by': model.get_service_self_identity(),
'created_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'description': u'',
'globs': [],
'members': [ident('123'), ident('456')],
'modified_by': model.get_service_self_identity(),
'modified_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'nested': [],
'owners': u'administrators',
},
}
self.assertEqual(expected_groups, fetch_groups())
def test_update_many_projects(self):
self.assertEqual(model.get_auth_db_revision(), 0)
cfg_rev = lambda proj, realm, rev_sfx: config.RealmsCfgRev(
project_id=proj,
config_rev='cfg-rev-' + rev_sfx,
config_digest='digest-' + rev_sfx,
config_body='realms{ name: "%s" }' % realm,
perms_rev=None)
# Create a bunch of project configs at once.
config.update_realms(fake_db('db-rev1'), [
cfg_rev('proj1', 'realm1', 'p1s1'),
cfg_rev('proj2', 'realm1', 'p2s1'),
], 'New config')
# Produced a single revision.
self.assertEqual(model.get_auth_db_revision(), 1)
# Present now.
revs = config.get_stored_revs_async().get_result()
self.assertEqual(revs, [
config.RealmsCfgRev(
project_id='proj1',
config_rev=u'cfg-rev-p1s1',
config_digest=u'digest-p1s1',
config_body=None,
perms_rev=u'db-rev1',
),
config.RealmsCfgRev(
project_id='proj2',
config_rev=u'cfg-rev-p2s1',
config_digest=u'digest-p2s1',
config_body=None,
perms_rev=u'db-rev1',
),
])
self.assertEqual(
model.project_realms_key('proj1').get().config_rev, 'cfg-rev-p1s1')
self.assertEqual(
model.project_realms_key('proj2').get().config_rev, 'cfg-rev-p2s1')
# One is modified significantly, another not.
config.update_realms(
fake_db('db-rev1'),
[
cfg_rev('proj1', 'realm1', 'p1s2'), # noop change
cfg_rev('proj2', 'realm2', 'p2s2'), # significant change
],
'New config')
revs = config.get_stored_revs_async().get_result()
self.assertEqual(
model.project_realms_key('proj1').get().config_rev, 'cfg-rev-p1s1')
self.assertEqual(
model.project_realms_key('proj2').get().config_rev, 'cfg-rev-p2s2')
# One config is broken.
config.update_realms(fake_db('db-rev1'), [
cfg_rev('proj1', 'realm3', 'p1s3'),
cfg_rev('proj2', '@@@@@@', 'p2s3'),
], 'New config')
revs = config.get_stored_revs_async().get_result()
self.assertEqual(
model.project_realms_key('proj1').get().config_rev, 'cfg-rev-p1s3')
self.assertEqual(
model.project_realms_key('proj2').get().config_rev, 'cfg-rev-p2s2')
def test_realms_config_lifecycle(self, project_id):
self.assertEqual(model.get_auth_db_revision(), 0)
# A new config appears.
rev = config.RealmsCfgRev(project_id=project_id,
config_rev='cfg_rev1',
config_digest='digest1',
config_body='realms{ name: "realm1" }',
perms_rev=None)
config.update_realms(fake_db('db-rev1'), [rev], 'New config')
# Generated new AuthDB revisions.
self.assertEqual(model.get_auth_db_revision(), 1)
# Stored now in the expanded form.
ent = model.project_realms_key(project_id).get()
self.assertEqual([r.name for r in ent.realms.realms],
['%s:@root' % project_id,
'%s:realm1' % project_id])
self.assertEqual(ent.config_rev, 'cfg_rev1')
self.assertEqual(ent.perms_rev, 'db-rev1')
# Permissions DB changes in a way that doesn't affect the expanded form.
config.update_realms(fake_db('db-rev2'), [rev], 'Reeval')
# Seeing the same AuthDB version.
self.assertEqual(model.get_auth_db_revision(), 1)
# The config body changes in a way that doesn't affect the expanded form.
rev = config.RealmsCfgRev(
project_id=project_id,
config_rev='cfg_rev2',
config_digest='digest2',
config_body='realms{ name: "realm1" } # blah blah',
perms_rev=None)
config.update_realms(fake_db('db-rev2'), [rev], 'Updated config')
# Still the same AuthDB version.
self.assertEqual(model.get_auth_db_revision(), 1)
# The config change significantly now.
rev = config.RealmsCfgRev(project_id=project_id,
config_rev='cfg_rev3',
config_digest='digest3',
config_body='realms{ name: "realm2" }',
perms_rev=None)
config.update_realms(fake_db('db-rev2'), [rev], 'Updated config')
# New revision.
self.assertEqual(model.get_auth_db_revision(), 2)
# And new body.
ent = model.project_realms_key(project_id).get()
self.assertEqual([r.name for r in ent.realms.realms],
['%s:@root' % project_id,
'%s:realm2' % project_id])
self.assertEqual(ent.config_rev, 'cfg_rev3')
self.assertEqual(ent.perms_rev, 'db-rev2')
# The config is gone.
config.delete_realms(project_id)
# This generated a new revision.
self.assertEqual(model.get_auth_db_revision(), 3)
# And it is indeed gone.
ent = model.project_realms_key(project_id).get()
self.assertIsNone(ent)
# The second deletion is noop.
config.delete_realms(project_id)
self.assertEqual(model.get_auth_db_revision(), 3)
def test_import_external_groups(self):
self.mock_now(datetime.datetime(2010, 1, 2, 3, 4, 5, 6))
importer.write_config("""
tarball {
domain: "example.com"
groups: "ldap/new"
oauth_scopes: "scope"
systems: "ldap"
url: "https://fake_tarball"
}
plainlist {
group: "external_1"
oauth_scopes: "scope"
url: "https://fake_external_1"
}
plainlist {
domain: "example.com"
group: "external_2"
oauth_scopes: "scope"
url: "https://fake_external_2"
}
""")
self.mock_urlfetch({
'https://fake_tarball': build_tar_gz({
'ldap/new': 'a\nb',
}),
'https://fake_external_1': '[email protected]\[email protected]\n',
'https://fake_external_2': '123\n456',
})
# Should be deleted during import, since not in a imported bundle.
group('ldap/deleted', []).put()
# Should be updated.
group('external/external_1', ['x', 'y']).put()
# Should be removed, since not in list of external groups.
group('external/deleted', []).put()
# Run the import.
initial_auth_db_rev = model.get_auth_db_revision()
importer.import_external_groups()
self.assertEqual(initial_auth_db_rev + 1, model.get_auth_db_revision())
# Verify final state.
expected_groups = {
'ldap/new': {
'auth_db_rev': 1,
'auth_db_prev_rev': None,
'created_by': model.get_service_self_identity(),
'created_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'description': u'',
'globs': [],
'members': [ident('a'), ident('b')],
'modified_by': model.get_service_self_identity(),
'modified_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'nested': [],
'owners': u'administrators',
},
'external/external_1': {
'auth_db_rev': 1,
'auth_db_prev_rev': None,
'created_by': ident('admin'),
'created_ts': datetime.datetime(1999, 1, 2, 3, 4, 5, 6),
'description': u'',
'globs': [],
'members': [ident('*****@*****.**'), ident('*****@*****.**')],
'modified_by': model.get_service_self_identity(),
'modified_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'nested': [],
'owners': u'administrators',
},
'external/external_2': {
'auth_db_rev': 1,
'auth_db_prev_rev': None,
'created_by': model.get_service_self_identity(),
'created_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'description': u'',
'globs': [],
'members': [ident('123'), ident('456')],
'modified_by': model.get_service_self_identity(),
'modified_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'nested': [],
'owners': u'administrators',
},
}
self.assertEqual(expected_groups, fetch_groups())
def test_import_external_groups(self):
self.mock_now(datetime.datetime(2010, 1, 2, 3, 4, 5, 6))
service_id = auth.Identity.from_bytes('service:some-service')
self.mock(auth, 'get_service_self_identity', lambda: service_id)
importer.write_config([
{
'domain': 'example.com',
'format': 'tarball',
'groups': ['ldap/new'],
'oauth_scopes': ['scope'],
'systems': ['ldap'],
'url': 'https://fake_tarball',
},
{
'format': 'plainlist',
'group': 'external_1',
'oauth_scopes': ['scope'],
'url': 'https://fake_external_1',
},
{
'description': 'Some external group',
'domain': 'example.com',
'format': 'plainlist',
'group': 'external_2',
'oauth_scopes': ['scope'],
'url': 'https://fake_external_2',
},
])
self.mock_urlfetch({
'https://fake_tarball': build_tar_gz({
'ldap/new': 'a\nb',
}),
'https://fake_external_1': '[email protected]\[email protected]\n',
'https://fake_external_2': '123\n456',
})
# Should be deleted during import, since not in a imported bundle.
group('ldap/deleted', []).put()
# Should be updated.
group('external/external_1', ['x', 'y']).put()
# Should be removed, since not in list of external groups.
group('external/deleted', []).put()
# Run the import.
initial_auth_db_rev = model.get_auth_db_revision()
importer.import_external_groups()
self.assertEqual(initial_auth_db_rev + 1, model.get_auth_db_revision())
# Verify final state.
expected_groups = {
'ldap/new': {
'created_by': service_id,
'created_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'description': u'',
'globs': [],
'members': [ident('a'), ident('b')],
'modified_by': service_id,
'modified_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'nested': [],
},
'external/external_1': {
'created_by': ident('admin'),
'created_ts': datetime.datetime(1999, 1, 2, 3, 4, 5, 6),
'description': u'',
'globs': [],
'members': [ident('*****@*****.**'), ident('*****@*****.**')],
'modified_by': service_id,
'modified_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'nested': [],
},
'external/external_2': {
'created_by': service_id,
'created_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'description': u'',
'globs': [],
'members': [ident('123'), ident('456')],
'modified_by': service_id,
'modified_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'nested': [],
},
}
self.assertEqual(expected_groups, fetch_groups())
def test_import_external_groups(self):
self.mock_now(datetime.datetime(2010, 1, 2, 3, 4, 5, 6))
service_id = auth.Identity.from_bytes('service:some-service')
self.mock(auth, 'get_service_self_identity', lambda: service_id)
importer.write_config([
{
'domain': 'example.com',
'format': 'tarball',
'groups': ['ldap/new'],
'oauth_scopes': ['scope'],
'systems': ['ldap'],
'url': 'https://fake_tarball',
},
{
'format': 'plainlist',
'group': 'external_1',
'oauth_scopes': ['scope'],
'url': 'https://fake_external_1',
},
{
'description': 'Some external group',
'domain': 'example.com',
'format': 'plainlist',
'group': 'external_2',
'oauth_scopes': ['scope'],
'url': 'https://fake_external_2',
},
])
self.mock_urlfetch({
'https://fake_tarball':
build_tar_gz({
'ldap/new': 'a\nb',
}),
'https://fake_external_1':
'[email protected]\[email protected]\n',
'https://fake_external_2':
'123\n456',
})
# Should be deleted during import, since not in a imported bundle.
group('ldap/deleted', []).put()
# Should be updated.
group('external/external_1', ['x', 'y']).put()
# Should be removed, since not in list of external groups.
group('external/deleted', []).put()
# Run the import.
initial_auth_db_rev = model.get_auth_db_revision()
importer.import_external_groups()
self.assertEqual(initial_auth_db_rev + 1, model.get_auth_db_revision())
# Verify final state.
expected_groups = {
'ldap/new': {
'created_by': service_id,
'created_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'description': u'',
'globs': [],
'members': [ident('a'), ident('b')],
'modified_by': service_id,
'modified_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'nested': [],
},
'external/external_1': {
'created_by': ident('admin'),
'created_ts': datetime.datetime(1999, 1, 2, 3, 4, 5, 6),
'description': u'',
'globs': [],
'members': [ident('*****@*****.**'),
ident('*****@*****.**')],
'modified_by': service_id,
'modified_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'nested': [],
},
'external/external_2': {
'created_by': service_id,
'created_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'description': u'',
'globs': [],
'members': [ident('123'), ident('456')],
'modified_by': service_id,
'modified_ts': datetime.datetime(2010, 1, 2, 3, 4, 5, 6),
'nested': [],
},
}
self.assertEqual(expected_groups, fetch_groups())