def test_bootstrap_ip_whitelist_assignment_modify(self):
self.mock_now(datetime.datetime(2014, 01, 01))
ret = model.bootstrap_ip_whitelist_assignment(
model.Identity(model.IDENTITY_USER, '*****@*****.**'),
'some ip whitelist', 'some comment')
self.assertTrue(ret)
ret = model.bootstrap_ip_whitelist_assignment(
model.Identity(model.IDENTITY_USER, '*****@*****.**'),
'another ip whitelist', 'another comment')
self.assertTrue(ret)
self.assertEqual(
{
'assignments': [
{
'comment':
'another comment',
'created_by':
model.get_service_self_identity(),
'created_ts':
datetime.datetime(2014, 01, 01),
'identity':
model.Identity(model.IDENTITY_USER, '*****@*****.**'),
'ip_whitelist':
'another ip whitelist',
},
],
},
model.ip_whitelist_assignments_key().get().to_dict())
def test_bootstrap_ip_whitelist_assignment_modify(self):
self.mock_now(datetime.datetime(2014, 01, 01))
ret = model.bootstrap_ip_whitelist_assignment(
model.Identity(model.IDENTITY_USER, '*****@*****.**'),
'some ip whitelist', 'some comment')
self.assertTrue(ret)
ret = model.bootstrap_ip_whitelist_assignment(
model.Identity(model.IDENTITY_USER, '*****@*****.**'),
'another ip whitelist', 'another comment')
self.assertTrue(ret)
self.assertEqual(
{
'assignments': [
{
'comment': 'another comment',
'created_by': model.get_service_self_identity(),
'created_ts': datetime.datetime(2014, 01, 01),
'identity': model.Identity(model.IDENTITY_USER, '*****@*****.**'),
'ip_whitelist': 'another ip whitelist',
},
],
}, model.ip_whitelist_assignments_key().get().to_dict())
def make_auth_db():
model.AuthGlobalConfig(key=model.root_key()).put()
model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key()).put()
model.AuthGroup(key=model.group_key('A group')).put()
model.AuthIPWhitelist(key=model.ip_whitelist_key('A whitelist')).put()
model.replicate_auth_db()
def touch_all():
make_group(
name='A group',
members=[ident('*****@*****.**'),
ident('*****@*****.**')],
description='Blah',
comment='New group')
make_ip_whitelist(name='An IP whitelist',
subnets=['127.0.0.1/32'],
description='Bluh',
comment='New IP whitelist')
a = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=ident('*****@*****.**'),
ip_whitelist='An IP whitelist')
])
a.record_revision(modified_by=ident('*****@*****.**'),
modified_ts=utils.utcnow(),
comment='New assignment')
a.put()
c = model.AuthGlobalConfig(key=model.root_key(),
oauth_client_id='client_id',
oauth_client_secret='client_secret',
oauth_additional_client_ids=['1', '2'])
c.record_revision(modified_by=ident('*****@*****.**'),
modified_ts=utils.utcnow(),
comment='Config change')
c.put()
def make_auth_db():
model.AuthGlobalConfig(key=model.root_key()).put()
model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key()).put()
model.AuthGroup(key=model.group_key('A group')).put()
model.AuthIPWhitelist(key=model.ip_whitelist_key('A whitelist')).put()
model.replicate_auth_db()
def test_fetch_ip_whitelists_non_empty(self):
ent = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key())
def add(identity, **kwargs):
kwargs['identity'] = model.Identity.from_bytes(identity)
ent.assignments.append(
model.AuthIPWhitelistAssignments.Assignment(**kwargs))
add('user:[email protected]', ip_whitelist='A')
add('user:[email protected]', ip_whitelist='A')
add('user:[email protected]', ip_whitelist='B')
add('user:[email protected]', ip_whitelist='missing')
ent.put()
def store_whitelist(name):
model.AuthIPWhitelist(key=model.ip_whitelist_key(name)).put()
store_whitelist('A')
store_whitelist('B')
store_whitelist('bots')
assignments, whitelists = model.fetch_ip_whitelists()
self.assertEqual(ent.to_dict(), assignments.to_dict())
self.assertEqual(['A', 'B', 'bots'], [e.key.id() for e in whitelists])
def test_ip_whitelist_assignments_serialization(self):
"""Serializing snapshot with non-trivial AuthIPWhitelistAssignments."""
entity = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=model.Identity.from_bytes('user:[email protected]'),
ip_whitelist='some whitelist',
comment='some comment',
created_ts=datetime.datetime(2020, 1, 1, 1, 1, 1),
created_by=model.Identity.from_bytes(
'user:[email protected]'),
),
],
)
auth_db = make_auth_db_proto(ip_whitelist_assignments=entity)
self.assertEqual(list(auth_db.ip_whitelist_assignments), [
replication_pb2.AuthIPWhitelistAssignment(
identity='user:[email protected]',
ip_whitelist='some whitelist',
comment='some comment',
created_ts=1577840461000000,
created_by='user:[email protected]',
)
])
def touch_all():
make_group(
name='A group',
members=[ident('*****@*****.**'), ident('*****@*****.**')],
description='Blah',
comment='New group')
make_ip_whitelist(
name='An IP whitelist',
subnets=['127.0.0.1/32'],
description='Bluh',
comment='New IP whitelist')
a = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=ident('*****@*****.**'),
ip_whitelist='An IP whitelist')
])
a.record_revision(
modified_by=ident('*****@*****.**'),
modified_ts=utils.utcnow(),
comment='New assignment')
a.put()
c = model.AuthGlobalConfig(
key=model.root_key(),
oauth_client_id='client_id',
oauth_client_secret='client_secret',
oauth_additional_client_ids=['1', '2'])
c.record_revision(
modified_by=ident('*****@*****.**'),
modified_ts=utils.utcnow(),
comment='Config change')
c.put()
def test_fetch_auth_db(self):
# Create AuthGlobalConfig.
global_config = model.AuthGlobalConfig(key=model.root_key())
global_config.oauth_client_id = '1'
global_config.oauth_client_secret = 'secret'
global_config.oauth_additional_client_ids = ['2', '3']
global_config.put()
# Create a bunch of (empty) groups.
groups = [
model.AuthGroup(key=model.group_key('Group A')),
model.AuthGroup(key=model.group_key('Group B')),
]
for group in groups:
group.put()
# And a bunch of secrets (local and global).
local_secrets = [
model.AuthSecret.bootstrap('local%d' % i, 'local')
for i in (0, 1, 2)
]
global_secrets = [
model.AuthSecret.bootstrap('global%d' % i, 'global')
for i in (0, 1, 2)
]
# And IP whitelist.
ip_whitelist_assignments = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=model.Anonymous,
ip_whitelist='some ip whitelist',
),
])
ip_whitelist_assignments.put()
some_ip_whitelist = model.AuthIPWhitelist(
key=model.ip_whitelist_key('some ip whitelist'),
subnets=['127.0.0.1/32'])
bots_ip_whitelist = model.AuthIPWhitelist(
key=model.ip_whitelist_key('bots'), subnets=['127.0.0.1/32'])
some_ip_whitelist.put()
bots_ip_whitelist.put()
# This all stuff should be fetched into AuthDB.
auth_db = api.fetch_auth_db()
self.assertEqual(global_config, auth_db.global_config)
self.assertEqual(set(g.key.id() for g in groups), set(auth_db.groups))
self.assertEqual(set(s.key.id() for s in local_secrets),
set(auth_db.secrets['local']))
self.assertEqual(set(s.key.id() for s in global_secrets),
set(auth_db.secrets['global']))
self.assertEqual(ip_whitelist_assignments,
auth_db.ip_whitelist_assignments)
self.assertEqual(
{
'bots': bots_ip_whitelist,
'some ip whitelist': some_ip_whitelist
}, auth_db.ip_whitelists)
def modify(assignments):
key = model.ip_whitelist_assignments_key()
e = key.get() or model.AuthIPWhitelistAssignments(key=key)
e.record_revision(
modified_by=model.Identity.from_bytes('user:[email protected]'),
modified_ts=datetime.datetime(2015, 1, 1, 1, 1),
comment='Comment')
e.assignments = assignments
e.put()
model.replicate_auth_db()
def modify(assignments):
key = model.ip_whitelist_assignments_key()
e = key.get() or model.AuthIPWhitelistAssignments(key=key)
e.record_revision(
modified_by=model.Identity.from_bytes('user:[email protected]'),
modified_ts=datetime.datetime(2015, 1, 1, 1, 1),
comment='Comment')
e.assignments = assignments
e.put()
model.replicate_auth_db()
def make_snapshot_obj(global_config=None,
groups=None,
ip_whitelists=None,
ip_whitelist_assignments=None):
"""Returns AuthDBSnapshot with omitted fields set to default values."""
return replication.AuthDBSnapshot(
global_config=global_config
or model.AuthGlobalConfig(key=model.root_key()),
groups=groups or [],
ip_whitelists=ip_whitelists or [],
ip_whitelist_assignments=(
ip_whitelist_assignments or model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key())),
)
def change():
a = model.ip_whitelist_assignments_key().get()
a.assignments = [
model.AuthIPWhitelistAssignments.Assignment(
identity=ident('*****@*****.**'),
ip_whitelist='Another IP whitelist'),
model.AuthIPWhitelistAssignments.Assignment(
identity=ident('*****@*****.**'),
ip_whitelist='IP whitelist'),
]
a.record_revision(modified_by=ident('*****@*****.**'),
modified_ts=utils.utcnow(),
comment='change')
a.put()
def change():
a = model.ip_whitelist_assignments_key().get()
a.assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=ident('*****@*****.**'),
ip_whitelist='Another IP whitelist'),
model.AuthIPWhitelistAssignments.Assignment(
identity=ident('*****@*****.**'),
ip_whitelist='IP whitelist'),
]
a.record_revision(
modified_by=ident('*****@*****.**'),
modified_ts=utils.utcnow(),
comment='change')
a.put()
def make_snapshot_obj(
global_config=None, groups=None, secrets=None,
ip_whitelists=None, ip_whitelist_assignments=None):
"""Returns AuthDBSnapshot with omitted fields set to default values."""
return replication.AuthDBSnapshot(
global_config=global_config or model.AuthGlobalConfig(
key=model.root_key()),
groups=groups or [],
secrets=secrets or [],
ip_whitelists=ip_whitelists or [],
ip_whitelist_assignments=(
ip_whitelist_assignments or
model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key())),
)
def create():
a = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=ident('*****@*****.**'),
ip_whitelist='An IP whitelist'),
model.AuthIPWhitelistAssignments.Assignment(
identity=ident('*****@*****.**'),
ip_whitelist='Another IP whitelist'),
])
a.record_revision(modified_by=ident('*****@*****.**'),
modified_ts=utils.utcnow(),
comment='New assignment')
a.put()
def test_ip_whitelist_assignments_serialization(self):
"""Serializing snapshot with non-trivial AuthIPWhitelistAssignments."""
entity = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=model.Identity.from_bytes('user:[email protected]'),
ip_whitelist='some whitelist',
comment='some comment',
created_ts=utils.utcnow(),
created_by=model.Identity.from_bytes('user:[email protected]'),
),
],
)
snapshot = make_snapshot_obj(ip_whitelist_assignments=entity)
self.assert_serialization_works(snapshot)
def test_ip_whitelist_assignments_serialization(self):
"""Serializing snapshot with non-trivial AuthIPWhitelistAssignments."""
entity = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=model.Identity.from_bytes('user:[email protected]'),
ip_whitelist='some whitelist',
comment='some comment',
created_ts=utils.utcnow(),
created_by=model.Identity.from_bytes('user:[email protected]'),
),
],
)
snapshot = make_snapshot_obj(ip_whitelist_assignments=entity)
self.assert_serialization_works(snapshot)
def create():
a = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=ident('*****@*****.**'),
ip_whitelist='An IP whitelist'),
model.AuthIPWhitelistAssignments.Assignment(
identity=ident('*****@*****.**'),
ip_whitelist='Another IP whitelist'),
])
a.record_revision(
modified_by=ident('*****@*****.**'),
modified_ts=utils.utcnow(),
comment='New assignment')
a.put()
def make_snapshot_obj(global_config=None,
groups=None,
ip_whitelists=None,
ip_whitelist_assignments=None):
"""Returns AuthDBSnapshot with empty list of groups and whitelists."""
return replication.AuthDBSnapshot(
global_config=global_config
or model.AuthGlobalConfig(key=model.root_key(),
oauth_client_id='oauth client id',
oauth_client_secret='oauth client secret',
token_server_url='token server',
security_config='security config blob'),
groups=groups or [],
ip_whitelists=ip_whitelists or [],
ip_whitelist_assignments=(
ip_whitelist_assignments or model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key())),
)
def test_fetch_auth_db(self):
# Create AuthGlobalConfig.
global_config = model.AuthGlobalConfig(key=model.root_key())
global_config.oauth_client_id = "1"
global_config.oauth_client_secret = "secret"
global_config.oauth_additional_client_ids = ["2", "3"]
global_config.put()
# Create a bunch of (empty) groups.
groups = [model.AuthGroup(key=model.group_key("Group A")), model.AuthGroup(key=model.group_key("Group B"))]
for group in groups:
group.put()
# And a bunch of secrets (local and global).
local_secrets = [model.AuthSecret.bootstrap("local%d" % i, "local") for i in (0, 1, 2)]
global_secrets = [model.AuthSecret.bootstrap("global%d" % i, "global") for i in (0, 1, 2)]
# And IP whitelist.
ip_whitelist_assignments = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(identity=model.Anonymous, ip_whitelist="some ip whitelist")
],
)
ip_whitelist_assignments.put()
some_ip_whitelist = model.AuthIPWhitelist(
key=model.ip_whitelist_key("some ip whitelist"), subnets=["127.0.0.1/32"]
)
bots_ip_whitelist = model.AuthIPWhitelist(key=model.ip_whitelist_key("bots"), subnets=["127.0.0.1/32"])
some_ip_whitelist.put()
bots_ip_whitelist.put()
# This all stuff should be fetched into AuthDB.
auth_db = api.fetch_auth_db()
self.assertEqual(global_config, auth_db.global_config)
self.assertEqual(set(g.key.id() for g in groups), set(auth_db.groups))
self.assertEqual(set(s.key.id() for s in local_secrets), set(auth_db.secrets["local"]))
self.assertEqual(set(s.key.id() for s in global_secrets), set(auth_db.secrets["global"]))
self.assertEqual(ip_whitelist_assignments, auth_db.ip_whitelist_assignments)
self.assertEqual({"bots": bots_ip_whitelist, "some ip whitelist": some_ip_whitelist}, auth_db.ip_whitelists)
def test_fetch_ip_whitelists_non_empty(self):
ent = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key())
def add(identity, **kwargs):
kwargs['identity'] = model.Identity.from_bytes(identity)
ent.assignments.append(
model.AuthIPWhitelistAssignments.Assignment(**kwargs))
add('user:[email protected]', ip_whitelist='A')
add('user:[email protected]', ip_whitelist='A')
add('user:[email protected]', ip_whitelist='B')
add('user:[email protected]', ip_whitelist='missing')
ent.put()
def store_whitelist(name):
model.AuthIPWhitelist(key=model.ip_whitelist_key(name)).put()
store_whitelist('A')
store_whitelist('B')
store_whitelist('bots')
assignments, whitelists = model.fetch_ip_whitelists()
self.assertEqual(ent.to_dict(), assignments.to_dict())
self.assertEqual(['A', 'B', 'bots'], [e.key.id() for e in whitelists])
def _update_ip_whitelist_config(rev, conf):
assert ndb.in_transaction(), 'Must be called in AuthDB transaction'
now = utils.utcnow()
# Existing whitelist entities.
existing_ip_whitelists = {
e.key.id(): e
for e in model.AuthIPWhitelist.query(ancestor=model.root_key())
}
# Whitelists being imported (name => IPWhitelist proto msg).
imported_ip_whitelists = {msg.name: msg for msg in conf.ip_whitelists}
to_put = []
to_delete = []
# New or modified IP whitelists.
for wl_proto in imported_ip_whitelists.itervalues():
# Convert proto magic list to a regular list.
subnets = list(wl_proto.subnets)
# Existing whitelist and it hasn't changed?
wl = existing_ip_whitelists.get(wl_proto.name)
if wl and wl.subnets == subnets:
continue
# Update existing (to preserve auth_db_prev_rev) or create a new one.
if not wl:
wl = model.AuthIPWhitelist(
key=model.ip_whitelist_key(wl_proto.name),
created_ts=now,
created_by=model.get_service_self_identity())
wl.subnets = subnets
wl.description = 'Imported from ip_whitelist.cfg at rev %s' % rev.revision
to_put.append(wl)
# Removed IP whitelists.
for wl in existing_ip_whitelists.itervalues():
if wl.key.id() not in imported_ip_whitelists:
to_delete.append(wl)
# Update assignments. Don't touch created_ts and created_by for existing ones.
ip_whitelist_assignments = (model.ip_whitelist_assignments_key().get()
or model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key()))
existing = {(a.identity.to_bytes(), a.ip_whitelist): a
for a in ip_whitelist_assignments.assignments}
updated = []
for a in conf.assignments:
key = (a.identity, a.ip_whitelist_name)
if key in existing:
updated.append(existing[key])
else:
new_one = model.AuthIPWhitelistAssignments.Assignment(
identity=model.Identity.from_bytes(a.identity),
ip_whitelist=a.ip_whitelist_name,
comment='Imported from ip_whitelist.cfg at rev %s' %
rev.revision,
created_ts=now,
created_by=model.get_service_self_identity())
updated.append(new_one)
# Something has changed?
updated_keys = [(a.identity.to_bytes(), a.ip_whitelist) for a in updated]
if set(updated_keys) != set(existing):
ip_whitelist_assignments.assignments = updated
to_put.append(ip_whitelist_assignments)
if not to_put and not to_delete:
return False
comment = 'Importing ip_whitelist.cfg at rev %s' % rev.revision
for e in to_put:
e.record_revision(modified_by=model.get_service_self_identity(),
modified_ts=now,
comment=comment)
for e in to_delete:
e.record_deletion(modified_by=model.get_service_self_identity(),
modified_ts=now,
comment=comment)
futures = []
futures.extend(ndb.put_multi_async(to_put))
futures.extend(ndb.delete_multi_async(e.key for e in to_delete))
for f in futures:
f.check_success()
return True
def test_fetch_ip_whitelists_empty(self): assignments, whitelists = model.fetch_ip_whitelists() self.assertEqual(model.ip_whitelist_assignments_key(), assignments.key) self.assertEqual(0, len(assignments.assignments)) self.assertEqual([], whitelists)
def test_fetch_auth_db(self):
# Create AuthGlobalConfig.
global_config = model.AuthGlobalConfig(key=model.root_key())
global_config.oauth_client_id = '1'
global_config.oauth_client_secret = 'secret'
global_config.oauth_additional_client_ids = ['2', '3']
global_config.put()
# Create a bunch of (empty) groups.
groups = [
model.AuthGroup(key=model.group_key('Group A')),
model.AuthGroup(key=model.group_key('Group B')),
]
for group in groups:
group.put()
# And a bunch of secrets (local and global).
local_secrets = [
model.AuthSecret.bootstrap('local%d' % i, 'local') for i in (0, 1, 2)
]
global_secrets = [
model.AuthSecret.bootstrap('global%d' % i, 'global') for i in (0, 1, 2)
]
# And IP whitelist.
ip_whitelist_assignments = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=model.Anonymous,
ip_whitelist='some ip whitelist',
),
])
ip_whitelist_assignments.put()
some_ip_whitelist = model.AuthIPWhitelist(
key=model.ip_whitelist_key('some ip whitelist'),
subnets=['127.0.0.1/32'])
bots_ip_whitelist = model.AuthIPWhitelist(
key=model.ip_whitelist_key('bots'),
subnets=['127.0.0.1/32'])
some_ip_whitelist.put()
bots_ip_whitelist.put()
# This all stuff should be fetched into AuthDB.
auth_db = api.fetch_auth_db()
self.assertEqual(global_config, auth_db.global_config)
self.assertEqual(
set(g.key.id() for g in groups),
set(auth_db.groups))
self.assertEqual(
set(s.key.id() for s in local_secrets),
set(auth_db.secrets['local']))
self.assertEqual(
set(s.key.id() for s in global_secrets),
set(auth_db.secrets['global']))
self.assertEqual(
ip_whitelist_assignments,
auth_db.ip_whitelist_assignments)
self.assertEqual(
{'bots': bots_ip_whitelist, 'some ip whitelist': some_ip_whitelist},
auth_db.ip_whitelists)
def test_non_empty(self):
self.mock_now(datetime.datetime(2014, 1, 1, 1, 1, 1))
state = model.AuthReplicationState(
key=model.replication_state_key(),
primary_id='blah',
primary_url='https://blah',
auth_db_rev=123)
state.put()
global_config = model.AuthGlobalConfig(
key=model.root_key(),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'),
oauth_client_id='oauth_client_id',
oauth_client_secret='oauth_client_secret',
oauth_additional_client_ids=['a', 'b'])
global_config.put()
group = model.AuthGroup(
key=model.group_key('Some group'),
members=[model.Identity.from_bytes('user:[email protected]')],
globs=[model.IdentityGlob.from_bytes('user:*@example.com')],
nested=[],
description='Some description',
created_ts=utils.utcnow(),
created_by=model.Identity.from_bytes('user:[email protected]'),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'))
group.put()
another = model.AuthGroup(
key=model.group_key('Another group'),
nested=['Some group'])
another.put()
global_secret = model.AuthSecret(
id='global_secret',
parent=model.secret_scope_key('global'),
values=['1234', '5678'],
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'))
global_secret.put()
# Local secret should not appear in a snapshot.
local_secret = model.AuthSecret(
id='local_secret',
parent=model.secret_scope_key('local'),
values=['1234', '5678'],
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'))
local_secret.put()
ip_whitelist = model.AuthIPWhitelist(
key=model.ip_whitelist_key('bots'),
subnets=['127.0.0.1/32'],
description='Some description',
created_ts=utils.utcnow(),
created_by=model.Identity.from_bytes('user:[email protected]'),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'))
ip_whitelist.put()
ip_whitelist_assignments = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=model.Identity.from_bytes('user:[email protected]'),
ip_whitelist='bots',
comment='some comment',
created_ts=utils.utcnow(),
created_by=model.Identity.from_bytes('user:[email protected]')),
])
ip_whitelist_assignments.put()
captured_state, snapshot = replication.new_auth_db_snapshot()
expected_state = {
'auth_db_rev': 123,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'primary_id': u'blah',
'primary_url': u'https://blah',
}
self.assertEqual(expected_state, captured_state.to_dict())
expected_snapshot = {
'global_config': {
'__id__': 'root',
'__parent__': None,
'auth_db_rev': None,
'auth_db_prev_rev': None,
'modified_by': model.Identity(kind='user', name='*****@*****.**'),
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'oauth_additional_client_ids': [u'a', u'b'],
'oauth_client_id': u'oauth_client_id',
'oauth_client_secret': u'oauth_client_secret',
},
'groups': [
{
'__id__': 'Another group',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': None,
'description': '',
'globs': [],
'members': [],
'modified_by': None,
'modified_ts': None,
'nested': [u'Some group'],
},
{
'__id__': 'Some group',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': model.Identity(kind='user', name='*****@*****.**'),
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': u'Some description',
'globs': [model.IdentityGlob(kind='user', pattern='*@example.com')],
'members': [model.Identity(kind='user', name='*****@*****.**')],
'modified_by': model.Identity(
kind='user', name='*****@*****.**'),
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'nested': [],
},
],
'secrets': [
{
'__id__': 'global_secret',
'__parent__': ndb.Key(
'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'),
'modified_by': model.Identity(
kind='user', name='*****@*****.**'),
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'values': ['1234', '5678'],
},
],
'ip_whitelists': [
{
'__id__': 'bots',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': model.Identity(kind='user', name='*****@*****.**'),
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': u'Some description',
'modified_by': model.Identity(
kind='user', name='*****@*****.**'),
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'subnets': ['127.0.0.1/32'],
},
],
'ip_whitelist_assignments': {
'__id__': 'default',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'assignments': [
{
'comment': 'some comment',
'created_by': model.Identity(
kind='user', name='*****@*****.**'),
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'identity': model.Identity(
kind='user', name='*****@*****.**'),
'ip_whitelist': 'bots',
},
],
'auth_db_rev': None,
'auth_db_prev_rev': None,
'modified_by': model.Identity(kind='user', name='*****@*****.**'),
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
},
}
self.assertEqual(expected_snapshot, snapshot_to_dict(snapshot))
def test_update_ip_whitelist_config(self):
@ndb.transactional
def run(conf):
return config._update_ip_whitelist_config(
config.Revision('ip_whitelist_cfg_rev', 'http://url'), conf)
# Pushing empty config to empty DB -> no changes.
self.assertFalse(run(config_pb2.IPWhitelistConfig()))
# Added a bunch of IP whitelists and assignments.
conf = config_pb2.IPWhitelistConfig(
ip_whitelists=[
config_pb2.IPWhitelistConfig.IPWhitelist(
name='abc',
subnets=['0.0.0.1/32']),
config_pb2.IPWhitelistConfig.IPWhitelist(
name='bots',
subnets=['0.0.0.2/32']),
config_pb2.IPWhitelistConfig.IPWhitelist(name='empty'),
],
assignments=[
config_pb2.IPWhitelistConfig.Assignment(
identity='user:[email protected]',
ip_whitelist_name='abc'),
config_pb2.IPWhitelistConfig.Assignment(
identity='user:[email protected]',
ip_whitelist_name='bots'),
config_pb2.IPWhitelistConfig.Assignment(
identity='user:[email protected]',
ip_whitelist_name='bots'),
])
self.mock_now(datetime.datetime(2014, 1, 2, 3, 4, 5))
self.assertTrue(run(conf))
# Verify everything is there.
self.assertEqual({
'assignments': [
{
'comment':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'created_by': model.Identity(kind='service', name='sample-app'),
'created_ts': datetime.datetime(2014, 1, 2, 3, 4, 5),
'identity': model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist': u'abc',
},
{
'comment':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'created_by': model.Identity(kind='service', name='sample-app'),
'created_ts': datetime.datetime(2014, 1, 2, 3, 4, 5),
'identity': model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist': u'bots',
},
{
'comment':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'created_by': model.Identity(kind='service', name='sample-app'),
'created_ts': datetime.datetime(2014, 1, 2, 3, 4, 5),
'identity': model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist': u'bots',
},
],
'auth_db_rev': 1,
'auth_db_prev_rev': None,
'modified_by': model.get_service_self_identity(),
'modified_ts': datetime.datetime(2014, 1, 2, 3, 4, 5),
}, model.ip_whitelist_assignments_key().get().to_dict())
self.assertEqual(
{
'abc': {
'created_by': 'service:sample-app',
'created_ts': 1388631845000000,
'description':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'modified_by': 'service:sample-app',
'modified_ts': 1388631845000000,
'subnets': [u'0.0.0.1/32'],
},
'bots': {
'created_by': 'service:sample-app',
'created_ts': 1388631845000000,
'description':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'modified_by': 'service:sample-app',
'modified_ts': 1388631845000000,
'subnets': [u'0.0.0.2/32'],
},
'empty': {
'created_by': 'service:sample-app',
'created_ts': 1388631845000000,
'description':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'modified_by': 'service:sample-app',
'modified_ts': 1388631845000000,
'subnets': [],
},
},
{
x.key.id(): x.to_serializable_dict()
for x in model.AuthIPWhitelist.query(ancestor=model.root_key())
})
# Exact same config a bit later -> no changes applied.
self.mock_now(datetime.datetime(2014, 2, 2, 3, 4, 5))
self.assertFalse(run(conf))
# Modify whitelist, add new one, remove some. Same for assignments.
conf = config_pb2.IPWhitelistConfig(
ip_whitelists=[
config_pb2.IPWhitelistConfig.IPWhitelist(
name='abc',
subnets=['0.0.0.3/32']),
config_pb2.IPWhitelistConfig.IPWhitelist(
name='bots',
subnets=['0.0.0.2/32']),
config_pb2.IPWhitelistConfig.IPWhitelist(name='another'),
],
assignments=[
config_pb2.IPWhitelistConfig.Assignment(
identity='user:[email protected]',
ip_whitelist_name='abc'),
config_pb2.IPWhitelistConfig.Assignment(
identity='user:[email protected]',
ip_whitelist_name='another'),
config_pb2.IPWhitelistConfig.Assignment(
identity='user:[email protected]',
ip_whitelist_name='bots'),
])
self.mock_now(datetime.datetime(2014, 3, 2, 3, 4, 5))
self.assertTrue(run(conf))
# Verify everything is there.
self.assertEqual({
'assignments': [
{
'comment':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'created_by': model.Identity(kind='service', name='sample-app'),
'created_ts': datetime.datetime(2014, 1, 2, 3, 4, 5),
'identity': model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist': u'abc',
},
{
'comment':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'created_by': model.Identity(kind='service', name='sample-app'),
'created_ts': datetime.datetime(2014, 3, 2, 3, 4, 5),
'identity': model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist': u'another',
},
{
'comment':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'created_by': model.Identity(kind='service', name='sample-app'),
'created_ts': datetime.datetime(2014, 3, 2, 3, 4, 5),
'identity': model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist': u'bots',
},
],
'auth_db_rev': 1,
'auth_db_prev_rev': 1, # replicate_auth_db is mocked, so no version bump
'modified_by': model.get_service_self_identity(),
'modified_ts': datetime.datetime(2014, 3, 2, 3, 4, 5),
}, model.ip_whitelist_assignments_key().get().to_dict())
self.assertEqual(
{
'abc': {
'created_by': 'service:sample-app',
'created_ts': 1388631845000000,
'description':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'modified_by': 'service:sample-app',
'modified_ts': 1393729445000000,
'subnets': [u'0.0.0.3/32'],
},
'bots': {
'created_by': 'service:sample-app',
'created_ts': 1388631845000000,
'description':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'modified_by': 'service:sample-app',
'modified_ts': 1388631845000000,
'subnets': [u'0.0.0.2/32'],
},
'another': {
'created_by': 'service:sample-app',
'created_ts': 1393729445000000,
'description':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'modified_by': 'service:sample-app',
'modified_ts': 1393729445000000,
'subnets': [],
},
},
{
x.key.id(): x.to_serializable_dict()
for x in model.AuthIPWhitelist.query(ancestor=model.root_key())
})
def test_works(self):
self.mock_now(datetime.datetime(2014, 1, 1, 1, 1, 1))
self.configure_as_replica(0)
# Prepare auth db state.
model.AuthGlobalConfig(
key=model.root_key(),
modified_ts=utils.utcnow(),
oauth_client_id='oauth_client_id',
oauth_client_secret='oauth_client_secret',
oauth_additional_client_ids=['a', 'b']).put()
def group(name, **kwargs):
return model.AuthGroup(
key=model.group_key(name),
created_ts=utils.utcnow(),
modified_ts=utils.utcnow(),
**kwargs)
group('Modify').put()
group('Delete').put()
group('Keep').put()
def secret(name, scope, **kwargs):
return model.AuthSecret(
id=name, parent=model.secret_scope_key(scope), **kwargs)
secret('modify', 'global').put()
secret('delete', 'global').put()
secret('keep', 'global').put()
secret('local', 'local').put()
def ip_whitelist(name, **kwargs):
return model.AuthIPWhitelist(
key=model.ip_whitelist_key(name),
created_ts=utils.utcnow(),
modified_ts=utils.utcnow(),
**kwargs)
ip_whitelist('modify').put()
ip_whitelist('delete').put()
ip_whitelist('keep').put()
def assignment(ident, ip_whitelist):
return model.AuthIPWhitelistAssignments.Assignment(
identity=model.Identity.from_bytes(ident),
ip_whitelist=ip_whitelist,
created_ts=utils.utcnow(),
comment='comment')
model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
modified_ts=utils.utcnow(),
assignments=[
assignment('user:[email protected]', 'modify'),
assignment('user:[email protected]', 'delete'),
assignment('user:[email protected]', 'keep'),
]).put()
# Prepare snapshot.
snapshot = replication.AuthDBSnapshot(
global_config=model.AuthGlobalConfig(
key=model.root_key(),
modified_ts=utils.utcnow(),
oauth_client_id='another_oauth_client_id',
oauth_client_secret='another_oauth_client_secret',
oauth_additional_client_ids=[]),
groups=[
group('New'),
group('Modify', description='blah'),
group('Keep'),
],
secrets=[
secret('new', 'global'),
secret('modify', 'global', values=['1234']),
secret('keep', 'global'),
],
ip_whitelists=[
ip_whitelist('new', subnets=['1.1.1.1/32']),
ip_whitelist('modify', subnets=['127.0.0.1/32', '192.168.0.1/32']),
ip_whitelist('keep'),
],
ip_whitelist_assignments=model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
assignments=[
assignment('user:[email protected]', 'new'),
assignment('user:[email protected]', 'modify'),
assignment('user:[email protected]', 'keep'),
],
),
)
# Push it.
updated, state = replication.replace_auth_db(
auth_db_rev=1234,
modified_ts=datetime.datetime(2014, 1, 1, 1, 1, 1),
snapshot=snapshot)
self.assertTrue(updated)
expected_state = {
'auth_db_rev': 1234,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'primary_id': u'primary',
'primary_url': u'https://primary',
}
self.assertEqual(expected_state, state.to_dict())
# Verify expected Auth db state.
current_state, current_snapshot = replication.new_auth_db_snapshot()
self.assertEqual(expected_state, current_state.to_dict())
expected_auth_db = {
'global_config': {
'__id__': 'root',
'__parent__': None,
'auth_db_rev': None,
'auth_db_prev_rev': None,
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'oauth_additional_client_ids': [],
'oauth_client_id': u'another_oauth_client_id',
'oauth_client_secret': u'another_oauth_client_secret'},
'groups': [
{
'__id__': 'Keep',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': '',
'globs': [],
'members': [],
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'nested': [],
},
{
'__id__': 'Modify',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': u'blah',
'globs': [],
'members': [],
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'nested': [],
},
{
'__id__': 'New',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': '',
'globs': [],
'members': [],
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'nested': [],
},
],
'secrets': [
{
'__id__': 'keep',
'__parent__': ndb.Key(
'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'),
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'values': [],
},
{
'__id__': 'modify',
'__parent__': ndb.Key(
'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'),
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'values': ['1234'],
},
{
'__id__': 'new',
'__parent__': ndb.Key(
'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'),
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'values': [],
},
],
'ip_whitelists': [
{
'__id__': 'keep',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': '',
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'subnets': [],
},
{
'__id__': 'modify',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': '',
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'subnets': ['127.0.0.1/32', '192.168.0.1/32'],
},
{
'__id__': 'new',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': '',
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'subnets': ['1.1.1.1/32'],
},
],
'ip_whitelist_assignments': {
'__id__': 'default',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'assignments': [
{
'comment': 'comment',
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'identity': model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist': 'new',
},
{
'comment': 'comment',
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'identity': model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist': 'modify',
},
{
'comment': 'comment',
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'identity': model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist': 'keep',
},
],
'auth_db_rev': None,
'auth_db_prev_rev': None,
'modified_by': None,
'modified_ts': None, # not transfered currently in proto
},
}
self.assertEqual(expected_auth_db, snapshot_to_dict(current_snapshot))
# Ensure local secret was left intact.
local_secrets = model.AuthSecret.query(
ancestor=model.secret_scope_key('local'))
expected_local_secrets = [
{
'__id__': 'local',
'__parent__': ndb.Key(
'AuthGlobalConfig', 'root', 'AuthSecretScope', 'local'),
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'values': [],
},
]
self.assertEqual(
expected_local_secrets, [entity_to_dict(s) for s in local_secrets])
def test_non_empty(self):
self.mock_now(datetime.datetime(2014, 1, 1, 1, 1, 1))
state = model.AuthReplicationState(
key=model.replication_state_key(),
primary_id='blah',
primary_url='https://blah',
auth_db_rev=123)
state.put()
global_config = model.AuthGlobalConfig(
key=model.root_key(),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'),
oauth_client_id='oauth_client_id',
oauth_client_secret='oauth_client_secret',
oauth_additional_client_ids=['a', 'b'])
global_config.put()
group = model.AuthGroup(
key=model.group_key('Some group'),
members=[model.Identity.from_bytes('user:[email protected]')],
globs=[model.IdentityGlob.from_bytes('user:*@example.com')],
nested=[],
description='Some description',
owners='owning-group',
created_ts=utils.utcnow(),
created_by=model.Identity.from_bytes('user:[email protected]'),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'))
group.put()
another = model.AuthGroup(
key=model.group_key('Another group'),
nested=['Some group'])
another.put()
global_secret = model.AuthSecret(
id='global_secret',
parent=model.secret_scope_key('global'),
values=['1234', '5678'],
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'))
global_secret.put()
# Local secret should not appear in a snapshot.
local_secret = model.AuthSecret(
id='local_secret',
parent=model.secret_scope_key('local'),
values=['1234', '5678'],
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'))
local_secret.put()
ip_whitelist = model.AuthIPWhitelist(
key=model.ip_whitelist_key('bots'),
subnets=['127.0.0.1/32'],
description='Some description',
created_ts=utils.utcnow(),
created_by=model.Identity.from_bytes('user:[email protected]'),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'))
ip_whitelist.put()
ip_whitelist_assignments = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=model.Identity.from_bytes('user:[email protected]'),
ip_whitelist='bots',
comment='some comment',
created_ts=utils.utcnow(),
created_by=model.Identity.from_bytes('user:[email protected]')),
])
ip_whitelist_assignments.put()
captured_state, snapshot = replication.new_auth_db_snapshot()
expected_state = {
'auth_db_rev': 123,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'primary_id': u'blah',
'primary_url': u'https://blah',
}
self.assertEqual(expected_state, captured_state.to_dict())
expected_snapshot = {
'global_config': {
'__id__': 'root',
'__parent__': None,
'auth_db_rev': None,
'auth_db_prev_rev': None,
'modified_by': model.Identity(kind='user', name='*****@*****.**'),
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'oauth_additional_client_ids': [u'a', u'b'],
'oauth_client_id': u'oauth_client_id',
'oauth_client_secret': u'oauth_client_secret',
},
'groups': [
{
'__id__': 'Another group',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': None,
'description': u'',
'globs': [],
'members': [],
'modified_by': None,
'modified_ts': None,
'nested': [u'Some group'],
'owners': u'administrators',
},
{
'__id__': 'Some group',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': model.Identity(kind='user', name='*****@*****.**'),
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': u'Some description',
'globs': [model.IdentityGlob(kind='user', pattern='*@example.com')],
'members': [model.Identity(kind='user', name='*****@*****.**')],
'modified_by': model.Identity(
kind='user', name='*****@*****.**'),
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'nested': [],
'owners': u'owning-group',
},
],
'secrets': [
{
'__id__': 'global_secret',
'__parent__': ndb.Key(
'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'),
'modified_by': model.Identity(
kind='user', name='*****@*****.**'),
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'values': ['1234', '5678'],
},
],
'ip_whitelists': [
{
'__id__': 'bots',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': model.Identity(kind='user', name='*****@*****.**'),
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': u'Some description',
'modified_by': model.Identity(
kind='user', name='*****@*****.**'),
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'subnets': [u'127.0.0.1/32'],
},
],
'ip_whitelist_assignments': {
'__id__': 'default',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'assignments': [
{
'comment': u'some comment',
'created_by': model.Identity(
kind='user', name='*****@*****.**'),
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'identity': model.Identity(
kind='user', name='*****@*****.**'),
'ip_whitelist': u'bots',
},
],
'auth_db_rev': None,
'auth_db_prev_rev': None,
'modified_by': model.Identity(kind='user', name='*****@*****.**'),
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
},
}
self.assertEqual(expected_snapshot, snapshot_to_dict(snapshot))
def test_non_empty(self):
self.mock_now(datetime.datetime(2014, 1, 1, 1, 1, 1))
state = model.AuthReplicationState(key=model.replication_state_key(),
primary_id='blah',
primary_url='https://blah',
auth_db_rev=123)
state.put()
global_config = model.AuthGlobalConfig(
key=model.root_key(),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'),
oauth_client_id='oauth_client_id',
oauth_client_secret='oauth_client_secret',
oauth_additional_client_ids=['a', 'b'],
token_server_url='https://token-server',
security_config='security config blob')
global_config.put()
group = model.AuthGroup(
key=model.group_key('Some group'),
members=[model.Identity.from_bytes('user:[email protected]')],
globs=[model.IdentityGlob.from_bytes('user:*@example.com')],
nested=[],
description='Some description',
owners='owning-group',
created_ts=utils.utcnow(),
created_by=model.Identity.from_bytes('user:[email protected]'),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'))
group.put()
another = model.AuthGroup(key=model.group_key('Another group'),
nested=['Some group'])
another.put()
ip_whitelist = model.AuthIPWhitelist(
key=model.ip_whitelist_key('bots'),
subnets=['127.0.0.1/32'],
description='Some description',
created_ts=utils.utcnow(),
created_by=model.Identity.from_bytes('user:[email protected]'),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'))
ip_whitelist.put()
ip_whitelist_assignments = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
modified_ts=utils.utcnow(),
modified_by=model.Identity.from_bytes('user:[email protected]'),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=model.Identity.from_bytes(
'user:[email protected]'),
ip_whitelist='bots',
comment='some comment',
created_ts=utils.utcnow(),
created_by=model.Identity.from_bytes(
'user:[email protected]')),
])
ip_whitelist_assignments.put()
realms_globals = model.AuthRealmsGlobals(
key=model.realms_globals_key(),
permissions=[
realms_pb2.Permission(name='luci.dev.p1'),
realms_pb2.Permission(name='luci.dev.p2'),
])
realms_globals.put()
model.AuthProjectRealms(key=model.project_realms_key('proj_id1'),
realms=realms_pb2.Realms(api_version=1234),
config_rev='rev1',
perms_rev='rev1').put()
model.AuthProjectRealms(key=model.project_realms_key('proj_id2'),
realms=realms_pb2.Realms(api_version=1234),
config_rev='rev2',
perms_rev='rev2').put()
captured_state, snapshot = replication.new_auth_db_snapshot()
expected_state = {
'auth_db_rev': 123,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'primary_id': u'blah',
'primary_url': u'https://blah',
'shard_ids': [],
}
self.assertEqual(expected_state, captured_state.to_dict())
expected_snapshot = {
'global_config': {
'__id__':
'root',
'__parent__':
None,
'auth_db_rev':
None,
'auth_db_prev_rev':
None,
'modified_by':
model.Identity(kind='user', name='*****@*****.**'),
'modified_ts':
datetime.datetime(2014, 1, 1, 1, 1, 1),
'oauth_additional_client_ids': [u'a', u'b'],
'oauth_client_id':
u'oauth_client_id',
'oauth_client_secret':
u'oauth_client_secret',
'security_config':
'security config blob',
'token_server_url':
u'https://token-server',
},
'groups': [
{
'__id__': 'Another group',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': None,
'description': u'',
'globs': [],
'members': [],
'modified_by': None,
'modified_ts': None,
'nested': [u'Some group'],
'owners': u'administrators',
},
{
'__id__':
'Some group',
'__parent__':
ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev':
None,
'auth_db_prev_rev':
None,
'created_by':
model.Identity(kind='user', name='*****@*****.**'),
'created_ts':
datetime.datetime(2014, 1, 1, 1, 1, 1),
'description':
u'Some description',
'globs':
[model.IdentityGlob(kind='user', pattern='*@example.com')],
'members':
[model.Identity(kind='user', name='*****@*****.**')],
'modified_by':
model.Identity(kind='user', name='*****@*****.**'),
'modified_ts':
datetime.datetime(2014, 1, 1, 1, 1, 1),
'nested': [],
'owners':
u'owning-group',
},
],
'ip_whitelists': [
{
'__id__':
'bots',
'__parent__':
ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev':
None,
'auth_db_prev_rev':
None,
'created_by':
model.Identity(kind='user', name='*****@*****.**'),
'created_ts':
datetime.datetime(2014, 1, 1, 1, 1, 1),
'description':
u'Some description',
'modified_by':
model.Identity(kind='user', name='*****@*****.**'),
'modified_ts':
datetime.datetime(2014, 1, 1, 1, 1, 1),
'subnets': [u'127.0.0.1/32'],
},
],
'ip_whitelist_assignments': {
'__id__':
'default',
'__parent__':
ndb.Key('AuthGlobalConfig', 'root'),
'assignments': [
{
'comment':
u'some comment',
'created_by':
model.Identity(kind='user',
name='*****@*****.**'),
'created_ts':
datetime.datetime(2014, 1, 1, 1, 1, 1),
'identity':
model.Identity(kind='user',
name='*****@*****.**'),
'ip_whitelist':
u'bots',
},
],
'auth_db_rev':
None,
'auth_db_prev_rev':
None,
'modified_by':
model.Identity(kind='user', name='*****@*****.**'),
'modified_ts':
datetime.datetime(2014, 1, 1, 1, 1, 1),
},
'realms_globals': {
'__id__':
'globals',
'__parent__':
ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_prev_rev':
None,
'auth_db_rev':
None,
'modified_by':
None,
'modified_ts':
None,
'permissions': [
realms_pb2.Permission(name='luci.dev.p1'),
realms_pb2.Permission(name='luci.dev.p2'),
],
},
'project_realms': [{
'__id__':
'proj_id1',
'__parent__':
ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_prev_rev':
None,
'auth_db_rev':
None,
'config_rev':
u'rev1',
'perms_rev':
u'rev1',
'modified_by':
None,
'modified_ts':
None,
'realms':
realms_pb2.Realms(api_version=1234),
}, {
'__id__':
'proj_id2',
'__parent__':
ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_prev_rev':
None,
'auth_db_rev':
None,
'config_rev':
u'rev2',
'perms_rev':
u'rev2',
'modified_by':
None,
'modified_ts':
None,
'realms':
realms_pb2.Realms(api_version=1234),
}],
}
self.assertEqual(expected_snapshot, snapshot_to_dict(snapshot))
def test_works(self):
self.mock_now(datetime.datetime(2014, 1, 1, 1, 1, 1))
self.configure_as_replica(0)
# Prepare auth db state.
model.AuthGlobalConfig(
key=model.root_key(),
modified_ts=utils.utcnow(),
oauth_client_id='oauth_client_id',
oauth_client_secret='oauth_client_secret',
oauth_additional_client_ids=['a', 'b']).put()
def group(name, **kwargs):
return model.AuthGroup(
key=model.group_key(name),
created_ts=utils.utcnow(),
modified_ts=utils.utcnow(),
**kwargs)
group('Modify').put()
group('Delete').put()
group('Keep').put()
def secret(name, scope, **kwargs):
return model.AuthSecret(
id=name, parent=model.secret_scope_key(scope), **kwargs)
secret('modify', 'global').put()
secret('delete', 'global').put()
secret('keep', 'global').put()
secret('local', 'local').put()
def ip_whitelist(name, **kwargs):
return model.AuthIPWhitelist(
key=model.ip_whitelist_key(name),
created_ts=utils.utcnow(),
modified_ts=utils.utcnow(),
**kwargs)
ip_whitelist('modify').put()
ip_whitelist('delete').put()
ip_whitelist('keep').put()
def assignment(ident, ip_whitelist):
return model.AuthIPWhitelistAssignments.Assignment(
identity=model.Identity.from_bytes(ident),
ip_whitelist=ip_whitelist,
created_ts=utils.utcnow(),
comment='comment')
model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
modified_ts=utils.utcnow(),
assignments=[
assignment('user:[email protected]', 'modify'),
assignment('user:[email protected]', 'delete'),
assignment('user:[email protected]', 'keep'),
]).put()
# Prepare snapshot.
snapshot = replication.AuthDBSnapshot(
global_config=model.AuthGlobalConfig(
key=model.root_key(),
modified_ts=utils.utcnow(),
oauth_client_id='another_oauth_client_id',
oauth_client_secret='another_oauth_client_secret',
oauth_additional_client_ids=[]),
groups=[
group('New'),
group('Modify', description='blah', owners='some-other-owners'),
group('Keep'),
],
secrets=[
secret('new', 'global'),
secret('modify', 'global', values=['1234']),
secret('keep', 'global'),
],
ip_whitelists=[
ip_whitelist('new', subnets=['1.1.1.1/32']),
ip_whitelist('modify', subnets=['127.0.0.1/32', '192.168.0.1/32']),
ip_whitelist('keep'),
],
ip_whitelist_assignments=model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
assignments=[
assignment('user:[email protected]', 'new'),
assignment('user:[email protected]', 'modify'),
assignment('user:[email protected]', 'keep'),
],
),
)
# Push it.
updated, state = replication.replace_auth_db(
auth_db_rev=1234,
modified_ts=datetime.datetime(2014, 1, 1, 1, 1, 1),
snapshot=snapshot)
self.assertTrue(updated)
expected_state = {
'auth_db_rev': 1234,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'primary_id': u'primary',
'primary_url': u'https://primary',
}
self.assertEqual(expected_state, state.to_dict())
# Verify expected Auth db state.
current_state, current_snapshot = replication.new_auth_db_snapshot()
self.assertEqual(expected_state, current_state.to_dict())
expected_auth_db = {
'global_config': {
'__id__': 'root',
'__parent__': None,
'auth_db_rev': None,
'auth_db_prev_rev': None,
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'oauth_additional_client_ids': [],
'oauth_client_id': u'another_oauth_client_id',
'oauth_client_secret': u'another_oauth_client_secret'},
'groups': [
{
'__id__': 'Keep',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': u'',
'globs': [],
'members': [],
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'nested': [],
'owners': u'administrators',
},
{
'__id__': 'Modify',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': u'blah',
'globs': [],
'members': [],
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'nested': [],
'owners': u'some-other-owners',
},
{
'__id__': 'New',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': u'',
'globs': [],
'members': [],
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'nested': [],
'owners': u'administrators',
},
],
'secrets': [
{
'__id__': 'keep',
'__parent__': ndb.Key(
'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'),
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'values': [],
},
{
'__id__': 'modify',
'__parent__': ndb.Key(
'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'),
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'values': ['1234'],
},
{
'__id__': 'new',
'__parent__': ndb.Key(
'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'),
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'values': [],
},
],
'ip_whitelists': [
{
'__id__': 'keep',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': u'',
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'subnets': [],
},
{
'__id__': 'modify',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': u'',
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'subnets': [u'127.0.0.1/32', u'192.168.0.1/32'],
},
{
'__id__': 'new',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'auth_db_rev': None,
'auth_db_prev_rev': None,
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'description': u'',
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'subnets': [u'1.1.1.1/32'],
},
],
'ip_whitelist_assignments': {
'__id__': 'default',
'__parent__': ndb.Key('AuthGlobalConfig', 'root'),
'assignments': [
{
'comment': u'comment',
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'identity': model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist': u'new',
},
{
'comment': u'comment',
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'identity': model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist': u'modify',
},
{
'comment': u'comment',
'created_by': None,
'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'identity': model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist': u'keep',
},
],
'auth_db_rev': None,
'auth_db_prev_rev': None,
'modified_by': None,
'modified_ts': None, # not transfered currently in proto
},
}
self.assertEqual(expected_auth_db, snapshot_to_dict(current_snapshot))
# Ensure local secret was left intact.
local_secrets = model.AuthSecret.query(
ancestor=model.secret_scope_key('local'))
expected_local_secrets = [
{
'__id__': 'local',
'__parent__': ndb.Key(
'AuthGlobalConfig', 'root', 'AuthSecretScope', 'local'),
'modified_by': None,
'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1),
'values': [],
},
]
self.assertEqual(
expected_local_secrets, [entity_to_dict(s) for s in local_secrets])
def test_fetch_auth_db(self):
# Client IDs callback. Disable config.ensure_configured() since it overrides
# _additional_client_ids_cb after we mock it.
self.mock(config, 'ensure_configured', lambda: None)
self.mock(api, '_additional_client_ids_cb',
lambda: ['', 'cb_client_id'])
self.mock(api, 'get_web_client_id', lambda: 'web_client_id')
# Create AuthGlobalConfig.
global_config = model.AuthGlobalConfig(key=model.root_key())
global_config.oauth_client_id = '1'
global_config.oauth_client_secret = 'secret'
global_config.oauth_additional_client_ids = ['2', '3']
global_config.put()
# Create a bunch of (empty) groups.
groups = [
model.AuthGroup(key=model.group_key('Group A')),
model.AuthGroup(key=model.group_key('Group B')),
]
for group in groups:
group.put()
# And a bunch of secrets.
secrets = [
model.AuthSecret.bootstrap('local%d' % i) for i in (0, 1, 2)
]
# And IP whitelist.
ip_whitelist_assignments = model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key(),
assignments=[
model.AuthIPWhitelistAssignments.Assignment(
identity=model.Anonymous,
ip_whitelist='some ip whitelist',
),
])
ip_whitelist_assignments.put()
some_ip_whitelist = model.AuthIPWhitelist(
key=model.ip_whitelist_key('some ip whitelist'),
subnets=['127.0.0.1/32'])
bots_ip_whitelist = model.AuthIPWhitelist(
key=model.ip_whitelist_key('bots'), subnets=['127.0.0.1/32'])
some_ip_whitelist.put()
bots_ip_whitelist.put()
# This all stuff should be fetched into AuthDB.
auth_db = api.fetch_auth_db()
self.assertEqual(global_config, auth_db.global_config)
self.assertEqual(set(g.key.id() for g in groups), set(auth_db.groups))
self.assertEqual(set(s.key.id() for s in secrets),
set(auth_db.secrets))
self.assertEqual(ip_whitelist_assignments,
auth_db.ip_whitelist_assignments)
self.assertEqual(
{
'bots': bots_ip_whitelist,
'some ip whitelist': some_ip_whitelist
}, auth_db.ip_whitelists)
self.assertTrue(auth_db.is_allowed_oauth_client_id('1'))
self.assertTrue(auth_db.is_allowed_oauth_client_id('cb_client_id'))
self.assertTrue(auth_db.is_allowed_oauth_client_id('web_client_id'))
self.assertFalse(auth_db.is_allowed_oauth_client_id(''))
def test_update_ip_whitelist_config(self):
@ndb.transactional
def run(conf):
return config._update_ip_whitelist_config(
config.Revision('ip_whitelist_cfg_rev', 'http://url'), conf)
# Pushing empty config to empty DB -> no changes.
self.assertFalse(run(config_pb2.IPWhitelistConfig()))
# Added a bunch of IP whitelists and assignments.
conf = config_pb2.IPWhitelistConfig(
ip_whitelists=[
config_pb2.IPWhitelistConfig.IPWhitelist(
name='abc', subnets=['0.0.0.1/32']),
config_pb2.IPWhitelistConfig.IPWhitelist(
name='bots', subnets=['0.0.0.2/32']),
config_pb2.IPWhitelistConfig.IPWhitelist(name='empty'),
],
assignments=[
config_pb2.IPWhitelistConfig.Assignment(
identity='user:[email protected]', ip_whitelist_name='abc'),
config_pb2.IPWhitelistConfig.Assignment(
identity='user:[email protected]', ip_whitelist_name='bots'),
config_pb2.IPWhitelistConfig.Assignment(
identity='user:[email protected]', ip_whitelist_name='bots'),
])
self.mock_now(datetime.datetime(2014, 1, 2, 3, 4, 5))
self.assertTrue(run(conf))
# Verify everything is there.
self.assertEqual(
{
'assignments': [
{
'comment':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'created_by':
model.Identity(kind='service', name='sample-app'),
'created_ts':
datetime.datetime(2014, 1, 2, 3, 4, 5),
'identity':
model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist':
u'abc',
},
{
'comment':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'created_by':
model.Identity(kind='service', name='sample-app'),
'created_ts':
datetime.datetime(2014, 1, 2, 3, 4, 5),
'identity':
model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist':
u'bots',
},
{
'comment':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'created_by':
model.Identity(kind='service', name='sample-app'),
'created_ts':
datetime.datetime(2014, 1, 2, 3, 4, 5),
'identity':
model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist':
u'bots',
},
],
'auth_db_rev':
1,
'auth_db_prev_rev':
None,
'modified_by':
model.get_service_self_identity(),
'modified_ts':
datetime.datetime(2014, 1, 2, 3, 4, 5),
},
model.ip_whitelist_assignments_key().get().to_dict())
self.assertEqual(
{
'abc': {
'created_by': 'service:sample-app',
'created_ts': 1388631845000000,
'description':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'modified_by': 'service:sample-app',
'modified_ts': 1388631845000000,
'subnets': [u'0.0.0.1/32'],
},
'bots': {
'created_by': 'service:sample-app',
'created_ts': 1388631845000000,
'description':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'modified_by': 'service:sample-app',
'modified_ts': 1388631845000000,
'subnets': [u'0.0.0.2/32'],
},
'empty': {
'created_by': 'service:sample-app',
'created_ts': 1388631845000000,
'description':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'modified_by': 'service:sample-app',
'modified_ts': 1388631845000000,
'subnets': [],
},
}, {
x.key.id(): x.to_serializable_dict()
for x in model.AuthIPWhitelist.query(ancestor=model.root_key())
})
# Exact same config a bit later -> no changes applied.
self.mock_now(datetime.datetime(2014, 2, 2, 3, 4, 5))
self.assertFalse(run(conf))
# Modify whitelist, add new one, remove some. Same for assignments.
conf = config_pb2.IPWhitelistConfig(
ip_whitelists=[
config_pb2.IPWhitelistConfig.IPWhitelist(
name='abc', subnets=['0.0.0.3/32']),
config_pb2.IPWhitelistConfig.IPWhitelist(
name='bots', subnets=['0.0.0.2/32']),
config_pb2.IPWhitelistConfig.IPWhitelist(name='another'),
],
assignments=[
config_pb2.IPWhitelistConfig.Assignment(
identity='user:[email protected]', ip_whitelist_name='abc'),
config_pb2.IPWhitelistConfig.Assignment(
identity='user:[email protected]',
ip_whitelist_name='another'),
config_pb2.IPWhitelistConfig.Assignment(
identity='user:[email protected]', ip_whitelist_name='bots'),
])
self.mock_now(datetime.datetime(2014, 3, 2, 3, 4, 5))
self.assertTrue(run(conf))
# Verify everything is there.
self.assertEqual(
{
'assignments': [
{
'comment':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'created_by':
model.Identity(kind='service', name='sample-app'),
'created_ts':
datetime.datetime(2014, 1, 2, 3, 4, 5),
'identity':
model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist':
u'abc',
},
{
'comment':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'created_by':
model.Identity(kind='service', name='sample-app'),
'created_ts':
datetime.datetime(2014, 3, 2, 3, 4, 5),
'identity':
model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist':
u'another',
},
{
'comment':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'created_by':
model.Identity(kind='service', name='sample-app'),
'created_ts':
datetime.datetime(2014, 3, 2, 3, 4, 5),
'identity':
model.Identity(kind='user', name='*****@*****.**'),
'ip_whitelist':
u'bots',
},
],
'auth_db_rev':
1,
'auth_db_prev_rev':
1, # replicate_auth_db is mocked, so no version bump
'modified_by':
model.get_service_self_identity(),
'modified_ts':
datetime.datetime(2014, 3, 2, 3, 4, 5),
},
model.ip_whitelist_assignments_key().get().to_dict())
self.assertEqual(
{
'abc': {
'created_by': 'service:sample-app',
'created_ts': 1388631845000000,
'description':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'modified_by': 'service:sample-app',
'modified_ts': 1393729445000000,
'subnets': [u'0.0.0.3/32'],
},
'bots': {
'created_by': 'service:sample-app',
'created_ts': 1388631845000000,
'description':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'modified_by': 'service:sample-app',
'modified_ts': 1388631845000000,
'subnets': [u'0.0.0.2/32'],
},
'another': {
'created_by': 'service:sample-app',
'created_ts': 1393729445000000,
'description':
u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev',
'modified_by': 'service:sample-app',
'modified_ts': 1393729445000000,
'subnets': [],
},
}, {
x.key.id(): x.to_serializable_dict()
for x in model.AuthIPWhitelist.query(ancestor=model.root_key())
})
def test_fetch_ip_whitelists_empty(self):
assignments, whitelists = model.fetch_ip_whitelists()
self.assertEqual(model.ip_whitelist_assignments_key(), assignments.key)
self.assertEqual(0, len(assignments.assignments))
self.assertEqual([], whitelists)
def _update_ip_whitelist_config(rev, conf):
assert ndb.in_transaction(), 'Must be called in AuthDB transaction'
now = utils.utcnow()
# Existing whitelist entities.
existing_ip_whitelists = {
e.key.id(): e
for e in model.AuthIPWhitelist.query(ancestor=model.root_key())
}
# Whitelists being imported (name => IPWhitelist proto msg).
imported_ip_whitelists = {msg.name: msg for msg in conf.ip_whitelists}
to_put = []
to_delete = []
# New or modified IP whitelists.
for wl_proto in imported_ip_whitelists.itervalues():
# Convert proto magic list to a regular list.
subnets = list(wl_proto.subnets)
# Existing whitelist and it hasn't changed?
wl = existing_ip_whitelists.get(wl_proto.name)
if wl and wl.subnets == subnets:
continue
# Update existing (to preserve auth_db_prev_rev) or create a new one.
if not wl:
wl = model.AuthIPWhitelist(
key=model.ip_whitelist_key(wl_proto.name),
created_ts=now,
created_by=model.get_service_self_identity())
wl.subnets = subnets
wl.description = 'Imported from ip_whitelist.cfg at rev %s' % rev.revision
to_put.append(wl)
# Removed IP whitelists.
for wl in existing_ip_whitelists.itervalues():
if wl.key.id() not in imported_ip_whitelists:
to_delete.append(wl)
# Update assignments. Don't touch created_ts and created_by for existing ones.
ip_whitelist_assignments = (
model.ip_whitelist_assignments_key().get() or
model.AuthIPWhitelistAssignments(
key=model.ip_whitelist_assignments_key()))
existing = {
(a.identity.to_bytes(), a.ip_whitelist): a
for a in ip_whitelist_assignments.assignments
}
updated = []
for a in conf.assignments:
key = (a.identity, a.ip_whitelist_name)
if key in existing:
updated.append(existing[key])
else:
new_one = model.AuthIPWhitelistAssignments.Assignment(
identity=model.Identity.from_bytes(a.identity),
ip_whitelist=a.ip_whitelist_name,
comment='Imported from ip_whitelist.cfg at rev %s' % rev.revision,
created_ts=now,
created_by=model.get_service_self_identity())
updated.append(new_one)
# Something has changed?
updated_keys = [
(a.identity.to_bytes(), a.ip_whitelist)
for a in updated
]
if set(updated_keys) != set(existing):
ip_whitelist_assignments.assignments = updated
to_put.append(ip_whitelist_assignments)
if not to_put and not to_delete:
return False
comment = 'Importing ip_whitelist.cfg at rev %s' % rev.revision
for e in to_put:
e.record_revision(
modified_by=model.get_service_self_identity(),
modified_ts=now,
comment=comment)
for e in to_delete:
e.record_deletion(
modified_by=model.get_service_self_identity(),
modified_ts=now,
comment=comment)
futures = []
futures.extend(ndb.put_multi_async(to_put))
futures.extend(ndb.delete_multi_async(e.key for e in to_delete))
for f in futures:
f.check_success()
return True
