def test_update_oauth_config(self): self.mock_now(datetime.datetime(2014, 1, 2, 3, 4, 5)) @ndb.transactional def run(conf): return config._update_oauth_config( config.Revision('oauth_cfg_rev', 'http://url'), conf) model.AuthGlobalConfig(key=model.root_key()).put() # Pushing empty config to empty state -> no changes. self.assertFalse(run(config_pb2.OAuthConfig())) # Updating config. self.assertTrue(run(config_pb2.OAuthConfig( primary_client_id='a', primary_client_secret='b', client_ids=['c', 'd']))) self.assertEqual({ 'auth_db_rev': 1, 'auth_db_prev_rev': None, 'modified_by': model.get_service_self_identity(), 'modified_ts': datetime.datetime(2014, 1, 2, 3, 4, 5), 'oauth_additional_client_ids': ['c', 'd'], 'oauth_client_id': 'a', 'oauth_client_secret': 'b', }, model.root_key().get().to_dict()) # Same config again -> no changes. self.assertFalse(run(config_pb2.OAuthConfig( primary_client_id='a', primary_client_secret='b', client_ids=['c', 'd'])))
def test_update_oauth_config(self): self.mock_now(datetime.datetime(2014, 1, 2, 3, 4, 5)) @ndb.transactional def run(conf): return config._update_oauth_config( config.Revision('oauth_cfg_rev', 'http://url'), conf) model.AuthGlobalConfig(key=model.root_key()).put() # Pushing empty config to empty state -> no changes. self.assertFalse(run(config_pb2.OAuthConfig())) # Updating config. self.assertTrue( run( config_pb2.OAuthConfig(primary_client_id='a', primary_client_secret='b', client_ids=['c', 'd']))) self.assertEqual( { 'auth_db_rev': 1, 'auth_db_prev_rev': None, 'modified_by': model.get_service_self_identity(), 'modified_ts': datetime.datetime(2014, 1, 2, 3, 4, 5), 'oauth_additional_client_ids': ['c', 'd'], 'oauth_client_id': 'a', 'oauth_client_secret': 'b', }, model.root_key().get().to_dict()) # Same config again -> no changes. self.assertFalse( run( config_pb2.OAuthConfig(primary_client_id='a', primary_client_secret='b', client_ids=['c', 'd'])))
def modify(**kwargs): e = model.root_key().get() or model.AuthGlobalConfig(key=model.root_key()) e.populate(**kwargs) e.record_revision( modified_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), comment='Comment') e.put() model.replicate_auth_db()
def grab_log(self, original_cls): copies = original_cls.get_historical_copy_class().query( ancestor=model.root_key()).fetch() # All keys under correct historical_revision_key(). for c in copies: self.assertEqual( ndb.Key('Rev', c.auth_db_rev, parent=model.root_key()), c.key.parent()) return {x.key: x.to_dict() for x in copies}
def test_fetch_auth_db_lazy_bootstrap(self): # Don't exist before the call. self.assertFalse(model.root_key().get()) # Run bootstrap. api._lazy_bootstrap_ran = False api.fetch_auth_db() # Exist now. self.assertTrue(model.root_key().get())
def make_auth_db(): model.AuthGlobalConfig(key=model.root_key()).put() model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key()).put() model.AuthGroup(key=model.group_key('A group')).put() model.AuthIPWhitelist(key=model.ip_whitelist_key('A whitelist')).put() model.replicate_auth_db()
def touch_all(): make_group( name='A group', members=[ident('*****@*****.**'), ident('*****@*****.**')], description='Blah', comment='New group') make_ip_whitelist( name='An IP whitelist', subnets=['127.0.0.1/32'], description='Bluh', comment='New IP whitelist') a = model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), assignments=[ model.AuthIPWhitelistAssignments.Assignment( identity=ident('*****@*****.**'), ip_whitelist='An IP whitelist') ]) a.record_revision( modified_by=ident('*****@*****.**'), modified_ts=utils.utcnow(), comment='New assignment') a.put() c = model.AuthGlobalConfig( key=model.root_key(), oauth_client_id='client_id', oauth_client_secret='client_secret', oauth_additional_client_ids=['1', '2']) c.record_revision( modified_by=ident('*****@*****.**'), modified_ts=utils.utcnow(), comment='Config change') c.put()
def modify(): c = model.root_key().get() c.oauth_additional_client_ids = ['1', '3'] c.record_revision(modified_by=ident('*****@*****.**'), modified_ts=utils.utcnow(), comment='Config change') c.put()
def touch_all(): make_group( name='A group', members=[ident('*****@*****.**'), ident('*****@*****.**')], description='Blah', comment='New group') make_ip_whitelist(name='An IP whitelist', subnets=['127.0.0.1/32'], description='Bluh', comment='New IP whitelist') a = model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), assignments=[ model.AuthIPWhitelistAssignments.Assignment( identity=ident('*****@*****.**'), ip_whitelist='An IP whitelist') ]) a.record_revision(modified_by=ident('*****@*****.**'), modified_ts=utils.utcnow(), comment='New assignment') a.put() c = model.AuthGlobalConfig(key=model.root_key(), oauth_client_id='client_id', oauth_client_secret='client_secret', oauth_additional_client_ids=['1', '2']) c.record_revision(modified_by=ident('*****@*****.**'), modified_ts=utils.utcnow(), comment='Config change') c.put()
def test_works(self): PRIMARY_URL = 'https://primary' AUTH_DB_REV = 1234 # Make some non-empty snapshot, its contents is not important. auth_db = replication.auth_db_snapshot_to_proto( make_snapshot_obj(global_config=model.AuthGlobalConfig( key=model.root_key(), oauth_client_id=u'some-client-id', oauth_client_secret=u'some-client-secret', oauth_additional_client_ids=[u'id1', u'id2'], token_server_url=u'https://example.com', security_config='security config blob'))) # Store in 50-byte shards. shard_ids = replication.store_sharded_auth_db(auth_db, PRIMARY_URL, AUTH_DB_REV, 50) self.assertEqual(2, len(shard_ids)) # Verify keys look OK and the shard size is respected. for shard_id in shard_ids: self.assertEqual(len(shard_id), 16) shard = model.snapshot_shard_key(PRIMARY_URL, AUTH_DB_REV, shard_id).get() self.assertTrue(len(shard.blob) <= 50) # Verify it can be reassembled back. reassembled = replication.load_sharded_auth_db(PRIMARY_URL, AUTH_DB_REV, shard_ids) self.assertEqual(reassembled, auth_db)
def _update_oauth_config(rev, conf): assert ndb.in_transaction(), 'Must be called in AuthDB transaction' existing = model.root_key().get() existing_as_dict = { 'oauth_client_id': existing.oauth_client_id, 'oauth_client_secret': existing.oauth_client_secret, 'oauth_additional_client_ids': list(existing.oauth_additional_client_ids), 'token_server_url': existing.token_server_url, } new_as_dict = { 'oauth_client_id': conf.primary_client_id, 'oauth_client_secret': conf.primary_client_secret, 'oauth_additional_client_ids': list(conf.client_ids), 'token_server_url': conf.token_server_url, } if new_as_dict == existing_as_dict: return False existing.populate(**new_as_dict) existing.record_revision(modified_by=model.get_service_self_identity(), modified_ts=utils.utcnow(), comment='Importing oauth.cfg at rev %s' % rev.revision) existing.put() return True
def setUp(self): super(ConfigTest, self).setUp() self.mock_now(datetime.datetime(2014, 1, 2, 3, 4, 5)) model.AuthGlobalConfig( key=model.root_key(), auth_db_rev=0, ).put()
def test_fetch_auth_db(self): # Create AuthGlobalConfig. global_config = model.AuthGlobalConfig(key=model.root_key()) global_config.oauth_client_id = '1' global_config.oauth_client_secret = 'secret' global_config.oauth_additional_client_ids = ['2', '3'] global_config.put() # Create a bunch of (empty) groups. groups = [ model.AuthGroup(key=model.group_key('Group A')), model.AuthGroup(key=model.group_key('Group B')), ] for group in groups: group.put() # And a bunch of secrets (local and global). local_secrets = [ model.AuthSecret.bootstrap('local%d' % i, 'local') for i in (0, 1, 2) ] global_secrets = [ model.AuthSecret.bootstrap('global%d' % i, 'global') for i in (0, 1, 2) ] # And IP whitelist. ip_whitelist_assignments = model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), assignments=[ model.AuthIPWhitelistAssignments.Assignment( identity=model.Anonymous, ip_whitelist='some ip whitelist', ), ]) ip_whitelist_assignments.put() some_ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('some ip whitelist'), subnets=['127.0.0.1/32']) bots_ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('bots'), subnets=['127.0.0.1/32']) some_ip_whitelist.put() bots_ip_whitelist.put() # This all stuff should be fetched into AuthDB. auth_db = api.fetch_auth_db() self.assertEqual(global_config, auth_db.global_config) self.assertEqual(set(g.key.id() for g in groups), set(auth_db.groups)) self.assertEqual(set(s.key.id() for s in local_secrets), set(auth_db.secrets['local'])) self.assertEqual(set(s.key.id() for s in global_secrets), set(auth_db.secrets['global'])) self.assertEqual(ip_whitelist_assignments, auth_db.ip_whitelist_assignments) self.assertEqual( { 'bots': bots_ip_whitelist, 'some ip whitelist': some_ip_whitelist }, auth_db.ip_whitelists)
def test_global_config_serialization(self): """Serializing snapshot with non-trivial AuthGlobalConfig.""" snapshot = make_snapshot_obj(global_config=model.AuthGlobalConfig( key=model.root_key(), oauth_client_id='some-client-id', oauth_client_secret='some-client-secret', oauth_additional_client_ids=['id1', 'id2'])) self.assert_serialization_works(snapshot)
def modify(): c = model.root_key().get() c.oauth_additional_client_ids = ['1', '3'] c.record_revision( modified_by=ident('*****@*****.**'), modified_ts=utils.utcnow(), comment='Config change') c.put()
def test_update_ip_whitelist_config_with_includes(self): def run(conf): return config._update_authdb_configs({ 'ip_whitelist.cfg': (config.Revision('ip_whitelist_cfg_rev', 'http://url'), conf), }) conf = config_pb2.IPWhitelistConfig(ip_whitelists=[ config_pb2.IPWhitelistConfig.IPWhitelist(name='a', subnets=['0.0.0.1/32']), config_pb2.IPWhitelistConfig.IPWhitelist( name='b', subnets=['0.0.0.1/32', '0.0.0.2/32'], includes=['a' ]), config_pb2.IPWhitelistConfig.IPWhitelist( name='c', subnets=['0.0.0.3/32'], includes=['a', 'b']), config_pb2.IPWhitelistConfig.IPWhitelist(name='d', includes=['c']), ]) self.assertTrue(run(conf)) # Verify everything is there. self.assertEqual( { 'a': { 'created_by': 'service:sample-app', 'created_ts': 1388631845000000, 'description': u'Imported from ip_whitelist.cfg', 'modified_by': 'service:sample-app', 'modified_ts': 1388631845000000, 'subnets': [u'0.0.0.1/32'], }, 'b': { 'created_by': 'service:sample-app', 'created_ts': 1388631845000000, 'description': u'Imported from ip_whitelist.cfg', 'modified_by': 'service:sample-app', 'modified_ts': 1388631845000000, 'subnets': [u'0.0.0.1/32', u'0.0.0.2/32'], }, 'c': { 'created_by': 'service:sample-app', 'created_ts': 1388631845000000, 'description': u'Imported from ip_whitelist.cfg', 'modified_by': 'service:sample-app', 'modified_ts': 1388631845000000, 'subnets': [u'0.0.0.1/32', u'0.0.0.2/32', u'0.0.0.3/32'], }, 'd': { 'created_by': 'service:sample-app', 'created_ts': 1388631845000000, 'description': u'Imported from ip_whitelist.cfg', 'modified_by': 'service:sample-app', 'modified_ts': 1388631845000000, 'subnets': [u'0.0.0.1/32', u'0.0.0.2/32', u'0.0.0.3/32'], }, }, { x.key.id(): x.to_serializable_dict() for x in model.AuthIPWhitelist.query(ancestor=model.root_key()) })
def create(): c = model.AuthGlobalConfig(key=model.root_key(), oauth_client_id='client_id', oauth_client_secret='client_secret', oauth_additional_client_ids=['1', '2']) c.record_revision(modified_by=ident('*****@*****.**'), modified_ts=utils.utcnow(), comment='Config change') c.put()
def modify(): c = model.root_key().get() c.oauth_additional_client_ids = ['1', '3'] c.token_server_url = 'https://token-server' c.security_config = security_config(['hi']) c.record_revision(modified_by=ident('*****@*****.**'), modified_ts=utils.utcnow(), comment='Config change') c.put()
def test_global_config_serialization(self): """Serializing snapshot with non-trivial AuthGlobalConfig.""" snapshot = make_snapshot_obj( global_config=model.AuthGlobalConfig( key=model.root_key(), oauth_client_id='some-client-id', oauth_client_secret='some-client-secret', oauth_additional_client_ids=['id1', 'id2'])) self.assert_serialization_works(snapshot)
def extract(): d = model.root_key().get() return { 'auth_db_rev': d.auth_db_rev, 'security_config': security_config_pb2.SecurityConfig.FromString( d.security_config), }
def test_global_config_serialization(self): """Serializing snapshot with non-trivial AuthGlobalConfig.""" snapshot = make_snapshot_obj(global_config=model.AuthGlobalConfig( key=model.root_key(), oauth_client_id=u'some-client-id', oauth_client_secret=u'some-client-secret', oauth_additional_client_ids=[u'id1', u'id2'], token_server_url=u'https://example.com', security_config='security config blob')) self.assert_serialization_works(snapshot)
def extract(): d = model.root_key().get() sec = security_config_pb2.SecurityConfig() if d.security_config: sec.MergeFromString(d.security_config) return { 'auth_db_rev': d.auth_db_rev, 'oauth_client_id': d.oauth_client_id, 'security_config': sec, }
def create(): c = model.AuthGlobalConfig( key=model.root_key(), oauth_client_id='client_id', oauth_client_secret='client_secret', oauth_additional_client_ids=['1', '2']) c.record_revision( modified_by=ident('*****@*****.**'), modified_ts=utils.utcnow(), comment='Config change') c.put()
def make_snapshot_obj(global_config=None, groups=None, ip_whitelists=None, ip_whitelist_assignments=None): """Returns AuthDBSnapshot with omitted fields set to default values.""" return replication.AuthDBSnapshot( global_config=global_config or model.AuthGlobalConfig(key=model.root_key()), groups=groups or [], ip_whitelists=ip_whitelists or [], ip_whitelist_assignments=( ip_whitelist_assignments or model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key())), )
def get_stored_revs_async(): """Returns a list of all stored RealmsCfgRev based on data in the AuthDB.""" out = [] metas = yield AuthProjectRealmsMeta.query( ancestor=model.root_key()).fetch_async() for meta in metas: out.append(RealmsCfgRev( project_id=meta.project_id, config_rev=meta.config_rev, config_digest=meta.config_digest, config_body=None, perms_rev=meta.perms_rev, )) raise ndb.Return(out)
def make_snapshot_obj( global_config=None, groups=None, secrets=None, ip_whitelists=None, ip_whitelist_assignments=None): """Returns AuthDBSnapshot with omitted fields set to default values.""" return replication.AuthDBSnapshot( global_config=global_config or model.AuthGlobalConfig( key=model.root_key()), groups=groups or [], secrets=secrets or [], ip_whitelists=ip_whitelists or [], ip_whitelist_assignments=( ip_whitelist_assignments or model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key())), )
def make_snapshot_obj(global_config=None, groups=None, ip_whitelists=None, ip_whitelist_assignments=None): """Returns AuthDBSnapshot with empty list of groups and whitelists.""" return replication.AuthDBSnapshot( global_config=global_config or model.AuthGlobalConfig(key=model.root_key(), oauth_client_id='oauth client id', oauth_client_secret='oauth client secret', token_server_url='token server', security_config='security config blob'), groups=groups or [], ip_whitelists=ip_whitelists or [], ip_whitelist_assignments=( ip_whitelist_assignments or model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key())), )
def test_global_config_serialization(self): """Serializing snapshot with non-trivial AuthGlobalConfig.""" auth_db = make_auth_db_proto(global_config=model.AuthGlobalConfig( key=model.root_key(), oauth_client_id=u'some-client-id', oauth_client_secret=u'some-client-secret', oauth_additional_client_ids=[u'id1', u'id2'], token_server_url=u'https://example.com', security_config='security config blob')) self.assertEqual( auth_db, replication_pb2.AuthDB( oauth_client_id=u'some-client-id', oauth_client_secret=u'some-client-secret', oauth_additional_client_ids=[u'id1', u'id2'], token_server_url=u'https://example.com', security_config='security config blob', realms={'api_version': realms.API_VERSION}, ))
def _update_authdb_configs(configs): """Pushes new configs to AuthDB entity group. Args: configs: dict {config path -> (Revision tuple, <config>)}. Returns: True if anything has changed since last import. """ # Get model.AuthGlobalConfig entity, to potentially update it. root = model.root_key().get() orig = root.to_dict() revs = _imported_config_revisions_key().get() if not revs: revs = _ImportedConfigRevisions(key=_imported_config_revisions_key(), revisions={}) ingested_revs = {} # path -> Revision for path, (rev, conf) in sorted(configs.items()): dirty = _CONFIG_SCHEMAS[path]['updater'](root, rev, conf) revs.revisions[path] = {'rev': rev.revision, 'url': rev.url} logging.info('Processed %s at rev %s: %s', path, rev.revision, 'updated' if dirty else 'up-to-date') if dirty: ingested_revs[path] = rev if root.to_dict() != orig: assert ingested_revs report = ', '.join('%s@%s' % (p, rev.revision) for p, rev in sorted(ingested_revs.items())) logging.info('Global config has been updated: %s', report) root.record_revision(modified_by=model.get_service_self_identity(), modified_ts=utils.utcnow(), comment='Importing configs: %s' % report) root.put() revs.put() if ingested_revs: model.replicate_auth_db() return bool(ingested_revs)
def test_fetch_auth_db(self): # Create AuthGlobalConfig. global_config = model.AuthGlobalConfig(key=model.root_key()) global_config.oauth_client_id = "1" global_config.oauth_client_secret = "secret" global_config.oauth_additional_client_ids = ["2", "3"] global_config.put() # Create a bunch of (empty) groups. groups = [model.AuthGroup(key=model.group_key("Group A")), model.AuthGroup(key=model.group_key("Group B"))] for group in groups: group.put() # And a bunch of secrets (local and global). local_secrets = [model.AuthSecret.bootstrap("local%d" % i, "local") for i in (0, 1, 2)] global_secrets = [model.AuthSecret.bootstrap("global%d" % i, "global") for i in (0, 1, 2)] # And IP whitelist. ip_whitelist_assignments = model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), assignments=[ model.AuthIPWhitelistAssignments.Assignment(identity=model.Anonymous, ip_whitelist="some ip whitelist") ], ) ip_whitelist_assignments.put() some_ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key("some ip whitelist"), subnets=["127.0.0.1/32"] ) bots_ip_whitelist = model.AuthIPWhitelist(key=model.ip_whitelist_key("bots"), subnets=["127.0.0.1/32"]) some_ip_whitelist.put() bots_ip_whitelist.put() # This all stuff should be fetched into AuthDB. auth_db = api.fetch_auth_db() self.assertEqual(global_config, auth_db.global_config) self.assertEqual(set(g.key.id() for g in groups), set(auth_db.groups)) self.assertEqual(set(s.key.id() for s in local_secrets), set(auth_db.secrets["local"])) self.assertEqual(set(s.key.id() for s in global_secrets), set(auth_db.secrets["global"])) self.assertEqual(ip_whitelist_assignments, auth_db.ip_whitelist_assignments) self.assertEqual({"bots": bots_ip_whitelist, "some ip whitelist": some_ip_whitelist}, auth_db.ip_whitelists)
def _update_oauth_config(rev, conf): assert ndb.in_transaction(), 'Must be called in AuthDB transaction' existing = model.root_key().get() existing_as_dict = { 'oauth_client_id': existing.oauth_client_id, 'oauth_client_secret': existing.oauth_client_secret, 'oauth_additional_client_ids': list(existing.oauth_additional_client_ids), } new_as_dict = { 'oauth_client_id': conf.primary_client_id, 'oauth_client_secret': conf.primary_client_secret, 'oauth_additional_client_ids': list(conf.client_ids), } if new_as_dict == existing_as_dict: return False existing.populate(**new_as_dict) existing.record_revision( modified_by=model.get_service_self_identity(), modified_ts=utils.utcnow(), comment='Importing oauth.cfg at rev %s' % rev.revision) existing.put() return True
def test_update_oauth_config(self): def run(conf): return config._update_authdb_configs({ 'oauth.cfg': (config.Revision('oauth_cfg_rev', 'http://url'), conf), }) # Pushing empty config to empty state -> no changes. self.assertFalse(run(config_pb2.OAuthConfig())) # Updating config. self.assertTrue( run( config_pb2.OAuthConfig( primary_client_id='a', primary_client_secret='b', client_ids=['c', 'd'], token_server_url='https://token-server'))) self.assertEqual( { 'auth_db_rev': 1, 'auth_db_prev_rev': 0, 'modified_by': model.get_service_self_identity(), 'modified_ts': datetime.datetime(2014, 1, 2, 3, 4, 5), 'oauth_additional_client_ids': [u'c', u'd'], 'oauth_client_id': u'a', 'oauth_client_secret': u'b', 'security_config': None, 'token_server_url': u'https://token-server', }, model.root_key().get().to_dict()) # Same config again -> no changes. self.assertFalse( run( config_pb2.OAuthConfig( primary_client_id='a', primary_client_secret='b', client_ids=['c', 'd'], token_server_url='https://token-server')))
def _update_ip_whitelist_config(rev, conf): assert ndb.in_transaction(), 'Must be called in AuthDB transaction' now = utils.utcnow() # Existing whitelist entities. existing_ip_whitelists = { e.key.id(): e for e in model.AuthIPWhitelist.query(ancestor=model.root_key()) } # Whitelists being imported (name => IPWhitelist proto msg). imported_ip_whitelists = {msg.name: msg for msg in conf.ip_whitelists} to_put = [] to_delete = [] # New or modified IP whitelists. for wl_proto in imported_ip_whitelists.itervalues(): # Convert proto magic list to a regular list. subnets = list(wl_proto.subnets) # Existing whitelist and it hasn't changed? wl = existing_ip_whitelists.get(wl_proto.name) if wl and wl.subnets == subnets: continue # Update existing (to preserve auth_db_prev_rev) or create a new one. if not wl: wl = model.AuthIPWhitelist( key=model.ip_whitelist_key(wl_proto.name), created_ts=now, created_by=model.get_service_self_identity()) wl.subnets = subnets wl.description = 'Imported from ip_whitelist.cfg at rev %s' % rev.revision to_put.append(wl) # Removed IP whitelists. for wl in existing_ip_whitelists.itervalues(): if wl.key.id() not in imported_ip_whitelists: to_delete.append(wl) # Update assignments. Don't touch created_ts and created_by for existing ones. ip_whitelist_assignments = ( model.ip_whitelist_assignments_key().get() or model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key())) existing = { (a.identity.to_bytes(), a.ip_whitelist): a for a in ip_whitelist_assignments.assignments } updated = [] for a in conf.assignments: key = (a.identity, a.ip_whitelist_name) if key in existing: updated.append(existing[key]) else: new_one = model.AuthIPWhitelistAssignments.Assignment( identity=model.Identity.from_bytes(a.identity), ip_whitelist=a.ip_whitelist_name, comment='Imported from ip_whitelist.cfg at rev %s' % rev.revision, created_ts=now, created_by=model.get_service_self_identity()) updated.append(new_one) # Something has changed? updated_keys = [ (a.identity.to_bytes(), a.ip_whitelist) for a in updated ] if set(updated_keys) != set(existing): ip_whitelist_assignments.assignments = updated to_put.append(ip_whitelist_assignments) if not to_put and not to_delete: return False comment = 'Importing ip_whitelist.cfg at rev %s' % rev.revision for e in to_put: e.record_revision( modified_by=model.get_service_self_identity(), modified_ts=now, comment=comment) for e in to_delete: e.record_deletion( modified_by=model.get_service_self_identity(), modified_ts=now, comment=comment) futures = [] futures.extend(ndb.put_multi_async(to_put)) futures.extend(ndb.delete_multi_async(e.key for e in to_delete)) for f in futures: f.check_success() return True
def test_global_config_log(self): @ndb.transactional def modify(**kwargs): e = model.root_key().get() or model.AuthGlobalConfig(key=model.root_key()) e.populate(**kwargs) e.record_revision( modified_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), comment='Comment') e.put() model.replicate_auth_db() # Global config is never deleted, so test only modifications. modify(oauth_client_id='1', oauth_additional_client_ids=[]) modify(oauth_client_id='2', oauth_additional_client_ids=['a']) modify(oauth_client_id='3', oauth_additional_client_ids=['a', 'b']) modify(oauth_client_id='4', oauth_additional_client_ids=[]) # Final state. self.assertEqual({ 'auth_db_rev': 4, 'auth_db_prev_rev': 3, 'oauth_additional_client_ids': [], 'oauth_client_id': u'4', 'oauth_client_secret': u'', 'modified_by': model.Identity.from_bytes('user:[email protected]'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), }, model.root_key().get().to_dict()) # Copies in the history. cpy = lambda rev: ndb.Key( 'Rev', rev, 'AuthGlobalConfigHistory', 'root', parent=model.root_key()) self.assertEqual({ cpy(1): { 'auth_db_rev': 1, 'auth_db_prev_rev': None, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'oauth_additional_client_ids': [], 'oauth_client_id': u'1', 'oauth_client_secret': u'', 'modified_by': model.Identity.from_bytes('user:[email protected]'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), }, cpy(2): { 'auth_db_rev': 2, 'auth_db_prev_rev': 1, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'oauth_additional_client_ids': [u'a'], 'oauth_client_id': u'2', 'oauth_client_secret': u'', 'modified_by': model.Identity.from_bytes('user:[email protected]'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), }, cpy(3): { 'auth_db_rev': 3, 'auth_db_prev_rev': 2, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'oauth_additional_client_ids': [u'a', u'b'], 'oauth_client_id': u'3', 'oauth_client_secret': u'', 'modified_by': model.Identity.from_bytes('user:[email protected]'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), }, cpy(4): { 'auth_db_rev': 4, 'auth_db_prev_rev': 3, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'oauth_additional_client_ids': [], 'oauth_client_id': u'4', 'oauth_client_secret': u'', 'modified_by': model.Identity.from_bytes('user:[email protected]'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), }, }, self.grab_log(model.AuthGlobalConfig))
def test_update_ip_whitelist_config(self): @ndb.transactional def run(conf): return config._update_ip_whitelist_config( config.Revision('ip_whitelist_cfg_rev', 'http://url'), conf) # Pushing empty config to empty DB -> no changes. self.assertFalse(run(config_pb2.IPWhitelistConfig())) # Added a bunch of IP whitelists and assignments. conf = config_pb2.IPWhitelistConfig( ip_whitelists=[ config_pb2.IPWhitelistConfig.IPWhitelist( name='abc', subnets=['0.0.0.1/32']), config_pb2.IPWhitelistConfig.IPWhitelist( name='bots', subnets=['0.0.0.2/32']), config_pb2.IPWhitelistConfig.IPWhitelist(name='empty'), ], assignments=[ config_pb2.IPWhitelistConfig.Assignment( identity='user:[email protected]', ip_whitelist_name='abc'), config_pb2.IPWhitelistConfig.Assignment( identity='user:[email protected]', ip_whitelist_name='bots'), config_pb2.IPWhitelistConfig.Assignment( identity='user:[email protected]', ip_whitelist_name='bots'), ]) self.mock_now(datetime.datetime(2014, 1, 2, 3, 4, 5)) self.assertTrue(run(conf)) # Verify everything is there. self.assertEqual( { 'assignments': [ { 'comment': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'created_by': model.Identity(kind='service', name='sample-app'), 'created_ts': datetime.datetime(2014, 1, 2, 3, 4, 5), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'abc', }, { 'comment': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'created_by': model.Identity(kind='service', name='sample-app'), 'created_ts': datetime.datetime(2014, 1, 2, 3, 4, 5), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'bots', }, { 'comment': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'created_by': model.Identity(kind='service', name='sample-app'), 'created_ts': datetime.datetime(2014, 1, 2, 3, 4, 5), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'bots', }, ], 'auth_db_rev': 1, 'auth_db_prev_rev': None, 'modified_by': model.get_service_self_identity(), 'modified_ts': datetime.datetime(2014, 1, 2, 3, 4, 5), }, model.ip_whitelist_assignments_key().get().to_dict()) self.assertEqual( { 'abc': { 'created_by': 'service:sample-app', 'created_ts': 1388631845000000, 'description': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'modified_by': 'service:sample-app', 'modified_ts': 1388631845000000, 'subnets': [u'0.0.0.1/32'], }, 'bots': { 'created_by': 'service:sample-app', 'created_ts': 1388631845000000, 'description': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'modified_by': 'service:sample-app', 'modified_ts': 1388631845000000, 'subnets': [u'0.0.0.2/32'], }, 'empty': { 'created_by': 'service:sample-app', 'created_ts': 1388631845000000, 'description': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'modified_by': 'service:sample-app', 'modified_ts': 1388631845000000, 'subnets': [], }, }, { x.key.id(): x.to_serializable_dict() for x in model.AuthIPWhitelist.query(ancestor=model.root_key()) }) # Exact same config a bit later -> no changes applied. self.mock_now(datetime.datetime(2014, 2, 2, 3, 4, 5)) self.assertFalse(run(conf)) # Modify whitelist, add new one, remove some. Same for assignments. conf = config_pb2.IPWhitelistConfig( ip_whitelists=[ config_pb2.IPWhitelistConfig.IPWhitelist( name='abc', subnets=['0.0.0.3/32']), config_pb2.IPWhitelistConfig.IPWhitelist( name='bots', subnets=['0.0.0.2/32']), config_pb2.IPWhitelistConfig.IPWhitelist(name='another'), ], assignments=[ config_pb2.IPWhitelistConfig.Assignment( identity='user:[email protected]', ip_whitelist_name='abc'), config_pb2.IPWhitelistConfig.Assignment( identity='user:[email protected]', ip_whitelist_name='another'), config_pb2.IPWhitelistConfig.Assignment( identity='user:[email protected]', ip_whitelist_name='bots'), ]) self.mock_now(datetime.datetime(2014, 3, 2, 3, 4, 5)) self.assertTrue(run(conf)) # Verify everything is there. self.assertEqual( { 'assignments': [ { 'comment': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'created_by': model.Identity(kind='service', name='sample-app'), 'created_ts': datetime.datetime(2014, 1, 2, 3, 4, 5), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'abc', }, { 'comment': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'created_by': model.Identity(kind='service', name='sample-app'), 'created_ts': datetime.datetime(2014, 3, 2, 3, 4, 5), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'another', }, { 'comment': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'created_by': model.Identity(kind='service', name='sample-app'), 'created_ts': datetime.datetime(2014, 3, 2, 3, 4, 5), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'bots', }, ], 'auth_db_rev': 1, 'auth_db_prev_rev': 1, # replicate_auth_db is mocked, so no version bump 'modified_by': model.get_service_self_identity(), 'modified_ts': datetime.datetime(2014, 3, 2, 3, 4, 5), }, model.ip_whitelist_assignments_key().get().to_dict()) self.assertEqual( { 'abc': { 'created_by': 'service:sample-app', 'created_ts': 1388631845000000, 'description': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'modified_by': 'service:sample-app', 'modified_ts': 1393729445000000, 'subnets': [u'0.0.0.3/32'], }, 'bots': { 'created_by': 'service:sample-app', 'created_ts': 1388631845000000, 'description': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'modified_by': 'service:sample-app', 'modified_ts': 1388631845000000, 'subnets': [u'0.0.0.2/32'], }, 'another': { 'created_by': 'service:sample-app', 'created_ts': 1393729445000000, 'description': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'modified_by': 'service:sample-app', 'modified_ts': 1393729445000000, 'subnets': [], }, }, { x.key.id(): x.to_serializable_dict() for x in model.AuthIPWhitelist.query(ancestor=model.root_key()) })
def test_non_empty(self): self.mock_now(datetime.datetime(2014, 1, 1, 1, 1, 1)) state = model.AuthReplicationState( key=model.replication_state_key(), primary_id='blah', primary_url='https://blah', auth_db_rev=123) state.put() global_config = model.AuthGlobalConfig( key=model.root_key(), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]'), oauth_client_id='oauth_client_id', oauth_client_secret='oauth_client_secret', oauth_additional_client_ids=['a', 'b']) global_config.put() group = model.AuthGroup( key=model.group_key('Some group'), members=[model.Identity.from_bytes('user:[email protected]')], globs=[model.IdentityGlob.from_bytes('user:*@example.com')], nested=[], description='Some description', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) group.put() another = model.AuthGroup( key=model.group_key('Another group'), nested=['Some group']) another.put() global_secret = model.AuthSecret( id='global_secret', parent=model.secret_scope_key('global'), values=['1234', '5678'], modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) global_secret.put() # Local secret should not appear in a snapshot. local_secret = model.AuthSecret( id='local_secret', parent=model.secret_scope_key('local'), values=['1234', '5678'], modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) local_secret.put() ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('bots'), subnets=['127.0.0.1/32'], description='Some description', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) ip_whitelist.put() ip_whitelist_assignments = model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]'), assignments=[ model.AuthIPWhitelistAssignments.Assignment( identity=model.Identity.from_bytes('user:[email protected]'), ip_whitelist='bots', comment='some comment', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes('user:[email protected]')), ]) ip_whitelist_assignments.put() captured_state, snapshot = replication.new_auth_db_snapshot() expected_state = { 'auth_db_rev': 123, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'primary_id': u'blah', 'primary_url': u'https://blah', } self.assertEqual(expected_state, captured_state.to_dict()) expected_snapshot = { 'global_config': { '__id__': 'root', '__parent__': None, 'auth_db_rev': None, 'auth_db_prev_rev': None, 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'oauth_additional_client_ids': [u'a', u'b'], 'oauth_client_id': u'oauth_client_id', 'oauth_client_secret': u'oauth_client_secret', }, 'groups': [ { '__id__': 'Another group', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': None, 'description': '', 'globs': [], 'members': [], 'modified_by': None, 'modified_ts': None, 'nested': [u'Some group'], }, { '__id__': 'Some group', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'Some description', 'globs': [model.IdentityGlob(kind='user', pattern='*@example.com')], 'members': [model.Identity(kind='user', name='*****@*****.**')], 'modified_by': model.Identity( kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'nested': [], }, ], 'secrets': [ { '__id__': 'global_secret', '__parent__': ndb.Key( 'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'), 'modified_by': model.Identity( kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'values': ['1234', '5678'], }, ], 'ip_whitelists': [ { '__id__': 'bots', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'Some description', 'modified_by': model.Identity( kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'subnets': ['127.0.0.1/32'], }, ], 'ip_whitelist_assignments': { '__id__': 'default', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'assignments': [ { 'comment': 'some comment', 'created_by': model.Identity( kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'identity': model.Identity( kind='user', name='*****@*****.**'), 'ip_whitelist': 'bots', }, ], 'auth_db_rev': None, 'auth_db_prev_rev': None, 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), }, } self.assertEqual(expected_snapshot, snapshot_to_dict(snapshot))
def test_fetch_auth_db(self): # Create AuthGlobalConfig. global_config = model.AuthGlobalConfig(key=model.root_key()) global_config.oauth_client_id = '1' global_config.oauth_client_secret = 'secret' global_config.oauth_additional_client_ids = ['2', '3'] global_config.put() # Create a bunch of (empty) groups. groups = [ model.AuthGroup(key=model.group_key('Group A')), model.AuthGroup(key=model.group_key('Group B')), ] for group in groups: group.put() # And a bunch of secrets (local and global). local_secrets = [ model.AuthSecret.bootstrap('local%d' % i, 'local') for i in (0, 1, 2) ] global_secrets = [ model.AuthSecret.bootstrap('global%d' % i, 'global') for i in (0, 1, 2) ] # And IP whitelist. ip_whitelist_assignments = model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), assignments=[ model.AuthIPWhitelistAssignments.Assignment( identity=model.Anonymous, ip_whitelist='some ip whitelist', ), ]) ip_whitelist_assignments.put() some_ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('some ip whitelist'), subnets=['127.0.0.1/32']) bots_ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('bots'), subnets=['127.0.0.1/32']) some_ip_whitelist.put() bots_ip_whitelist.put() # This all stuff should be fetched into AuthDB. auth_db = api.fetch_auth_db() self.assertEqual(global_config, auth_db.global_config) self.assertEqual( set(g.key.id() for g in groups), set(auth_db.groups)) self.assertEqual( set(s.key.id() for s in local_secrets), set(auth_db.secrets['local'])) self.assertEqual( set(s.key.id() for s in global_secrets), set(auth_db.secrets['global'])) self.assertEqual( ip_whitelist_assignments, auth_db.ip_whitelist_assignments) self.assertEqual( {'bots': bots_ip_whitelist, 'some ip whitelist': some_ip_whitelist}, auth_db.ip_whitelists)
def test_works(self): self.mock_now(datetime.datetime(2014, 1, 1, 1, 1, 1)) self.configure_as_replica(0) # Prepare auth db state. model.AuthGlobalConfig( key=model.root_key(), modified_ts=utils.utcnow(), oauth_client_id='oauth_client_id', oauth_client_secret='oauth_client_secret', oauth_additional_client_ids=['a', 'b']).put() def group(name, **kwargs): return model.AuthGroup( key=model.group_key(name), created_ts=utils.utcnow(), modified_ts=utils.utcnow(), **kwargs) group('Modify').put() group('Delete').put() group('Keep').put() def secret(name, scope, **kwargs): return model.AuthSecret( id=name, parent=model.secret_scope_key(scope), **kwargs) secret('modify', 'global').put() secret('delete', 'global').put() secret('keep', 'global').put() secret('local', 'local').put() def ip_whitelist(name, **kwargs): return model.AuthIPWhitelist( key=model.ip_whitelist_key(name), created_ts=utils.utcnow(), modified_ts=utils.utcnow(), **kwargs) ip_whitelist('modify').put() ip_whitelist('delete').put() ip_whitelist('keep').put() def assignment(ident, ip_whitelist): return model.AuthIPWhitelistAssignments.Assignment( identity=model.Identity.from_bytes(ident), ip_whitelist=ip_whitelist, created_ts=utils.utcnow(), comment='comment') model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), modified_ts=utils.utcnow(), assignments=[ assignment('user:[email protected]', 'modify'), assignment('user:[email protected]', 'delete'), assignment('user:[email protected]', 'keep'), ]).put() # Prepare snapshot. snapshot = replication.AuthDBSnapshot( global_config=model.AuthGlobalConfig( key=model.root_key(), modified_ts=utils.utcnow(), oauth_client_id='another_oauth_client_id', oauth_client_secret='another_oauth_client_secret', oauth_additional_client_ids=[]), groups=[ group('New'), group('Modify', description='blah'), group('Keep'), ], secrets=[ secret('new', 'global'), secret('modify', 'global', values=['1234']), secret('keep', 'global'), ], ip_whitelists=[ ip_whitelist('new', subnets=['1.1.1.1/32']), ip_whitelist('modify', subnets=['127.0.0.1/32', '192.168.0.1/32']), ip_whitelist('keep'), ], ip_whitelist_assignments=model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), assignments=[ assignment('user:[email protected]', 'new'), assignment('user:[email protected]', 'modify'), assignment('user:[email protected]', 'keep'), ], ), ) # Push it. updated, state = replication.replace_auth_db( auth_db_rev=1234, modified_ts=datetime.datetime(2014, 1, 1, 1, 1, 1), snapshot=snapshot) self.assertTrue(updated) expected_state = { 'auth_db_rev': 1234, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'primary_id': u'primary', 'primary_url': u'https://primary', } self.assertEqual(expected_state, state.to_dict()) # Verify expected Auth db state. current_state, current_snapshot = replication.new_auth_db_snapshot() self.assertEqual(expected_state, current_state.to_dict()) expected_auth_db = { 'global_config': { '__id__': 'root', '__parent__': None, 'auth_db_rev': None, 'auth_db_prev_rev': None, 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'oauth_additional_client_ids': [], 'oauth_client_id': u'another_oauth_client_id', 'oauth_client_secret': u'another_oauth_client_secret'}, 'groups': [ { '__id__': 'Keep', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': '', 'globs': [], 'members': [], 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'nested': [], }, { '__id__': 'Modify', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'blah', 'globs': [], 'members': [], 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'nested': [], }, { '__id__': 'New', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': '', 'globs': [], 'members': [], 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'nested': [], }, ], 'secrets': [ { '__id__': 'keep', '__parent__': ndb.Key( 'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'), 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'values': [], }, { '__id__': 'modify', '__parent__': ndb.Key( 'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'), 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'values': ['1234'], }, { '__id__': 'new', '__parent__': ndb.Key( 'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'), 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'values': [], }, ], 'ip_whitelists': [ { '__id__': 'keep', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': '', 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'subnets': [], }, { '__id__': 'modify', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': '', 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'subnets': ['127.0.0.1/32', '192.168.0.1/32'], }, { '__id__': 'new', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': '', 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'subnets': ['1.1.1.1/32'], }, ], 'ip_whitelist_assignments': { '__id__': 'default', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'assignments': [ { 'comment': 'comment', 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': 'new', }, { 'comment': 'comment', 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': 'modify', }, { 'comment': 'comment', 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': 'keep', }, ], 'auth_db_rev': None, 'auth_db_prev_rev': None, 'modified_by': None, 'modified_ts': None, # not transfered currently in proto }, } self.assertEqual(expected_auth_db, snapshot_to_dict(current_snapshot)) # Ensure local secret was left intact. local_secrets = model.AuthSecret.query( ancestor=model.secret_scope_key('local')) expected_local_secrets = [ { '__id__': 'local', '__parent__': ndb.Key( 'AuthGlobalConfig', 'root', 'AuthSecretScope', 'local'), 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'values': [], }, ] self.assertEqual( expected_local_secrets, [entity_to_dict(s) for s in local_secrets])
def test_ip_whitelist_log(self): @ndb.transactional def modify(name, **kwargs): k = model.ip_whitelist_key(name) e = k.get() if not e: e = model.AuthIPWhitelist( key=k, created_by=model.Identity.from_bytes('user:[email protected]'), created_ts=utils.utcnow()) e.record_revision( modified_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), comment='Comment') e.populate(**kwargs) e.put() model.replicate_auth_db() @ndb.transactional def remove(name): e = model.ip_whitelist_key(name).get() if e: e.record_deletion( modified_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), comment='Comment') e.key.delete() model.replicate_auth_db() # Very similar to test_groups_log, so do less test cases. modify('A', subnets=['127.0.0.1/32']) modify('A', description='Blah') modify('A', subnets=['1.0.0.0/32']) remove('A') # Copies in the history. cpy = lambda name, rev: ndb.Key( 'Rev', rev, 'AuthIPWhitelistHistory', name, parent=model.root_key()) self.assertEqual({ cpy('A', 1): { 'auth_db_rev': 1, 'auth_db_prev_rev': None, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2015, 1, 1, 1, 1), 'description': u'', 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), 'subnets': [u'127.0.0.1/32'], }, cpy('A', 2): { 'auth_db_rev': 2, 'auth_db_prev_rev': 1, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2015, 1, 1, 1, 1), 'description': u'Blah', 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), 'subnets': [u'127.0.0.1/32'], }, cpy('A', 3): { 'auth_db_rev': 3, 'auth_db_prev_rev': 2, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2015, 1, 1, 1, 1), 'description': u'Blah', 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), 'subnets': [u'1.0.0.0/32'], }, cpy('A', 4): { 'auth_db_rev': 4, 'auth_db_prev_rev': 3, 'auth_db_app_version': u'v1a', 'auth_db_deleted': True, 'auth_db_change_comment': u'Comment', 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2015, 1, 1, 1, 1), 'description': u'Blah', 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), 'subnets': [u'1.0.0.0/32'], }, }, self.grab_log(model.AuthIPWhitelist))
def test_update_ip_whitelist_config(self): @ndb.transactional def run(conf): return config._update_ip_whitelist_config( config.Revision('ip_whitelist_cfg_rev', 'http://url'), conf) # Pushing empty config to empty DB -> no changes. self.assertFalse(run(config_pb2.IPWhitelistConfig())) # Added a bunch of IP whitelists and assignments. conf = config_pb2.IPWhitelistConfig( ip_whitelists=[ config_pb2.IPWhitelistConfig.IPWhitelist( name='abc', subnets=['0.0.0.1/32']), config_pb2.IPWhitelistConfig.IPWhitelist( name='bots', subnets=['0.0.0.2/32']), config_pb2.IPWhitelistConfig.IPWhitelist(name='empty'), ], assignments=[ config_pb2.IPWhitelistConfig.Assignment( identity='user:[email protected]', ip_whitelist_name='abc'), config_pb2.IPWhitelistConfig.Assignment( identity='user:[email protected]', ip_whitelist_name='bots'), config_pb2.IPWhitelistConfig.Assignment( identity='user:[email protected]', ip_whitelist_name='bots'), ]) self.mock_now(datetime.datetime(2014, 1, 2, 3, 4, 5)) self.assertTrue(run(conf)) # Verify everything is there. self.assertEqual({ 'assignments': [ { 'comment': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'created_by': model.Identity(kind='service', name='sample-app'), 'created_ts': datetime.datetime(2014, 1, 2, 3, 4, 5), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'abc', }, { 'comment': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'created_by': model.Identity(kind='service', name='sample-app'), 'created_ts': datetime.datetime(2014, 1, 2, 3, 4, 5), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'bots', }, { 'comment': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'created_by': model.Identity(kind='service', name='sample-app'), 'created_ts': datetime.datetime(2014, 1, 2, 3, 4, 5), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'bots', }, ], 'auth_db_rev': 1, 'auth_db_prev_rev': None, 'modified_by': model.get_service_self_identity(), 'modified_ts': datetime.datetime(2014, 1, 2, 3, 4, 5), }, model.ip_whitelist_assignments_key().get().to_dict()) self.assertEqual( { 'abc': { 'created_by': 'service:sample-app', 'created_ts': 1388631845000000, 'description': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'modified_by': 'service:sample-app', 'modified_ts': 1388631845000000, 'subnets': [u'0.0.0.1/32'], }, 'bots': { 'created_by': 'service:sample-app', 'created_ts': 1388631845000000, 'description': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'modified_by': 'service:sample-app', 'modified_ts': 1388631845000000, 'subnets': [u'0.0.0.2/32'], }, 'empty': { 'created_by': 'service:sample-app', 'created_ts': 1388631845000000, 'description': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'modified_by': 'service:sample-app', 'modified_ts': 1388631845000000, 'subnets': [], }, }, { x.key.id(): x.to_serializable_dict() for x in model.AuthIPWhitelist.query(ancestor=model.root_key()) }) # Exact same config a bit later -> no changes applied. self.mock_now(datetime.datetime(2014, 2, 2, 3, 4, 5)) self.assertFalse(run(conf)) # Modify whitelist, add new one, remove some. Same for assignments. conf = config_pb2.IPWhitelistConfig( ip_whitelists=[ config_pb2.IPWhitelistConfig.IPWhitelist( name='abc', subnets=['0.0.0.3/32']), config_pb2.IPWhitelistConfig.IPWhitelist( name='bots', subnets=['0.0.0.2/32']), config_pb2.IPWhitelistConfig.IPWhitelist(name='another'), ], assignments=[ config_pb2.IPWhitelistConfig.Assignment( identity='user:[email protected]', ip_whitelist_name='abc'), config_pb2.IPWhitelistConfig.Assignment( identity='user:[email protected]', ip_whitelist_name='another'), config_pb2.IPWhitelistConfig.Assignment( identity='user:[email protected]', ip_whitelist_name='bots'), ]) self.mock_now(datetime.datetime(2014, 3, 2, 3, 4, 5)) self.assertTrue(run(conf)) # Verify everything is there. self.assertEqual({ 'assignments': [ { 'comment': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'created_by': model.Identity(kind='service', name='sample-app'), 'created_ts': datetime.datetime(2014, 1, 2, 3, 4, 5), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'abc', }, { 'comment': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'created_by': model.Identity(kind='service', name='sample-app'), 'created_ts': datetime.datetime(2014, 3, 2, 3, 4, 5), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'another', }, { 'comment': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'created_by': model.Identity(kind='service', name='sample-app'), 'created_ts': datetime.datetime(2014, 3, 2, 3, 4, 5), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'bots', }, ], 'auth_db_rev': 1, 'auth_db_prev_rev': 1, # replicate_auth_db is mocked, so no version bump 'modified_by': model.get_service_self_identity(), 'modified_ts': datetime.datetime(2014, 3, 2, 3, 4, 5), }, model.ip_whitelist_assignments_key().get().to_dict()) self.assertEqual( { 'abc': { 'created_by': 'service:sample-app', 'created_ts': 1388631845000000, 'description': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'modified_by': 'service:sample-app', 'modified_ts': 1393729445000000, 'subnets': [u'0.0.0.3/32'], }, 'bots': { 'created_by': 'service:sample-app', 'created_ts': 1388631845000000, 'description': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'modified_by': 'service:sample-app', 'modified_ts': 1388631845000000, 'subnets': [u'0.0.0.2/32'], }, 'another': { 'created_by': 'service:sample-app', 'created_ts': 1393729445000000, 'description': u'Imported from ip_whitelist.cfg at rev ip_whitelist_cfg_rev', 'modified_by': 'service:sample-app', 'modified_ts': 1393729445000000, 'subnets': [], }, }, { x.key.id(): x.to_serializable_dict() for x in model.AuthIPWhitelist.query(ancestor=model.root_key()) })
def _imported_config_revisions_key(): return ndb.Key(_ImportedConfigRevisions, 'self', parent=model.root_key())
def snapshot_groups(): """Fetches all existing groups and AuthDB revision number.""" groups = model.AuthGroup.query(ancestor=model.root_key()).fetch_async() return auth.get_auth_db_revision(), groups.get_result()
def test_non_empty(self): self.mock_now(datetime.datetime(2014, 1, 1, 1, 1, 1)) state = model.AuthReplicationState( key=model.replication_state_key(), primary_id='blah', primary_url='https://blah', auth_db_rev=123) state.put() global_config = model.AuthGlobalConfig( key=model.root_key(), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]'), oauth_client_id='oauth_client_id', oauth_client_secret='oauth_client_secret', oauth_additional_client_ids=['a', 'b']) global_config.put() group = model.AuthGroup( key=model.group_key('Some group'), members=[model.Identity.from_bytes('user:[email protected]')], globs=[model.IdentityGlob.from_bytes('user:*@example.com')], nested=[], description='Some description', owners='owning-group', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) group.put() another = model.AuthGroup( key=model.group_key('Another group'), nested=['Some group']) another.put() global_secret = model.AuthSecret( id='global_secret', parent=model.secret_scope_key('global'), values=['1234', '5678'], modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) global_secret.put() # Local secret should not appear in a snapshot. local_secret = model.AuthSecret( id='local_secret', parent=model.secret_scope_key('local'), values=['1234', '5678'], modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) local_secret.put() ip_whitelist = model.AuthIPWhitelist( key=model.ip_whitelist_key('bots'), subnets=['127.0.0.1/32'], description='Some description', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]')) ip_whitelist.put() ip_whitelist_assignments = model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), modified_ts=utils.utcnow(), modified_by=model.Identity.from_bytes('user:[email protected]'), assignments=[ model.AuthIPWhitelistAssignments.Assignment( identity=model.Identity.from_bytes('user:[email protected]'), ip_whitelist='bots', comment='some comment', created_ts=utils.utcnow(), created_by=model.Identity.from_bytes('user:[email protected]')), ]) ip_whitelist_assignments.put() captured_state, snapshot = replication.new_auth_db_snapshot() expected_state = { 'auth_db_rev': 123, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'primary_id': u'blah', 'primary_url': u'https://blah', } self.assertEqual(expected_state, captured_state.to_dict()) expected_snapshot = { 'global_config': { '__id__': 'root', '__parent__': None, 'auth_db_rev': None, 'auth_db_prev_rev': None, 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'oauth_additional_client_ids': [u'a', u'b'], 'oauth_client_id': u'oauth_client_id', 'oauth_client_secret': u'oauth_client_secret', }, 'groups': [ { '__id__': 'Another group', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': None, 'description': u'', 'globs': [], 'members': [], 'modified_by': None, 'modified_ts': None, 'nested': [u'Some group'], 'owners': u'administrators', }, { '__id__': 'Some group', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'Some description', 'globs': [model.IdentityGlob(kind='user', pattern='*@example.com')], 'members': [model.Identity(kind='user', name='*****@*****.**')], 'modified_by': model.Identity( kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'nested': [], 'owners': u'owning-group', }, ], 'secrets': [ { '__id__': 'global_secret', '__parent__': ndb.Key( 'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'), 'modified_by': model.Identity( kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'values': ['1234', '5678'], }, ], 'ip_whitelists': [ { '__id__': 'bots', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'Some description', 'modified_by': model.Identity( kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'subnets': [u'127.0.0.1/32'], }, ], 'ip_whitelist_assignments': { '__id__': 'default', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'assignments': [ { 'comment': u'some comment', 'created_by': model.Identity( kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'identity': model.Identity( kind='user', name='*****@*****.**'), 'ip_whitelist': u'bots', }, ], 'auth_db_rev': None, 'auth_db_prev_rev': None, 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), }, } self.assertEqual(expected_snapshot, snapshot_to_dict(snapshot))
def test_works(self): self.mock_now(datetime.datetime(2014, 1, 1, 1, 1, 1)) self.configure_as_replica(0) # Prepare auth db state. model.AuthGlobalConfig( key=model.root_key(), modified_ts=utils.utcnow(), oauth_client_id='oauth_client_id', oauth_client_secret='oauth_client_secret', oauth_additional_client_ids=['a', 'b']).put() def group(name, **kwargs): return model.AuthGroup( key=model.group_key(name), created_ts=utils.utcnow(), modified_ts=utils.utcnow(), **kwargs) group('Modify').put() group('Delete').put() group('Keep').put() def secret(name, scope, **kwargs): return model.AuthSecret( id=name, parent=model.secret_scope_key(scope), **kwargs) secret('modify', 'global').put() secret('delete', 'global').put() secret('keep', 'global').put() secret('local', 'local').put() def ip_whitelist(name, **kwargs): return model.AuthIPWhitelist( key=model.ip_whitelist_key(name), created_ts=utils.utcnow(), modified_ts=utils.utcnow(), **kwargs) ip_whitelist('modify').put() ip_whitelist('delete').put() ip_whitelist('keep').put() def assignment(ident, ip_whitelist): return model.AuthIPWhitelistAssignments.Assignment( identity=model.Identity.from_bytes(ident), ip_whitelist=ip_whitelist, created_ts=utils.utcnow(), comment='comment') model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), modified_ts=utils.utcnow(), assignments=[ assignment('user:[email protected]', 'modify'), assignment('user:[email protected]', 'delete'), assignment('user:[email protected]', 'keep'), ]).put() # Prepare snapshot. snapshot = replication.AuthDBSnapshot( global_config=model.AuthGlobalConfig( key=model.root_key(), modified_ts=utils.utcnow(), oauth_client_id='another_oauth_client_id', oauth_client_secret='another_oauth_client_secret', oauth_additional_client_ids=[]), groups=[ group('New'), group('Modify', description='blah', owners='some-other-owners'), group('Keep'), ], secrets=[ secret('new', 'global'), secret('modify', 'global', values=['1234']), secret('keep', 'global'), ], ip_whitelists=[ ip_whitelist('new', subnets=['1.1.1.1/32']), ip_whitelist('modify', subnets=['127.0.0.1/32', '192.168.0.1/32']), ip_whitelist('keep'), ], ip_whitelist_assignments=model.AuthIPWhitelistAssignments( key=model.ip_whitelist_assignments_key(), assignments=[ assignment('user:[email protected]', 'new'), assignment('user:[email protected]', 'modify'), assignment('user:[email protected]', 'keep'), ], ), ) # Push it. updated, state = replication.replace_auth_db( auth_db_rev=1234, modified_ts=datetime.datetime(2014, 1, 1, 1, 1, 1), snapshot=snapshot) self.assertTrue(updated) expected_state = { 'auth_db_rev': 1234, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'primary_id': u'primary', 'primary_url': u'https://primary', } self.assertEqual(expected_state, state.to_dict()) # Verify expected Auth db state. current_state, current_snapshot = replication.new_auth_db_snapshot() self.assertEqual(expected_state, current_state.to_dict()) expected_auth_db = { 'global_config': { '__id__': 'root', '__parent__': None, 'auth_db_rev': None, 'auth_db_prev_rev': None, 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'oauth_additional_client_ids': [], 'oauth_client_id': u'another_oauth_client_id', 'oauth_client_secret': u'another_oauth_client_secret'}, 'groups': [ { '__id__': 'Keep', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'', 'globs': [], 'members': [], 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'nested': [], 'owners': u'administrators', }, { '__id__': 'Modify', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'blah', 'globs': [], 'members': [], 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'nested': [], 'owners': u'some-other-owners', }, { '__id__': 'New', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'', 'globs': [], 'members': [], 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'nested': [], 'owners': u'administrators', }, ], 'secrets': [ { '__id__': 'keep', '__parent__': ndb.Key( 'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'), 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'values': [], }, { '__id__': 'modify', '__parent__': ndb.Key( 'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'), 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'values': ['1234'], }, { '__id__': 'new', '__parent__': ndb.Key( 'AuthGlobalConfig', 'root', 'AuthSecretScope', 'global'), 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'values': [], }, ], 'ip_whitelists': [ { '__id__': 'keep', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'', 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'subnets': [], }, { '__id__': 'modify', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'', 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'subnets': [u'127.0.0.1/32', u'192.168.0.1/32'], }, { '__id__': 'new', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'auth_db_rev': None, 'auth_db_prev_rev': None, 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'description': u'', 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'subnets': [u'1.1.1.1/32'], }, ], 'ip_whitelist_assignments': { '__id__': 'default', '__parent__': ndb.Key('AuthGlobalConfig', 'root'), 'assignments': [ { 'comment': u'comment', 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'new', }, { 'comment': u'comment', 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'modify', }, { 'comment': u'comment', 'created_by': None, 'created_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'keep', }, ], 'auth_db_rev': None, 'auth_db_prev_rev': None, 'modified_by': None, 'modified_ts': None, # not transfered currently in proto }, } self.assertEqual(expected_auth_db, snapshot_to_dict(current_snapshot)) # Ensure local secret was left intact. local_secrets = model.AuthSecret.query( ancestor=model.secret_scope_key('local')) expected_local_secrets = [ { '__id__': 'local', '__parent__': ndb.Key( 'AuthGlobalConfig', 'root', 'AuthSecretScope', 'local'), 'modified_by': None, 'modified_ts': datetime.datetime(2014, 1, 1, 1, 1, 1), 'values': [], }, ] self.assertEqual( expected_local_secrets, [entity_to_dict(s) for s in local_secrets])
def test_success(self): self.mock_now(datetime.datetime(2015, 1, 1)) @ndb.tasklet def urlfetch(url, payload, **_rest): urlfetch.called = True self.assertEqual( url, 'https://tokens.example.com/prpc/tokenserver.minter.TokenMinter/' 'MintDelegationToken') payload = json.loads(payload) self.assertEqual(payload, urlfetch.expected_payload) res = { 'token': 'deadbeef', 'serviceVersion': 'app-id/version-id', 'delegationSubtoken': { 'kind': 'BEARER_DELEGATION_TOKEN', 'validityDuration': payload['validityDuration'], 'subtokenId': '12345', }, } raise ndb.Return( self.Response(200, ")]}'\n" + json.dumps(res, sort_keys=True))) urlfetch.expected_payload = { u'audience': [ u'REQUESTOR', u'group:g', u'user:[email protected]', u'user:[email protected]', ], u'services': [u'https://example.com', u'service:1', u'service:2'], u'delegatedIdentity': u'user:[email protected]', u'tags': [u'a:b', u'c:d'], u'validityDuration': 3000, } urlfetch.called = False self.mock(delegation, '_urlfetch_async', urlfetch) model.AuthReplicationState( key=model.replication_state_key(), primary_url='https://auth.example.com', primary_id='example-app-id', ).put() model.AuthGlobalConfig( key=model.root_key(), token_server_url='https://tokens.example.com', ).put() args = { 'audience': [ 'user:[email protected]', model.Identity('user', '*****@*****.**'), 'group:g', 'REQUESTOR', ], 'services': [ 'service:1', model.Identity('service', '2'), 'https://example.com', ], 'max_validity_duration_sec': 3000, 'impersonate': model.Identity('user', '*****@*****.**'), 'tags': ['c:d', 'a:b'], } result = delegation.delegate(**args) self.assertTrue(urlfetch.called) self.assertEqual(result.token, 'deadbeef') self.assertEqual(result.expiry, utils.utcnow() + datetime.timedelta(seconds=3000)) # Get from cache. urlfetch.called = False delegation.delegate(**args) self.assertFalse(urlfetch.called) # Get from cache with larger validity duration. urlfetch.called = False args['min_validity_duration_sec'] = 5000 args['max_validity_duration_sec'] = 5000 urlfetch.expected_payload['validityDuration'] = 5000 result = delegation.delegate(**args) self.assertTrue(urlfetch.called) self.assertEqual(result.token, 'deadbeef') self.assertEqual(result.expiry, utils.utcnow() + datetime.timedelta(seconds=5000)) self.assertTrue(urlfetch.called)
def test_groups_log(self): ident_a = model.Identity.from_bytes('user:[email protected]') ident_b = model.Identity.from_bytes('user:[email protected]') glob_a = model.IdentityGlob.from_bytes('user:*@a.com') glob_b = model.IdentityGlob.from_bytes('user:*@b.com') @ndb.transactional def modify(name, commit=True, **kwargs): k = model.group_key(name) e = k.get() if not e: e = model.AuthGroup( key=k, created_by=ident_a, created_ts=utils.utcnow()) e.record_revision( modified_by=ident_a, modified_ts=utils.utcnow(), comment='Comment') e.populate(**kwargs) e.put() if commit: model.replicate_auth_db() @ndb.transactional def remove(name, commit=True): e = model.group_key(name).get() if e: e.record_deletion( modified_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=utils.utcnow(), comment='Comment') e.key.delete() if commit: model.replicate_auth_db() modify('A', members=[]) modify('A', members=[ident_a], globs=[glob_a]) modify('B', members=[ident_b], globs=[glob_b]) modify('A', nested=['B']) @ndb.transactional def batch(): modify('B', commit=False, description='Blah') remove('A', commit=True) batch() modify('B', members=[ident_a, ident_b], globs=[glob_a, glob_b]) # Final state. self.assertIsNone(model.group_key('A').get()) self.assertEqual({ 'auth_db_rev': 6, 'auth_db_prev_rev': 5, 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2015, 1, 1, 1, 1), 'description': u'Blah', 'globs': [ model.IdentityGlob(kind='user', pattern='*@a.com'), model.IdentityGlob(kind='user', pattern='*@b.com'), ], 'members': [ model.Identity(kind='user', name='*****@*****.**'), model.Identity(kind='user', name='*****@*****.**'), ], 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), 'nested': [], 'owners': u'administrators', }, model.group_key('B').get().to_dict()) # Copies in the history. cpy = lambda name, rev: ndb.Key( 'Rev', rev, 'AuthGroupHistory', name, parent=model.root_key()) self.assertEqual({ cpy('A', 1): { 'auth_db_rev': 1, 'auth_db_prev_rev': None, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2015, 1, 1, 1, 1), 'description': u'', 'globs': [], 'members': [], 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), 'nested': [], 'owners': u'administrators', }, cpy('A', 2): { 'auth_db_rev': 2, 'auth_db_prev_rev': 1, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2015, 1, 1, 1, 1), 'description': u'', 'globs': [glob_a], 'members': [ident_a], 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), 'nested': [], 'owners': u'administrators', }, cpy('B', 3): { 'auth_db_rev': 3, 'auth_db_prev_rev': None, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2015, 1, 1, 1, 1), 'description': u'', 'globs': [glob_b], 'members': [ident_b], 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), 'nested': [], 'owners': u'administrators', }, cpy('A', 4): { 'auth_db_rev': 4, 'auth_db_prev_rev': 2, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2015, 1, 1, 1, 1), 'description': u'', 'globs': [glob_a], 'members': [ident_a], 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), 'nested': [u'B'], 'owners': u'administrators', }, # Batch revision. cpy('A', 5): { 'auth_db_rev': 5, 'auth_db_prev_rev': 4, 'auth_db_app_version': u'v1a', 'auth_db_deleted': True, 'auth_db_change_comment': u'Comment', 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2015, 1, 1, 1, 1), 'description': u'', 'globs': [glob_a], 'members': [ident_a], 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), 'nested': [u'B'], 'owners': u'administrators', }, cpy('B', 5): { 'auth_db_rev': 5, 'auth_db_prev_rev': 3, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2015, 1, 1, 1, 1), 'description': u'Blah', 'globs': [glob_b], 'members': [ident_b], 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), 'nested': [], 'owners': u'administrators', }, # /end of batch revision cpy('B', 6): { 'auth_db_rev': 6, 'auth_db_prev_rev': 5, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'created_by': model.Identity(kind='user', name='*****@*****.**'), 'created_ts': datetime.datetime(2015, 1, 1, 1, 1), 'description': u'Blah', 'globs': [glob_a, glob_b], 'members': [ident_a, ident_b], 'modified_by': model.Identity(kind='user', name='*****@*****.**'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), 'nested': [], 'owners': u'administrators', }, }, self.grab_log(model.AuthGroup))
def test_ip_whitelist_assignment_log(self): # AuthIPWhitelistAssignments is special, it has LocalStructuredProperty. @ndb.transactional def modify(assignments): key = model.ip_whitelist_assignments_key() e = key.get() or model.AuthIPWhitelistAssignments(key=key) e.record_revision( modified_by=model.Identity.from_bytes('user:[email protected]'), modified_ts=datetime.datetime(2015, 1, 1, 1, 1), comment='Comment') e.assignments = assignments e.put() model.replicate_auth_db() Assignment = model.AuthIPWhitelistAssignments.Assignment modify([]) modify([ Assignment( identity=model.Identity.from_bytes('user:[email protected]'), ip_whitelist='bots', comment='Blah'), ]) modify([]) cpy = lambda rev: ndb.Key( 'Rev', rev, 'AuthIPWhitelistAssignmentsHistory', 'default', parent=model.root_key()) self.assertEqual({ cpy(1): { 'assignments': [], 'auth_db_rev': 1, 'auth_db_prev_rev': None, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'modified_by': model.Identity.from_bytes('user:[email protected]'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), }, cpy(2): { 'assignments': [{ 'comment': u'Blah', 'created_by': None, 'created_ts': None, 'identity': model.Identity(kind='user', name='*****@*****.**'), 'ip_whitelist': u'bots', }], 'auth_db_rev': 2, 'auth_db_prev_rev': 1, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'modified_by': model.Identity.from_bytes('user:[email protected]'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), }, cpy(3): { 'assignments': [], 'auth_db_rev': 3, 'auth_db_prev_rev': 2, 'auth_db_app_version': u'v1a', 'auth_db_deleted': False, 'auth_db_change_comment': u'Comment', 'modified_by': model.Identity.from_bytes('user:[email protected]'), 'modified_ts': datetime.datetime(2015, 1, 1, 1, 1), }, }, self.grab_log(model.AuthIPWhitelistAssignments))
def test_success(self): self.mock_now(datetime.datetime(2015, 1, 1)) def totimestamp(datetimeobj): return utils.datetime_to_timestamp(datetimeobj) / 10**6 @ndb.tasklet def urlfetch(url, payload, **_rest): urlfetch.called = True self.assertEqual( url, 'https://tokens.example.com/prpc/tokenserver.minter.TokenMinter/' 'MintProjectToken') payload = json.loads(payload) self.assertEqual(payload, urlfetch.expected_payload) expiry = utils.utcnow() + datetime.timedelta(seconds=1800) res = { 'accessToken': 'deadbeef', 'serviceAccountEmail': '*****@*****.**', 'expiry': expiry.isoformat('T') + 'Z', } raise ndb.Return( self.Response(200, json.dumps(res, sort_keys=True))) urlfetch.expected_payload = { u'luci_project': u'test-project', u'oauth_scope': [ u'https://www.googleapis.com/auth/cloud-platform', ], u'min_validity_duration': 300, u'audit_tags': [], } urlfetch.called = False self.mock(service_account, '_urlfetch_async', urlfetch) model.AuthReplicationState( key=model.replication_state_key(), primary_url='https://auth.example.com', primary_id='example-app-id', ).put() model.AuthGlobalConfig( key=model.root_key(), token_server_url='https://tokens.example.com', ).put() args = { 'project_id': 'test-project', 'oauth_scopes': [ u'https://www.googleapis.com/auth/cloud-platform', ], 'min_validity_duration_sec': 300, 'auth_request_func': service_account.authenticated_request_async, } result = project_tokens.project_token(**args) self.assertTrue(urlfetch.called) self.assertEqual(result['access_token'], 'deadbeef') self.assertEqual( result['exp_ts'], totimestamp(utils.utcnow() + datetime.timedelta(seconds=1800)))