def __init__(self, forseti_config_file_path, forseti_db_connect_string,
endpoint):
"""Initialize.
Args:
forseti_config_file_path (str): Path to Forseti configuration file
forseti_db_connect_string (str): Forseti database string
endpoint (str): server endpoint
"""
super(ServiceConfig, self).__init__()
self.thread_pool = ThreadPool()
# Enable pool_pre_ping to ensure that disconnected or errored
# connections are dropped and recreated before use.
self.engine = create_engine(forseti_db_connect_string,
pool_recycle=3600,
pool_pre_ping=True)
self.model_manager = ModelManager(self.engine)
self.sessionmaker = db.create_scoped_sessionmaker(self.engine)
self.endpoint = endpoint
self.forseti_config_file_path = forseti_config_file_path
self.inventory_config = None
self.scanner_config = None
self.notifier_config = None
self.global_config = None
self.forseti_config = None
self.update_lock = threading.RLock()
def __init__(self):
self.engine = create_test_engine()
initialize(self.engine)
self.sessionmaker = db.create_scoped_sessionmaker(self.engine)
self.inventory_config = InventoryConfig('organizations/1', '', {}, '',
{})
self.inventory_config.set_service_config(self)
def test_storage_with_timestamps(self):
"""Crawl from project, verify every resource has a timestamp."""
def verify_resource_timestamps_from_storage(storage):
session = storage.session
inventory_index_id = storage.inventory_index.id
for i, item in enumerate(DataAccess.iter(session,
inventory_index_id,
list()),
start=1):
self.assertTrue('timestamp' in item.get_other())
return i
initialize(self.engine)
scoped_sessionmaker = db.create_scoped_sessionmaker(self.engine)
res_org = ResourceMock('1', {'id': 'test'}, 'organization', 'resource')
with scoped_sessionmaker() as session:
with Storage(session, self.engine) as storage:
storage.write(res_org)
storage.commit()
resource_count = (
verify_resource_timestamps_from_storage(storage))
self.assertEqual(1, resource_count,
'Unexpected number of resources in inventory')
def __init__(self):
self.engine = create_test_engine()
self.model_manager = ModelManager(self.engine)
self.sessionmaker = db.create_scoped_sessionmaker(self.engine)
self.workers = ThreadPool(10)
self.inventory_config = InventoryConfig(gcp_api_mocks.ORGANIZATION_ID,
'', {}, '', {})
self.inventory_config.set_service_config(self)
def test_basic(self):
"""Test storing a few resources, then iterate."""
engine = create_test_engine()
initialize(engine)
scoped_sessionmaker = db.create_scoped_sessionmaker(engine)
res_org = ResourceMock('1', {'id': 'test'}, 'organization', 'resource')
res_proj1 = ResourceMock('2', {'id': 'test'}, 'project', 'resource',
res_org)
res_proj1 = ResourceMock('2', {'id': 'test'}, 'project', 'iam_policy',
res_proj1)
res_proj1 = ResourceMock('2', {'id': 'test'}, 'project',
'billing_info', res_proj1)
res_buc1 = ResourceMock('3', {'id': 'test'}, 'bucket', 'resource',
res_proj1)
res_proj2 = ResourceMock('4', {'id': 'test'}, 'project', 'resource',
res_org)
res_buc2 = ResourceMock('5', {'id': 'test'}, 'bucket', 'resource',
res_proj2)
res_obj2 = ResourceMock('6', {'id': 'test'}, 'object', 'resource',
res_buc2)
resources = [
res_org, res_proj1, res_buc1, res_proj2, res_buc2, res_obj2
]
with scoped_sessionmaker() as session:
with Storage(session) as storage:
for resource in resources:
storage.write(resource)
storage.commit()
self.assertEqual(
3,
len(
self.reduced_inventory(storage,
['organization', 'bucket'])),
'Only 1 organization and 2 buckets')
self.assertEqual(6, len(self.reduced_inventory(storage, [])),
'No types should yield empty list')
with scoped_sessionmaker() as session:
storage = Storage(session)
_ = storage.open()
for resource in resources:
storage.write(resource)
storage.buffer.flush()
self.assertEqual(
3,
len(self.reduced_inventory(storage,
['organization', 'bucket'])),
'Only 1 organization and 2 buckets')
self.assertEqual(6, len(self.reduced_inventory(storage, [])),
'No types should yield empty list')
def __init__(self, engine):
self.engine = engine
self.model_manager = ModelManager(self.engine)
self.sessionmaker = db.create_scoped_sessionmaker(self.engine)
self.workers = ThreadPool(1)
self.inventory_config = InventoryConfig(
None, '', {}, 0, {'enabled': False},
['folders/1032', 'projects/1041'])
self.inventory_config.set_service_config(self)
def __init__(self, engine):
self.engine = engine
self.model_manager = ModelManager(self.engine)
self.sessionmaker = db.create_scoped_sessionmaker(self.engine)
self.workers = ThreadPool(1)
self.inventory_config = InventoryConfig(
gcp_api_mocks.ORGANIZATION_ID, '', {}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket'
})
self.inventory_config.set_service_config(self)
def populate_data(self):
self.engine = create_test_engine()
initialize(self.engine)
self.scoped_sessionmaker = db.create_scoped_sessionmaker(self.engine)
with self.scoped_sessionmaker() as session:
inventory_indices = [
InventoryIndex(id='one_day_old',
created_at_datetime=datetime(
2010, 12, 30, 8, 0, 0)),
InventoryIndex(id='seven_days_old',
created_at_datetime=datetime(
2010, 12, 24, 8, 0, 0)),
InventoryIndex(id='nine_days_old',
created_at_datetime=datetime(
2010, 12, 22, 8, 0, 0))
]
for i in inventory_indices:
session.add(i)
session.commit()
session.expunge_all()
inventory_resources = [
Inventory(id=1, full_name=1, inventory_index_id='one_day_old'),
Inventory(id=2, full_name=2, inventory_index_id='one_day_old'),
Inventory(id=3,
full_name=3,
inventory_index_id='seven_days_old'),
Inventory(id=4,
full_name=4,
inventory_index_id='seven_days_old'),
Inventory(id=5,
full_name=5,
inventory_index_id='nine_days_old'),
Inventory(id=6,
full_name=6,
inventory_index_id='nine_days_old'),
]
for i in inventory_resources:
session.add(i)
session.commit()
session.expunge_all()
return session
def test_basic(self):
"""Test storing a few resources, then iterate."""
initialize(self.engine)
scoped_sessionmaker = db.create_scoped_sessionmaker(self.engine)
res_org = ResourceMock('1', {'id': 'test'}, 'organization', 'resource')
res_org.set_access_policy(MOCK_ACCESS_POLICY)
res_org.set_org_policy(MOCK_ORG_POLICY)
res_proj1 = ResourceMock('2', {'id': 'test'}, 'project', 'resource',
res_org)
res_proj1.set_iam_policy({'id': 'test'})
res_proj1.set_billing_info({'id': 'test'})
res_buc1 = ResourceMock('3', {'id': 'test'}, 'bucket', 'resource',
res_proj1)
res_proj2 = ResourceMock('4', {'id': 'test'}, 'project', 'resource',
res_org)
res_buc2 = ResourceMock('5', {'id': 'test'}, 'bucket', 'resource',
res_proj2)
res_obj2 = ResourceMock('6', {'id': 'test'}, 'object', 'resource',
res_buc2)
resources = [
res_org,
res_proj1,
res_buc1,
res_proj2,
res_buc2,
res_obj2
]
with scoped_sessionmaker() as session:
with Storage(session, self.engine) as storage:
for resource in resources:
storage.write(resource)
storage.commit()
inventory_index_id = storage.inventory_index.id
self.assertEqual(3,
len(self.reduced_inventory(
session,
inventory_index_id,
['organization', 'bucket'])),
'Only 1 organization and 2 buckets')
self.assertEqual(6,
len(self.reduced_inventory(session,
inventory_index_id,
[])),
'No types should yield empty list')
access_policy = self.reduced_inventory(
session, inventory_index_id, ['organization'],
Categories.access_policy)
self.assertEqual(1, len(access_policy),
'Access Policy not found in inventory.')
org_policy = self.reduced_inventory(
session, inventory_index_id, ['organization'],
Categories.org_policy)
self.assertEqual(1, len(org_policy),
'Org Policy not found in inventory.')
with scoped_sessionmaker() as session:
storage = Storage(session, self.engine)
_ = storage.open()
for resource in resources:
storage.write(resource)
storage.commit()
inventory_index_id = storage.inventory_index.id
self.assertEqual(3,
len(self.reduced_inventory(
session,
inventory_index_id,
['organization', 'bucket'])),
'Only 1 organization and 2 buckets')
self.assertEqual(6,
len(self.reduced_inventory(session,
inventory_index_id,
[])),
'No types should yield empty list')
def test_whether_resource_should_be_inserted_or_skipped(self):
"""Whether the resource should be inserted or skipped.
All resources should not be written if they have been previously
written. Except group members, where members can be in multiple groups.
"""
initialize(self.engine)
scoped_sessionmaker = db.create_scoped_sessionmaker(self.engine)
res_org = ResourceMock('1', {'id': 'test'}, 'organization', 'resource')
res_proj1 = ResourceMock('2', {'id': 'test'}, 'project', 'resource',
res_org)
res_proj1.set_iam_policy({'id': 'test'})
res_proj1.set_billing_info({'id': 'test'})
res_buc1 = ResourceMock('5', {'id': 'test'}, 'bucket', 'resource',
res_org)
res_proj2 = ResourceMock('6', {'id': 'test'}, 'project', 'resource',
res_org)
res_buc2 = ResourceMock('7', {'id': 'test'}, 'bucket', 'resource',
res_proj2)
res_obj2 = ResourceMock('8', {'id': 'test'}, 'object', 'resource',
res_buc2)
res_group1 = ResourceMock('9', {'id': 'test'}, 'google_group',
'resource', res_org)
res_group2 = ResourceMock('10', {'id': 'test'}, 'google_group',
'resource', res_org)
res_group_member1 = ResourceMock('11', {'id': 'user111',
'kind': 'admin#directory#member'},
'gsuite_group_member',
'resource', res_group1)
res_group_member2 = ResourceMock('11', {'id': 'user111',
'kind': 'admin#directory#member'},
'gsuite_group_member',
'resource', res_group2)
res_group_member3 = ResourceMock('12', {'id': 'user222',
'kind': 'admin#directory#member'},
'gsuite_group_member',
'resource', res_group1)
res_proj3 = ResourceMock('6', {'id': 'dup_proj'}, 'project',
'resource', res_org)
resources = [
res_org,
res_proj1,
res_buc1,
res_proj2,
res_buc2,
res_obj2,
res_proj3,
res_group1,
res_group2,
res_group_member1,
res_group_member2,
res_group_member3
]
with scoped_sessionmaker() as session:
with Storage(session, self.engine) as storage:
for resource in resources:
storage.write(resource)
storage.commit()
inventory_index_id = storage.inventory_index.id
self.assertEqual(3,
len(self.reduced_inventory(
session,
inventory_index_id,
['organization', 'project'])),
'Only 1 organization and 2 unique projects')
self.assertEqual(3,
len(self.reduced_inventory(
session,
inventory_index_id,
['gsuite_group_member'])),
'All group members should be stored.')
self.assertEqual(11,
len(self.reduced_inventory(
session,
inventory_index_id,
[])),
'No types should yield empty list')
