def setUp(self):
"""Setup method."""
unittest_utils.ForsetiTestCase.setUp(self)
self.engine, self.dbfile = create_test_engine_with_file()
_session_maker = sessionmaker()
self.session = _session_maker(bind=self.engine)
storage.initialize(self.engine)
self.inventory_config = InventoryConfig(
'organizations/987654321', '', {}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket'
})
# Ensure test data doesn't get deleted
self.mock_unlink = mock.patch.object(os, 'unlink',
autospec=True).start()
self.mock_export_assets = mock.patch.object(
cloudasset_api.CloudAssetClient, 'export_assets',
autospec=True).start()
self.mock_copy_file_from_gcs = mock.patch.object(
file_loader, 'copy_file_from_gcs', autospec=True).start()
self.mock_auth = mock.patch.object(
google.auth,
'default',
return_value=(mock.Mock(spec_set=credentials.Credentials),
'test-project')).start()
def setUp(self):
"""Setup method."""
CrawlerBase.setUp(self)
self.engine, self.dbfile = create_test_engine_with_file()
session_maker = sessionmaker()
self.session = session_maker(bind=self.engine)
initialize(self.engine)
self.inventory_config = InventoryConfig(
gcp_api_mocks.ORGANIZATION_ID, '', {}, 0, {
'enabled': True,
'gcs_path': 'gs://test-bucket'
})
self.inventory_config.set_service_config(FakeServerConfig(self.engine))
# Ensure test data doesn't get deleted
self.mock_unlink = mock.patch.object(os, 'unlink',
autospec=True).start()
self.mock_copy_file_from_gcs = mock.patch.object(
file_loader, 'copy_file_from_gcs', autospec=True).start()
self.maxDiff = None
# Mock copy_file_from_gcs to return correct test data file
def _copy_file_from_gcs(file_path, *args, **kwargs):
"""Fake copy_file_from_gcs."""
del args, kwargs
if 'resource' in file_path:
return os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_resources.dump')
elif 'iam_policy' in file_path:
return os.path.join(TEST_RESOURCE_DIR_PATH,
'mock_cai_iam_policies.dump')
self.mock_copy_file_from_gcs.side_effect = _copy_file_from_gcs
def test_storage_with_timestamps(self):
"""Crawl from project, verify every resource has a timestamp."""
def verify_resource_timestamps_from_storage(storage):
session = storage.session
inventory_index_id = storage.inventory_index.id
for i, item in enumerate(DataAccess.iter(session,
inventory_index_id,
list()),
start=1):
self.assertTrue('timestamp' in item.get_other())
return i
initialize(self.engine)
scoped_sessionmaker = db.create_scoped_sessionmaker(self.engine)
res_org = ResourceMock('1', {'id': 'test'}, 'organization', 'resource')
with scoped_sessionmaker() as session:
with Storage(session, self.engine) as storage:
storage.write(res_org)
storage.commit()
resource_count = (
verify_resource_timestamps_from_storage(storage))
self.assertEqual(1, resource_count,
'Unexpected number of resources in inventory')
def setUp(self):
"""Setup method."""
ForsetiTestCase.setUp(self)
self.engine, self.dbfile = create_test_engine_with_file()
_session_maker = sessionmaker()
self.session = _session_maker(bind=self.engine)
initialize(self.engine)
def __init__(self):
self.engine = create_test_engine()
initialize(self.engine)
self.sessionmaker = db.create_scoped_sessionmaker(self.engine)
self.inventory_config = InventoryConfig('organizations/1', '', {}, '',
{})
self.inventory_config.set_service_config(self)
def setUp(self):
"""Setup method."""
ForsetiTestCase.setUp(self)
self.fake_utcnow = datetime(year=1910, month=9, day=8, hour=7, minute=6)
self.engine, self.dbfile = create_test_engine_with_file()
_session_maker = sessionmaker()
self.session = _session_maker(bind=self.engine)
initialize(self.engine)
def test_basic(self):
"""Test storing a few resources, then iterate."""
engine = create_test_engine()
initialize(engine)
scoped_sessionmaker = db.create_scoped_sessionmaker(engine)
res_org = ResourceMock('1', {'id': 'test'}, 'organization', 'resource')
res_proj1 = ResourceMock('2', {'id': 'test'}, 'project', 'resource',
res_org)
res_proj1 = ResourceMock('2', {'id': 'test'}, 'project', 'iam_policy',
res_proj1)
res_proj1 = ResourceMock('2', {'id': 'test'}, 'project',
'billing_info', res_proj1)
res_buc1 = ResourceMock('3', {'id': 'test'}, 'bucket', 'resource',
res_proj1)
res_proj2 = ResourceMock('4', {'id': 'test'}, 'project', 'resource',
res_org)
res_buc2 = ResourceMock('5', {'id': 'test'}, 'bucket', 'resource',
res_proj2)
res_obj2 = ResourceMock('6', {'id': 'test'}, 'object', 'resource',
res_buc2)
resources = [
res_org, res_proj1, res_buc1, res_proj2, res_buc2, res_obj2
]
with scoped_sessionmaker() as session:
with Storage(session) as storage:
for resource in resources:
storage.write(resource)
storage.commit()
self.assertEqual(
3,
len(
self.reduced_inventory(storage,
['organization', 'bucket'])),
'Only 1 organization and 2 buckets')
self.assertEqual(6, len(self.reduced_inventory(storage, [])),
'No types should yield empty list')
with scoped_sessionmaker() as session:
storage = Storage(session)
_ = storage.open()
for resource in resources:
storage.write(resource)
storage.buffer.flush()
self.assertEqual(
3,
len(self.reduced_inventory(storage,
['organization', 'bucket'])),
'Only 1 organization and 2 buckets')
self.assertEqual(6, len(self.reduced_inventory(storage, [])),
'No types should yield empty list')
def setUp(self):
"""Setup method."""
ForsetiTestCase.setUp(self)
self.engine, self.dbfile = create_test_engine_with_file()
session_maker = sessionmaker()
self.session = session_maker(bind=self.engine)
storage.initialize(self.engine)
scanner_dao.initialize(self.engine)
self.session.flush()
self.violation_access = scanner_dao.ViolationAccess(self.session)
self.inv_index_id1, self.inv_index_id2, self.inv_index_id3 = (
_setup_inv_indices(self.session))
def populate_data(self):
self.engine = create_test_engine()
initialize(self.engine)
self.scoped_sessionmaker = db.create_scoped_sessionmaker(self.engine)
with self.scoped_sessionmaker() as session:
inventory_indices = [
InventoryIndex(id='one_day_old',
created_at_datetime=datetime(
2010, 12, 30, 8, 0, 0)),
InventoryIndex(id='seven_days_old',
created_at_datetime=datetime(
2010, 12, 24, 8, 0, 0)),
InventoryIndex(id='nine_days_old',
created_at_datetime=datetime(
2010, 12, 22, 8, 0, 0))
]
for i in inventory_indices:
session.add(i)
session.commit()
session.expunge_all()
inventory_resources = [
Inventory(id=1, full_name=1, inventory_index_id='one_day_old'),
Inventory(id=2, full_name=2, inventory_index_id='one_day_old'),
Inventory(id=3,
full_name=3,
inventory_index_id='seven_days_old'),
Inventory(id=4,
full_name=4,
inventory_index_id='seven_days_old'),
Inventory(id=5,
full_name=5,
inventory_index_id='nine_days_old'),
Inventory(id=6,
full_name=6,
inventory_index_id='nine_days_old'),
]
for i in inventory_resources:
session.add(i)
session.commit()
session.expunge_all()
return session
def setUp(self):
self.engine = create_test_engine()
_session_maker = sessionmaker()
self.session = _session_maker(bind=self.engine)
initialize(self.engine)
res_user1 = ResourceMock('1', {
'primaryEmail': '*****@*****.**',
'suspended': False
}, 'gsuite_user', 'resource')
res_user2 = ResourceMock('2', {
'primaryEmail': '*****@*****.**',
'suspended': False
}, 'gsuite_user', 'resource')
res_user3 = ResourceMock('3', {
'primaryEmail': '*****@*****.**',
'suspended': False
}, 'gsuite_user', 'resource')
self.resources = [res_user1, res_user2, res_user3]
self.storage = Storage(self.session)
_ = self.storage.open()
for resource in self.resources:
self.storage.write(resource)
self.storage.commit()
self.service_config = TestServiceConfig()
for subclass in cls.__subclasses__():
results.append(subclass)
return results
if __name__ == '__main__':
# If the DB connection string is passed in, use that, otherwise
# fall back to the default DB connection string.
print(sys.argv)
DB_CONN_STR = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_DB_CONN_STR
SQL_ENGINE = general_dao.create_engine(DB_CONN_STR, pool_recycle=3600)
# Drop the CaiTemporaryStore table to ensure it is using the
# latest schema.
inventory_dao.initialize(SQL_ENGINE)
INVENTORY_TABLES = inventory_dao.BASE.metadata.tables
CAI_TABLE = INVENTORY_TABLES.get(
inventory_dao.CaiTemporaryStore.__tablename__)
CAI_TABLE.drop(SQL_ENGINE)
# Create tables if not exists.
inventory_dao.initialize(SQL_ENGINE)
scanner_dao.initialize(SQL_ENGINE)
# Find all the child classes inherited from declarative base class.
SCANNER_DAO_CLASSES = _find_subclasses(scanner_dao.BASE)
INVENTORY_DAO_CLASSES = _find_subclasses(inventory_dao.BASE)
INVENTORY_DAO_CLASSES.extend([inventory_dao.CaiTemporaryStore])
def test_basic(self):
"""Test storing a few resources, then iterate."""
initialize(self.engine)
scoped_sessionmaker = db.create_scoped_sessionmaker(self.engine)
res_org = ResourceMock('1', {'id': 'test'}, 'organization', 'resource')
res_org.set_access_policy(MOCK_ACCESS_POLICY)
res_org.set_org_policy(MOCK_ORG_POLICY)
res_proj1 = ResourceMock('2', {'id': 'test'}, 'project', 'resource',
res_org)
res_proj1.set_iam_policy({'id': 'test'})
res_proj1.set_billing_info({'id': 'test'})
res_buc1 = ResourceMock('3', {'id': 'test'}, 'bucket', 'resource',
res_proj1)
res_proj2 = ResourceMock('4', {'id': 'test'}, 'project', 'resource',
res_org)
res_buc2 = ResourceMock('5', {'id': 'test'}, 'bucket', 'resource',
res_proj2)
res_obj2 = ResourceMock('6', {'id': 'test'}, 'object', 'resource',
res_buc2)
resources = [
res_org,
res_proj1,
res_buc1,
res_proj2,
res_buc2,
res_obj2
]
with scoped_sessionmaker() as session:
with Storage(session, self.engine) as storage:
for resource in resources:
storage.write(resource)
storage.commit()
inventory_index_id = storage.inventory_index.id
self.assertEqual(3,
len(self.reduced_inventory(
session,
inventory_index_id,
['organization', 'bucket'])),
'Only 1 organization and 2 buckets')
self.assertEqual(6,
len(self.reduced_inventory(session,
inventory_index_id,
[])),
'No types should yield empty list')
access_policy = self.reduced_inventory(
session, inventory_index_id, ['organization'],
Categories.access_policy)
self.assertEqual(1, len(access_policy),
'Access Policy not found in inventory.')
org_policy = self.reduced_inventory(
session, inventory_index_id, ['organization'],
Categories.org_policy)
self.assertEqual(1, len(org_policy),
'Org Policy not found in inventory.')
with scoped_sessionmaker() as session:
storage = Storage(session, self.engine)
_ = storage.open()
for resource in resources:
storage.write(resource)
storage.commit()
inventory_index_id = storage.inventory_index.id
self.assertEqual(3,
len(self.reduced_inventory(
session,
inventory_index_id,
['organization', 'bucket'])),
'Only 1 organization and 2 buckets')
self.assertEqual(6,
len(self.reduced_inventory(session,
inventory_index_id,
[])),
'No types should yield empty list')
def test_whether_resource_should_be_inserted_or_skipped(self):
"""Whether the resource should be inserted or skipped.
All resources should not be written if they have been previously
written. Except group members, where members can be in multiple groups.
"""
initialize(self.engine)
scoped_sessionmaker = db.create_scoped_sessionmaker(self.engine)
res_org = ResourceMock('1', {'id': 'test'}, 'organization', 'resource')
res_proj1 = ResourceMock('2', {'id': 'test'}, 'project', 'resource',
res_org)
res_proj1.set_iam_policy({'id': 'test'})
res_proj1.set_billing_info({'id': 'test'})
res_buc1 = ResourceMock('5', {'id': 'test'}, 'bucket', 'resource',
res_org)
res_proj2 = ResourceMock('6', {'id': 'test'}, 'project', 'resource',
res_org)
res_buc2 = ResourceMock('7', {'id': 'test'}, 'bucket', 'resource',
res_proj2)
res_obj2 = ResourceMock('8', {'id': 'test'}, 'object', 'resource',
res_buc2)
res_group1 = ResourceMock('9', {'id': 'test'}, 'google_group',
'resource', res_org)
res_group2 = ResourceMock('10', {'id': 'test'}, 'google_group',
'resource', res_org)
res_group_member1 = ResourceMock('11', {'id': 'user111',
'kind': 'admin#directory#member'},
'gsuite_group_member',
'resource', res_group1)
res_group_member2 = ResourceMock('11', {'id': 'user111',
'kind': 'admin#directory#member'},
'gsuite_group_member',
'resource', res_group2)
res_group_member3 = ResourceMock('12', {'id': 'user222',
'kind': 'admin#directory#member'},
'gsuite_group_member',
'resource', res_group1)
res_proj3 = ResourceMock('6', {'id': 'dup_proj'}, 'project',
'resource', res_org)
resources = [
res_org,
res_proj1,
res_buc1,
res_proj2,
res_buc2,
res_obj2,
res_proj3,
res_group1,
res_group2,
res_group_member1,
res_group_member2,
res_group_member3
]
with scoped_sessionmaker() as session:
with Storage(session, self.engine) as storage:
for resource in resources:
storage.write(resource)
storage.commit()
inventory_index_id = storage.inventory_index.id
self.assertEqual(3,
len(self.reduced_inventory(
session,
inventory_index_id,
['organization', 'project'])),
'Only 1 organization and 2 unique projects')
self.assertEqual(3,
len(self.reduced_inventory(
session,
inventory_index_id,
['gsuite_group_member'])),
'All group members should be stored.')
self.assertEqual(11,
len(self.reduced_inventory(
session,
inventory_index_id,
[])),
'No types should yield empty list')
