def test_storage_with_timestamps(self):
"""Crawl from project, verify every resource has a timestamp."""
def verify_resource_timestamps_from_storage(storage):
session = storage.session
inventory_index_id = storage.inventory_index.id
for i, item in enumerate(DataAccess.iter(session,
inventory_index_id,
list()),
start=1):
self.assertTrue('timestamp' in item.get_other())
return i
initialize(self.engine)
scoped_sessionmaker = db.create_scoped_sessionmaker(self.engine)
res_org = ResourceMock('1', {'id': 'test'}, 'organization', 'resource')
with scoped_sessionmaker() as session:
with Storage(session, self.engine) as storage:
storage.write(res_org)
storage.commit()
resource_count = (
verify_resource_timestamps_from_storage(storage))
self.assertEqual(1, resource_count,
'Unexpected number of resources in inventory')
def setUp(self):
self.service_config = TestServiceConfig()
res_user1 = ResourceMock('1', {
'primaryEmail': '*****@*****.**',
'suspended': False
}, 'gsuite_user', 'resource')
res_user2 = ResourceMock('2', {
'primaryEmail': '*****@*****.**',
'suspended': False
}, 'gsuite_user', 'resource')
res_user3 = ResourceMock('3', {
'primaryEmail': '*****@*****.**',
'suspended': False
}, 'gsuite_user', 'resource')
resources = [res_user1, res_user2, res_user3]
with self.service_config.scoped_session() as session:
with Storage(session, self.service_config.get_engine()) as storage:
for resource in resources:
storage.write(resource)
storage.commit()
def test_basic(self):
"""Test storing a few resources, then iterate."""
initialize(self.engine)
scoped_sessionmaker = db.create_scoped_sessionmaker(self.engine)
res_org = ResourceMock('1', {'id': 'test'}, 'organization', 'resource')
res_org.set_access_policy(MOCK_ACCESS_POLICY)
res_org.set_org_policy(MOCK_ORG_POLICY)
res_proj1 = ResourceMock('2', {'id': 'test'}, 'project', 'resource',
res_org)
res_proj1.set_iam_policy({'id': 'test'})
res_proj1.set_billing_info({'id': 'test'})
res_buc1 = ResourceMock('3', {'id': 'test'}, 'bucket', 'resource',
res_proj1)
res_proj2 = ResourceMock('4', {'id': 'test'}, 'project', 'resource',
res_org)
res_buc2 = ResourceMock('5', {'id': 'test'}, 'bucket', 'resource',
res_proj2)
res_obj2 = ResourceMock('6', {'id': 'test'}, 'object', 'resource',
res_buc2)
resources = [
res_org,
res_proj1,
res_buc1,
res_proj2,
res_buc2,
res_obj2
]
with scoped_sessionmaker() as session:
with Storage(session, self.engine) as storage:
for resource in resources:
storage.write(resource)
storage.commit()
inventory_index_id = storage.inventory_index.id
self.assertEqual(3,
len(self.reduced_inventory(
session,
inventory_index_id,
['organization', 'bucket'])),
'Only 1 organization and 2 buckets')
self.assertEqual(6,
len(self.reduced_inventory(session,
inventory_index_id,
[])),
'No types should yield empty list')
access_policy = self.reduced_inventory(
session, inventory_index_id, ['organization'],
Categories.access_policy)
self.assertEqual(1, len(access_policy),
'Access Policy not found in inventory.')
org_policy = self.reduced_inventory(
session, inventory_index_id, ['organization'],
Categories.org_policy)
self.assertEqual(1, len(org_policy),
'Org Policy not found in inventory.')
with scoped_sessionmaker() as session:
storage = Storage(session, self.engine)
_ = storage.open()
for resource in resources:
storage.write(resource)
storage.commit()
inventory_index_id = storage.inventory_index.id
self.assertEqual(3,
len(self.reduced_inventory(
session,
inventory_index_id,
['organization', 'bucket'])),
'Only 1 organization and 2 buckets')
self.assertEqual(6,
len(self.reduced_inventory(session,
inventory_index_id,
[])),
'No types should yield empty list')
def test_get_summary(self):
res_org = ResourceMock('1', {'id': 'test'}, 'organization', 'resource')
res_proj1 = ResourceMock('2', {'id': 'test'}, 'project', 'resource',
res_org)
res_proj1.set_iam_policy({'id': 'test'})
res_proj1.set_billing_info({'id': 'test'})
res_buc1 = ResourceMock('5', {'id': 'test'}, 'bucket', 'resource',
res_proj1)
res_proj2 = ResourceMock('6', {'id': 'test'}, 'project', 'resource',
res_org)
res_buc2 = ResourceMock('7', {'id': 'test'}, 'bucket', 'resource',
res_proj2)
res_obj2 = ResourceMock('8', {'id': 'test'}, 'object', 'resource',
res_buc2)
resources = [
res_org, res_proj1, res_buc1, res_proj2, res_buc2, res_obj2]
storage = Storage(self.session, self.engine)
inv_index_id = storage.open()
for resource in resources:
storage.write(resource)
storage.commit()
# add more resource data that belongs to a different inventory index
storage = Storage(self.session, self.engine)
storage.open()
for resource in resources:
storage.write(resource)
storage.commit()
inv_index = self.session.query(InventoryIndex).get(inv_index_id)
expected = {'bucket': 2, 'object': 1, 'organization': 1, 'project': 2}
inv_summary = inv_index.get_summary(self.session)
self.assertEqual(expected, inv_summary)
def test_whether_resource_should_be_inserted_or_skipped(self):
"""Whether the resource should be inserted or skipped.
All resources should not be written if they have been previously
written. Except group members, where members can be in multiple groups.
"""
initialize(self.engine)
scoped_sessionmaker = db.create_scoped_sessionmaker(self.engine)
res_org = ResourceMock('1', {'id': 'test'}, 'organization', 'resource')
res_proj1 = ResourceMock('2', {'id': 'test'}, 'project', 'resource',
res_org)
res_proj1.set_iam_policy({'id': 'test'})
res_proj1.set_billing_info({'id': 'test'})
res_buc1 = ResourceMock('5', {'id': 'test'}, 'bucket', 'resource',
res_org)
res_proj2 = ResourceMock('6', {'id': 'test'}, 'project', 'resource',
res_org)
res_buc2 = ResourceMock('7', {'id': 'test'}, 'bucket', 'resource',
res_proj2)
res_obj2 = ResourceMock('8', {'id': 'test'}, 'object', 'resource',
res_buc2)
res_group1 = ResourceMock('9', {'id': 'test'}, 'google_group',
'resource', res_org)
res_group2 = ResourceMock('10', {'id': 'test'}, 'google_group',
'resource', res_org)
res_group_member1 = ResourceMock('11', {'id': 'user111',
'kind': 'admin#directory#member'},
'gsuite_group_member',
'resource', res_group1)
res_group_member2 = ResourceMock('11', {'id': 'user111',
'kind': 'admin#directory#member'},
'gsuite_group_member',
'resource', res_group2)
res_group_member3 = ResourceMock('12', {'id': 'user222',
'kind': 'admin#directory#member'},
'gsuite_group_member',
'resource', res_group1)
res_proj3 = ResourceMock('6', {'id': 'dup_proj'}, 'project',
'resource', res_org)
resources = [
res_org,
res_proj1,
res_buc1,
res_proj2,
res_buc2,
res_obj2,
res_proj3,
res_group1,
res_group2,
res_group_member1,
res_group_member2,
res_group_member3
]
with scoped_sessionmaker() as session:
with Storage(session, self.engine) as storage:
for resource in resources:
storage.write(resource)
storage.commit()
inventory_index_id = storage.inventory_index.id
self.assertEqual(3,
len(self.reduced_inventory(
session,
inventory_index_id,
['organization', 'project'])),
'Only 1 organization and 2 unique projects')
self.assertEqual(3,
len(self.reduced_inventory(
session,
inventory_index_id,
['gsuite_group_member'])),
'All group members should be stored.')
self.assertEqual(11,
len(self.reduced_inventory(
session,
inventory_index_id,
[])),
'No types should yield empty list')