def test_empty_list_api_response(self, mock_list): source_id = 'organizations/123/sources/456' violations = [{ 'violation_hash': '311', 'resource_name': 'readme1', 'resource_data': {u'ipv4Enabled': True, u'authorizedNetworks': [ { u'expirationTime': u'1970-01-01T00:00:00Z', u'kind': u'sql#aclEntry', u'value': u'0.0.0.0/0'}]}, 'resource_id': 'readme1', 'violation_type': 'CLOUD_SQL_VIOLATION', 'created_at_datetime': '2018-03-26T04:37:51Z', 'scanner_index_id': 122, 'rule_name': 'Cloud SQL rule to search for publicly exposed instances', 'full_name': 'organization/123/project/cicd-henry/cloudsqlinstance/456/', 'rule_index': 0, 'violation_data': {u'instance_name': u'readme1', u'require_ssl': False, u'project_id': u'readme1', u'authorized_networks': [u'0.0.0.0/0'], u'full_name': u'organization/123/project/cicd-henry/cloudsqlinstance/456/'}, 'id': 99185, 'resource_type': 'cloudsqlinstance'}] mock_list.list_findings.return_value = {'readTime': '111'} notifier = cscc_notifier.CsccNotifier('abc', self.api_quota) notifier._send_findings_to_cscc(violations, source_id) self.assertFalse(mock_list.update_finding.called)
def test_modes_are_run_correctly(self, mock_logger): # This whole test case is for alpha API, and can be deleted # when CSCC alpha support is removed. notifier = cscc_notifier.CsccNotifier(None) notifier._send_findings_to_gcs = mock.MagicMock() notifier._send_findings_to_cscc = mock.MagicMock() notifier.LOGGER = mock.MagicMock() self.assertEquals(0, notifier._send_findings_to_gcs.call_count) notifier.run(None, None, None, None) self.assertEquals(1, notifier._send_findings_to_gcs.call_count) notifier.run(None, None, 'bucket', None) self.assertEquals(2, notifier._send_findings_to_gcs.call_count) # alpha api self.assertEquals(0, notifier._send_findings_to_cscc.call_count) notifier.run(None, None, 'api', None) self.assertEquals(1, notifier._send_findings_to_cscc.call_count) self.assertEquals(3, mock_logger.info.call_count) notifier.run(None, None, 'foo', None) self.assertEquals(5, mock_logger.info.call_count) self.assertTrue( 'not selected' in mock_logger.info.call_args_list[4][0][0])
def test_can_transform_to_alpha_findings_in_api_mode(self): expected_alpha_findings = [{ 'assetIds': ['full_name_111'], 'category': 'disallow_all_ports_111', 'eventTime': '2010-08-28T10:20:30Z', 'id': '539cfbdb1113a74ec18edf583eada77a', 'properties': { 'db_source': 'table:violations/id:1', 'inventory_index_id': 'iii', 'resource_data': 'inventory_data_111', 'resource_id': 'fake_firewall_111', 'resource_type': 'firewall_rule', 'rule_index': 111, 'scanner_index_id': 1282990830000000, 'violation_data': '{"policy_names": ["fw-tag-match_111"], "recommended_actions": {"DELETE_FIREWALL_RULES": ["fw-tag-match_111"]}}' }, 'source_id': 'FORSETI' }, { 'assetIds': ['full_name_222'], 'category': 'disallow_all_ports_222', 'eventTime': '2010-08-28T10:20:30Z', 'id': '3eff279ccb96799d9eb18e6b76055b22', 'properties': { 'db_source': 'table:violations/id:2', 'inventory_index_id': 'iii', 'resource_data': 'inventory_data_222', 'resource_id': 'fake_firewall_222', 'resource_type': 'firewall_rule', 'rule_index': 222, 'scanner_index_id': 1282990830000000, 'violation_data': '{"policy_names": ["fw-tag-match_222"], "recommended_actions": {"DELETE_FIREWALL_RULES": ["fw-tag-match_222"]}}' }, 'source_id': 'FORSETI' }] violations_as_dict = self._populate_and_retrieve_violations() finding_results = (cscc_notifier.CsccNotifier( 'iii')._transform_for_api(violations_as_dict)) self.assertEquals(expected_alpha_findings, finding_results)
def test_beta_api_is_invoked_correctly(self): notifier = cscc_notifier.CsccNotifier(None) notifier._send_findings_to_cscc = mock.MagicMock() notifier.LOGGER = mock.MagicMock() self.assertEquals(0, notifier._send_findings_to_cscc.call_count) notifier.run(None, None, 'api', None, source_id='111') calls = notifier._send_findings_to_cscc.call_args_list call = calls[0] _, kwargs = call self.assertEquals('111', kwargs['source_id'])
def test_can_transform_to_findings_in_bucket_mode(self, mock_get_utc_now): expected_findings = [{ 'finding_id': '539cfbdb1113a74ec18edf583eada77ab1a60542c6edcb4120b50f34629b6b69041c13f0447ab7b2526d4c944c88670b6f151fa88444c30771f47a3b813552ff', 'finding_summary': 'disallow_all_ports_111', 'finding_source_id': 'FORSETI', 'finding_category': 'FIREWALL_BLACKLIST_VIOLATION_111', 'finding_asset_ids': 'full_name_111', 'finding_time_event': '2010-08-28T10:20:30Z', 'finding_callback_url': 'gs://foo_bucket', 'finding_properties': { 'db_source': 'table:violations/id:1', 'inventory_index_id': 'iii', 'resource_id': 'fake_firewall_111', 'resource_data': 'inventory_data_111', 'rule_index': 111, 'scanner_index_id': 1282990830000000, 'violation_data': '{"policy_names": ["fw-tag-match_111"], "recommended_actions": {"DELETE_FIREWALL_RULES": ["fw-tag-match_111"]}}', 'resource_type': u'firewall_rule' }, }, { 'finding_id': '3eff279ccb96799d9eb18e6b76055b2242d1f2e6f14c1fb3bb48f7c8c03b4ce4db577d67c0b91c5914902d906bf1703d5bbba0ceaf29809ac90fef3bf6aa5417', 'finding_summary': 'disallow_all_ports_222', 'finding_source_id': 'FORSETI', 'finding_category': 'FIREWALL_BLACKLIST_VIOLATION_222', 'finding_asset_ids': 'full_name_222', 'finding_time_event': '2010-08-28T10:20:30Z', 'finding_callback_url': 'gs://foo_bucket', 'finding_properties': { 'db_source': 'table:violations/id:2', 'inventory_index_id': 'iii', 'resource_id': 'fake_firewall_222', 'resource_data': 'inventory_data_222', 'rule_index': 222, 'scanner_index_id': 1282990830000000, 'violation_data': '{"policy_names": ["fw-tag-match_222"], "recommended_actions": {"DELETE_FIREWALL_RULES": ["fw-tag-match_222"]}}', 'resource_type': u'firewall_rule' }, }] violations_as_dict = self._populate_and_retrieve_violations() finding_results = ( cscc_notifier.CsccNotifier('iii')._transform_for_gcs( violations_as_dict, 'gs://foo_bucket')) self.assertEquals(expected_findings, finding_results)
def test_can_transform_to_beta_findings_in_api_mode(self): expected_beta_findings = [ ['539cfbdb1113a74ec18edf583eada77a', {'category': 'FIREWALL_BLACKLIST_VIOLATION_111', 'resource_name': 'full_name_111', 'name': 'organizations/11111/sources/22222/findings/539cfbdb1113a74ec18edf583eada77a', 'parent': 'organizations/11111/sources/22222', 'event_time': '2010-08-28T10:20:30Z', 'state': 'ACTIVE', 'source_properties': { 'source': 'FORSETI', 'rule_name': 'disallow_all_ports_111', 'inventory_index_id': 'iii', 'resource_data': '"inventory_data_111"', 'db_source': 'table:violations/id:1', 'rule_index': 111, 'violation_data': '"{\\"policy_names\\": [\\"fw-tag-match_111\\"], \\"recommended_actions\\": {\\"DELETE_FIREWALL_RULES\\": [\\"fw-tag-match_111\\"]}}"', 'resource_id': 'fake_firewall_111', 'scanner_index_id': 1282990830000000, 'resource_type': 'firewall_rule'}}], ['3eff279ccb96799d9eb18e6b76055b22', {'category': 'FIREWALL_BLACKLIST_VIOLATION_222', 'resource_name': 'full_name_222', 'name': 'organizations/11111/sources/22222/findings/3eff279ccb96799d9eb18e6b76055b22', 'parent': 'organizations/11111/sources/22222', 'event_time': '2010-08-28T10:20:30Z', 'state': 'ACTIVE', 'source_properties': { 'source': 'FORSETI', 'rule_name': 'disallow_all_ports_222', 'inventory_index_id': 'iii', 'resource_data': '"inventory_data_222"', 'db_source': 'table:violations/id:2', 'rule_index': 222, 'violation_data': '"{\\"policy_names\\": [\\"fw-tag-match_222\\"], \\"recommended_actions\\": {\\"DELETE_FIREWALL_RULES\\": [\\"fw-tag-match_222\\"]}}"', 'resource_id': 'fake_firewall_222', 'scanner_index_id': 1282990830000000, 'resource_type': 'firewall_rule'}}]] violations_as_dict = self._populate_and_retrieve_violations() finding_results = ( cscc_notifier.CsccNotifier('iii')._transform_for_api( violations_as_dict, source_id='organizations/11111/sources/22222')) self.assertEquals(expected_beta_findings, ast.literal_eval(json.dumps(finding_results)))
def test_outdated_findings_are_not_found(self): NEW_FINDINGS = [['abc', {'category': 'BUCKET_VIOLATION', 'resource_name': 'organization/123/project/inventoryscanner/bucket/isthispublic/', 'name': 'organizations/123/sources/560/findings/abc', 'parent': 'organizations/123/sources/560', 'event_time': '2019-03-12T16:06:19Z', 'state': 'ACTIVE', 'source_properties': {'source': 'FORSETI', 'rule_name': 'Bucket acls rule to search for public buckets', 'inventory_index_id': 789, 'resource_data': '{"bucket": "isthispublic", "entity": "allUsers", "id": "isthispublic/allUsers", "role": "READER"}', 'db_source': 'table:violations/id:94953', 'rule_index': 0, 'violation_data': '{"bucket": "isthispublic", "domain": "", "email": "", "entity": "allUsers", "full_name": "organization/123/project/inventoryscanner/bucket/isthispublic/", "project_id": "inventoryscanner-henry", "role": "READER"}', 'resource_id': 'isthispublic', 'scanner_index_id': 1551913369403591, 'resource_type': 'bucket'}}]] FINDINGS_IN_CSCC = [['abc', {'category': 'BUCKET_VIOLATION', 'resource_name': 'organization/123/project/inventoryscanner/bucket/isthispublic/', 'name': 'organizations/123/sources/560/findings/abc', 'parent': 'organizations/123/sources/560', 'event_time': '2019-03-12T16:06:19Z', 'state': 'ACTIVE', 'source_properties': {'source': 'FORSETI', 'rule_name': 'Bucket acls rule to search for public buckets', 'inventory_index_id': 789, 'resource_data': '{"bucket": "isthispublic", "entity": "allUsers", "id": "isthispublic/allUsers", "role": "READER"}', 'db_source': 'table:violations/id:94953', 'rule_index': 0, 'violation_data': '{"bucket": "isthispublic", "domain": "", "email": "", "entity": "allUsers", "full_name": "organization/123/project/inventoryscanner/bucket/isthispublic/", "project_id": "inventoryscanner-henry", "role": "READER"}', 'resource_id': 'isthispublic', 'scanner_index_id': 1551913369403591, 'resource_type': 'bucket'}}]] notifier = cscc_notifier.CsccNotifier('123', self.api_quota) inactive_findings = notifier.find_inactive_findings(NEW_FINDINGS, FINDINGS_IN_CSCC) assert(len(inactive_findings)) == 0
'Bucket acls rule to search for public buckets', 'inventory_index_id': 789, 'resource_data': '{"bucket": "isthispublic", "entity": "allUsers", "id": "isthispublic/allUsers", "role": "READER"}', 'db_source': 'table:violations/id:94953', 'rule_index': 0L, 'violation_data': '{"bucket": "isthispublic", "domain": "", "email": "", "entity": "allUsers", "full_name": "organization/123/project/inventoryscanner/bucket/isthispublic/", "project_id": "inventoryscanner", "role": "READER"}', 'resource_id': 'isthispublic', 'scanner_index_id': 1551913369403591L, 'resource_type': 'bucket' } } ]] notifier = cscc_notifier.CsccNotifier('123') inactive_findings = notifier.find_inactive_findings( NEW_FINDINGS, FINDINGS_IN_CSCC) self.assertEquals(EXPECTED_INACTIVE_FINDINGS[0][1]['state'], inactive_findings[0][1]['state']) def test_outdated_findings_are_not_found(self): NEW_FINDINGS = [[ 'abc', { 'category': 'BUCKET_VIOLATION', 'resource_name': 'organization/123/project/inventoryscanner/bucket/isthispublic/', 'name': 'organizations/123/sources/560/findings/abc', 'parent': 'organizations/123/sources/560',
def test_can_transform_to_findings_in_api_mode(self): expected_findings = [ [ 'f3eb2be2ed015563d7dc4d4aea798a0b', { 'category': 'FIREWALL_BLACKLIST_VIOLATION_111', 'resource_name': 'full_name_111', 'name': 'organizations/11111/sources/22222/findings/f3eb2be2ed015563d7dc4d4aea798a0b', 'parent': 'organizations/11111/sources/22222', 'event_time': '2010-08-28T10:20:30Z', 'state': 'ACTIVE', 'source_properties': { 'source': 'FORSETI', 'rule_name': 'disallow_all_ports_111', 'inventory_index_id': 'iii', 'resource_data': '"inventory_data_111"', 'db_source': 'table:violations/id:1', 'rule_index': 111, 'violation_data': '"{\\"policy_names\\": [\\"fw-tag-match_111\\"], \\"recommended_actions\\": {\\"DELETE_FIREWALL_RULES\\": [\\"fw-tag-match_111\\"]}}"', 'resource_id': 'fake_firewall_111', 'scanner_index_id': 1282990830000000, 'resource_type': 'firewall_rule' } } ], [ '73f4a4ac87a76a2e9d2c7854ac8fa077', { 'category': 'FIREWALL_BLACKLIST_VIOLATION_222', 'resource_name': 'full_name_222', 'name': 'organizations/11111/sources/22222/findings/73f4a4ac87a76a2e9d2c7854ac8fa077', 'parent': 'organizations/11111/sources/22222', 'event_time': '2010-08-28T10:20:30Z', 'state': 'ACTIVE', 'source_properties': { 'source': 'FORSETI', 'rule_name': 'disallow_all_ports_222', 'inventory_index_id': 'iii', 'resource_data': '"inventory_data_222"', 'db_source': 'table:violations/id:2', 'rule_index': 222, 'violation_data': '"{\\"policy_names\\": [\\"fw-tag-match_222\\"], \\"recommended_actions\\": {\\"DELETE_FIREWALL_RULES\\": [\\"fw-tag-match_222\\"]}}"', 'resource_id': 'fake_firewall_222', 'scanner_index_id': 1282990830000000, 'resource_type': 'firewall_rule' } } ] ] violations_as_dict = self._populate_and_retrieve_violations() finding_results = (cscc_notifier.CsccNotifier( 'iii', self.api_quota)._transform_for_api( violations_as_dict, source_id='organizations/11111/sources/22222')) self.assertEqual(expected_findings, ast.literal_eval(json.dumps(finding_results)))
def run(inventory_index_id, scanner_index_id, progress_queue, service_config=None): """Run the notifier. Entry point when the notifier is run as a library. Args: inventory_index_id (int64): Inventory index id. scanner_index_id (int64): Scanner index id. progress_queue (Queue): The progress queue. service_config (ServiceConfig): Forseti 2.0 service configs. Returns: int: Status code. """ # pylint: disable=too-many-locals global_configs = service_config.get_global_config() notifier_configs = service_config.get_notifier_config() with service_config.scoped_session() as session: if scanner_index_id: inventory_index_id = ( DataAccess.get_inventory_index_id_by_scanner_index_id( session, scanner_index_id)) else: if not inventory_index_id: inventory_index_id = ( DataAccess.get_latest_inventory_index_id(session)) scanner_index_id = scanner_dao.get_latest_scanner_index_id( session, inventory_index_id) if not scanner_index_id: LOGGER.error( 'No success or partial success scanner index found for ' 'inventory index: "%s".', str(inventory_index_id)) else: # get violations violation_access = scanner_dao.ViolationAccess(session) violations = violation_access.list( scanner_index_id=scanner_index_id) violations_as_dict = [] for violation in violations: violations_as_dict.append( scanner_dao.convert_sqlalchemy_object_to_dict(violation)) violations_as_dict = convert_to_timestamp(violations_as_dict) violation_map = scanner_dao.map_by_resource(violations_as_dict) for retrieved_v in violation_map: log_message = ( 'Retrieved {} violations for resource \'{}\''.format( len(violation_map[retrieved_v]), retrieved_v)) LOGGER.info(log_message) progress_queue.put(log_message) # build notification notifiers notifiers = [] for resource in notifier_configs['resources']: if violation_map.get(resource['resource']) is None: log_message = 'Resource \'{}\' has no violations'.format( resource['resource']) progress_queue.put(log_message) LOGGER.info(log_message) continue if not resource['should_notify']: LOGGER.debug('Not notifying for: %s', resource['resource']) continue for notifier in resource['notifiers']: log_message = ( 'Running \'{}\' notifier for resource \'{}\''.format( notifier['name'], resource['resource'])) progress_queue.put(log_message) LOGGER.info(log_message) chosen_pipeline = find_notifiers(notifier['name']) notifiers.append(chosen_pipeline( resource['resource'], inventory_index_id, violation_map[resource['resource']], global_configs, notifier_configs, notifier.get('configuration'))) # Run the notifiers. for notifier in notifiers: notifier.run() # Run the CSCC notifier. violation_configs = notifier_configs.get('violation') if violation_configs: if violation_configs.get('cscc').get('enabled'): source_id = violation_configs.get('cscc').get('source_id') if source_id: # beta mode LOGGER.debug( 'Running CSCC notifier with beta API. source_id: ' '%s', source_id) (cscc_notifier.CsccNotifier(inventory_index_id) .run(violations_as_dict, source_id=source_id)) else: # alpha mode LOGGER.debug('Running CSCC notifier with alpha API.') gcs_path = ( violation_configs.get('cscc').get('gcs_path')) mode = violation_configs.get('cscc').get('mode') organization_id = ( violation_configs.get('cscc').get( 'organization_id')) (cscc_notifier.CsccNotifier(inventory_index_id) .run(violations_as_dict, gcs_path, mode, organization_id)) InventorySummary(service_config, inventory_index_id).run() log_message = 'Notification completed!' progress_queue.put(log_message) progress_queue.put(None) LOGGER.info(log_message) return 0