def secrets(config, **kwargs): sys.path.append(config.configuration.env_dir) secrets = SecretsValidator(configuration=config.configuration, is_circle=kwargs['post_commit'], ignore_entropy=kwargs['ignore_entropy'], white_list_path=kwargs['whitelist']) return secrets.run()
class TestSecrets: FILES_PATH = os.path.normpath( os.path.join(__file__, f'{git_path()}/demisto_sdk/tests', 'test_files')) TEST_BASE_PATH = os.path.join(FILES_PATH, 'fake_integration/') TEST_YML_FILE = TEST_BASE_PATH + 'fake_integration.yml' TEST_PY_FILE = TEST_BASE_PATH + 'fake_integration.py' TEST_WHITELIST_FILE_PACKS = TEST_BASE_PATH + 'fake.secrets-ignore' TEST_WHITELIST_FILE = TEST_BASE_PATH + 'fake_secrets_white_list.json' TEST_BASE_64_STRING = 'OCSn7JGqKehoyIyMCm7gPFjKXpawXvh2M32' * 20 + ' sade' WHITE_LIST_FILE_NAME = 'secrets_white_list.json' FILE_HASH_LIST = [ '123c8fc6532ba547d7ef598', '456c8fc6532ba547d7bb5e880a', '789c8fc6532ba57ef5985bb5e' ] MAIL_LIST = ['*****@*****.**', '*****@*****.**', '*****@*****.**'] IP_LIST = ['ip-172-31-15-237', '1.1.1.1', '12.25.12.14'] TEMP_DIR = os.path.join(FILES_PATH, 'temp') TEST_FILE_WITH_SECRETS = os.path.join(TEMP_DIR, 'file_with_secrets_in_it.yml') validator = SecretsValidator(is_circle=True, white_list_path=os.path.join( FILES_PATH, WHITE_LIST_FILE_NAME)) @classmethod def setup_class(cls): print("Setups TestSecrets class") if not os.path.exists(TestSecrets.TEMP_DIR): os.mkdir(TestSecrets.TEMP_DIR) @classmethod def teardown_class(cls): print("Tearing down TestSecrets class") if os.path.exists(TestSecrets.TEMP_DIR): shutil.rmtree(TestSecrets.TEMP_DIR, ignore_errors=False, onerror=None) def test_get_diff_text_files(self): changed_files = '''A\tIntegrations/Recorded_Future/Recorded_Future.yml\n D\tIntegrations/integration-Recorded_Future.yml''' get_diff = self.validator.get_diff_text_files(changed_files) assert 'Integrations/Recorded_Future/Recorded_Future.yml' in get_diff def test_is_text_file(self): changed_files = 'Integrations/Recorded_Future/Recorded_Future.yml' is_txt = self.validator.is_text_file(changed_files) assert is_txt is True def test_search_potential_secrets__no_secrets_found(self): secret_to_location = self.validator.search_potential_secrets( [self.TEST_YML_FILE]) assert not secret_to_location def test_search_potential_secrets__secrets_found(self): create_empty_whitelist_secrets_file( os.path.join(TestSecrets.TEMP_DIR, TestSecrets.WHITE_LIST_FILE_NAME)) validator = SecretsValidator(is_circle=True, white_list_path=os.path.join( TestSecrets.TEMP_DIR, TestSecrets.WHITE_LIST_FILE_NAME)) with io.open(self.TEST_FILE_WITH_SECRETS, 'w') as f: f.write(''' print('This is our dummy code') a = 100 b = 300 c = a + b API_KEY = OIifdsnsjkgnj3254nkdfsjKNJD0345 # this is our secret some_dict = { 'some_foo': 100 } print(some_dict.some_foo) ''') secrets_found = validator.search_potential_secrets( [self.TEST_FILE_WITH_SECRETS]) assert secrets_found[self.TEST_FILE_WITH_SECRETS] == { 7: ['OIifdsnsjkgnj3254nkdfsjKNJD0345'] } def test_ignore_entropy(self): """ - no items in the whitelist - file contains 2 secrets: - email - password - run validate secrets with --ignore-entropy=True - ensure email found - ensure entropy code was not executed - no secrets have found """ create_empty_whitelist_secrets_file( os.path.join(TestSecrets.TEMP_DIR, TestSecrets.WHITE_LIST_FILE_NAME)) validator = SecretsValidator(is_circle=True, ignore_entropy=True, white_list_path=os.path.join( TestSecrets.TEMP_DIR, TestSecrets.WHITE_LIST_FILE_NAME)) with io.open(self.TEST_FILE_WITH_SECRETS, 'w') as f: f.write(''' print('This is our dummy code') my_email = "*****@*****.**" API_KEY = OIifdsnsjkgnj3254nkdfsjKNJD0345 # this is our secret some_dict = { 'some_foo': 100 } ''') secrets_found = validator.search_potential_secrets( [self.TEST_FILE_WITH_SECRETS], True) assert secrets_found[self.TEST_FILE_WITH_SECRETS] == { 4: ['*****@*****.**'] } def test_two_files_with_same_name(self): """ - no items in the whitelist - file contains 1 secret: - email - run validate secrets with --ignore-entropy=True - ensure secret is found in two files from different directories with the same base name """ create_empty_whitelist_secrets_file( os.path.join(TestSecrets.TEMP_DIR, TestSecrets.WHITE_LIST_FILE_NAME)) dir1_path = os.path.join(TestSecrets.TEMP_DIR, "dir1") dir2_path = os.path.join(TestSecrets.TEMP_DIR, "dir2") os.mkdir(dir1_path) os.mkdir(dir2_path) validator = SecretsValidator(is_circle=True, ignore_entropy=True, white_list_path=os.path.join( TestSecrets.TEMP_DIR, TestSecrets.WHITE_LIST_FILE_NAME)) file_name = 'README.md' file1_path = os.path.join(dir1_path, file_name) file2_path = os.path.join(dir2_path, file_name) for file_path in [file1_path, file2_path]: with io.open(file_path, 'w') as f: f.write(''' print('This is our dummy code') my_email = "*****@*****.**" ''') secrets_found = validator.search_potential_secrets( [file1_path, file2_path], True) assert secrets_found[os.path.join(dir1_path, file_name)] == { 4: ['*****@*****.**'] } assert secrets_found[os.path.join(dir2_path, file_name)] == { 4: ['*****@*****.**'] } def test_remove_white_list_regex(self): white_list = '155.165.45.232' file_contents = ''' boop shmoop 155.165.45.232 ''' file_contents = self.validator.remove_whitelisted_items_from_file( file_contents, {white_list}) assert white_list not in file_contents def test_remove_whitelisted_items_from_file_escaped_whitelist(self): """ Given - White list with a term that can be regex (***.). - String with no content When - Removing terms containing that regex Then - Ensure secrets that the secret isn't in the output. - Ensure no error raised """ white_list = '***.url' file_contents = ''' Random and unmeaningful file content a string containing ***.url ''' file_contents = self.validator.remove_whitelisted_items_from_file( file_contents, {white_list}) assert white_list not in file_contents def test_remove_whitelisted_items_from_file_substring(self): white_list = 'url.com' file_contents = ''' url.com boop [email protected] shmoop https://url.com ''' assert white_list not in self.validator.remove_whitelisted_items_from_file( file_contents, {white_list}) def test_temp_white_list(self): file_contents = self.validator.get_file_contents( self.TEST_YML_FILE, '.yml') temp_white_list = self.validator.create_temp_white_list(file_contents) assert 'sha256' in temp_white_list def test_get_related_yml_contents(self): yml_file_contents = self.validator.retrieve_related_yml( os.path.dirname(self.TEST_PY_FILE)) assert 'Use the Zoom integration manage your Zoom users and meetings' in yml_file_contents def test_regex_for_secrets(self): line = 'dockerimage: demisto/duoadmin:1.0.0.147 199.199.178.199 123e4567-e89b-12d3-a456-426655440000' secrets, false_positives = self.validator.regex_for_secrets(line) assert '1.0.0.147' in false_positives assert '123e4567-e89b-12d3-a456-426655440000' in false_positives assert '199.199.178.199' in secrets def test_calculate_shannon_entropy(self): test_string = 'SADE' entropy = self.validator.calculate_shannon_entropy(test_string) assert entropy == 2.0 def test_get_packs_white_list(self): final_white_list, ioc_white_list, files_while_list = \ self.validator.get_packs_white_list(self.TEST_WHITELIST_FILE_PACKS) assert ioc_white_list == [] assert files_while_list == [] assert final_white_list == [ 'boop', 'sade', 'sade.txt', '*****@*****.**', '' ] def test_get_generic_white_list(self): final_white_list, ioc_white_list, files_while_list = \ self.validator.get_generic_white_list(self.TEST_WHITELIST_FILE) assert ioc_white_list == ['*****@*****.**'] assert files_while_list == ['sade.txt'] assert final_white_list == ['*****@*****.**', 'aboop', 'asade'] def test_remove_false_positives(self): line = '[I AM MISTER MEESEEKS LOOK AT ME] sade' line = self.validator.remove_false_positives(line) assert line == ' sade' def test_is_secrets_disabled(self): line1 = 'disable-secrets-detection' skip_secrets = {'skip_once': False, 'skip_multi': False} skip_secrets = self.validator.is_secrets_disabled(line1, skip_secrets) assert skip_secrets['skip_once'] and not skip_secrets['skip_multi'] skip_secrets['skip_once'] = False line2 = 'disable-secrets-detection-start' skip_secrets = self.validator.is_secrets_disabled(line2, skip_secrets) assert not skip_secrets['skip_once'] and skip_secrets['skip_multi'] line3 = 'disable-secrets-detection-end' skip_secrets = self.validator.is_secrets_disabled(line3, skip_secrets) assert not skip_secrets['skip_once'] and not skip_secrets['skip_multi'] def test_ignore_base64(self): file_contents = self.TEST_BASE_64_STRING file_contents = self.validator.ignore_base64(file_contents) assert file_contents.lstrip() == 'sade' def test_get_white_listed_items_not_pack(self): final_white_list, ioc_white_list, files_white_list = self.validator.get_white_listed_items( False, None) assert final_white_list == { 'https://api.zoom.us', 'PaloAltoNetworksXDR', 'ip-172-31-15-237' } assert ioc_white_list == {'https://api.zoom.us'} assert files_white_list == set() def test_get_white_listed_items_pack(self, monkeypatch): monkeypatch.setattr('demisto_sdk.commands.secrets.secrets.PACKS_DIR', self.FILES_PATH) final_white_list, ioc_white_list, files_white_list = self.validator.get_white_listed_items( True, 'fake_pack') assert final_white_list == { 'https://www.demisto.com', 'https://api.zoom.us', 'PaloAltoNetworksXDR', 'ip-172-31-15-237' } assert ioc_white_list == {'https://api.zoom.us'} assert files_white_list == set() def test_reformat_secrets_output(self): secrets_output = self.validator.reformat_secrets_output( self.FILE_HASH_LIST) assert secrets_output == '123c8fc6532ba547d7ef598\n456c8fc6532ba547d7bb5e880a\n789c8fc6532ba57ef5985bb5e' secrets_output = self.validator.reformat_secrets_output(self.MAIL_LIST) assert secrets_output == '[email protected]\[email protected]\[email protected]' secrets_output = self.validator.reformat_secrets_output(self.IP_LIST) assert secrets_output == 'ip-172-31-15-237\n1.1.1.1\n12.25.12.14' secrets_output = self.validator.reformat_secrets_output([]) assert secrets_output == '' def test_get_all_diff_text_files(self, mocker): mocker.patch( 'demisto_sdk.commands.secrets.secrets.run_command', return_value='m\tPacks/Integrations/integration/testing.py\n') validator = SecretsValidator(is_circle=True, white_list_path=os.path.join( TestSecrets.TEMP_DIR, TestSecrets.WHITE_LIST_FILE_NAME)) assert validator.get_all_diff_text_files( 'master', True) == ['Packs/Integrations/integration/testing.py'] assert validator.get_all_diff_text_files( 'master', False) == ['Packs/Integrations/integration/testing.py'] validator.prev_ver = 'Testing_branch' assert validator.get_all_diff_text_files( 'master', True) == ['Packs/Integrations/integration/testing.py'] assert validator.get_all_diff_text_files( 'master', False) == ['Packs/Integrations/integration/testing.py']
class TestSecrets: FILES_PATH = os.path.normpath( os.path.join(__file__, f'{git_path()}/demisto_sdk/tests', 'test_files')) TEST_BASE_PATH = os.path.join(FILES_PATH, 'fake_integration/') TEST_YML_FILE = TEST_BASE_PATH + 'fake_integration.yml' TEST_PY_FILE = TEST_BASE_PATH + 'fake_integration.py' TEST_WHITELIST_FILE_PACKS = TEST_BASE_PATH + 'fake.secrets-ignore' TEST_WHITELIST_FILE = TEST_BASE_PATH + 'fake_secrets_white_list.json' TEST_BASE_64_STRING = 'OCSn7JGqKehoyIyMCm7gPFjKXpawXvh2M32' * 20 + ' sade' WHITE_LIST_FILE_NAME = 'secrets_white_list.json' TEMP_DIR = os.path.join(FILES_PATH, 'temp') TEST_FILE_WITH_SECRETS = os.path.join(TEMP_DIR, 'file_with_secrets_in_it.yml') validator = SecretsValidator(is_circle=True, white_list_path=os.path.join( FILES_PATH, WHITE_LIST_FILE_NAME)) @classmethod def setup_class(cls): print("Setups TestSecrets class") if not os.path.exists(TestSecrets.TEMP_DIR): os.mkdir(TestSecrets.TEMP_DIR) @classmethod def teardown_class(cls): print("Tearing down TestSecrets class") if os.path.exists(TestSecrets.TEMP_DIR): shutil.rmtree(TestSecrets.TEMP_DIR, ignore_errors=False, onerror=None) def test_get_diff_text_files(self): changed_files = ''' A Integrations/Recorded_Future/Recorded_Future.yml D Integrations/integration-Recorded_Future.yml''' get_diff = self.validator.get_diff_text_files(changed_files) assert 'Integrations/Recorded_Future/Recorded_Future.yml' in get_diff def test_is_text_file(self): changed_files = 'Integrations/Recorded_Future/Recorded_Future.yml' is_txt = self.validator.is_text_file(changed_files) assert is_txt is True def test_search_potential_secrets__no_secrets_found(self): secrets_found = self.validator.search_potential_secrets( [self.TEST_YML_FILE]) assert not secrets_found def test_search_potential_secrets__secrets_found(self): create_empty_whitelist_secrets_file( os.path.join(TestSecrets.TEMP_DIR, TestSecrets.WHITE_LIST_FILE_NAME)) validator = SecretsValidator(is_circle=True, white_list_path=os.path.join( TestSecrets.TEMP_DIR, TestSecrets.WHITE_LIST_FILE_NAME)) with io.open(self.TEST_FILE_WITH_SECRETS, 'w') as f: f.write(''' print('This is our dummy code') a = 100 b = 300 c = a + b API_KEY = OIifdsnsjkgnj3254nkdfsjKNJD0345 # this is our secret some_dict = { 'some_foo': 100 } print(some_dict.some_foo) ''') secrets_found = validator.search_potential_secrets( [self.TEST_FILE_WITH_SECRETS]) assert secrets_found['file_with_secrets_in_it.yml'] == [ 'OIifdsnsjkgnj3254nkdfsjKNJD0345' ] def test_ignore_entropy(self): """ - no items in the whitelist - file contains 2 secrets: - email - password - run validate secrets with --ignore-entropy=True - ensure email found - ensure entropy code was not executed - no secrets have found """ create_empty_whitelist_secrets_file( os.path.join(TestSecrets.TEMP_DIR, TestSecrets.WHITE_LIST_FILE_NAME)) validator = SecretsValidator(is_circle=True, ignore_entropy=True, white_list_path=os.path.join( TestSecrets.TEMP_DIR, TestSecrets.WHITE_LIST_FILE_NAME)) with io.open(self.TEST_FILE_WITH_SECRETS, 'w') as f: f.write(''' print('This is our dummy code') my_email = "*****@*****.**" API_KEY = OIifdsnsjkgnj3254nkdfsjKNJD0345 # this is our secret some_dict = { 'some_foo': 100 } ''') secrets_found = validator.search_potential_secrets( [self.TEST_FILE_WITH_SECRETS], True) assert secrets_found['file_with_secrets_in_it.yml'] == [ '*****@*****.**' ] def test_remove_white_list_regex(self): white_list = '155.165.45.232' file_contents = ''' boop shmoop 155.165.45.232 ''' file_contents = self.validator.remove_white_list_regex( white_list, file_contents) assert white_list not in file_contents def test_temp_white_list(self): file_contents = self.validator.get_file_contents( self.TEST_YML_FILE, '.yml') temp_white_list = self.validator.create_temp_white_list(file_contents) assert 'sha256' in temp_white_list def test_get_related_yml_contents(self): yml_file_contents = self.validator.retrieve_related_yml( os.path.dirname(self.TEST_PY_FILE)) assert 'Use the Zoom integration manage your Zoom users and meetings' in yml_file_contents def test_regex_for_secrets(self): line = 'dockerimage: demisto/duoadmin:1.0.0.147 199.199.178.199 123e4567-e89b-12d3-a456-426655440000' secrets, false_positives = self.validator.regex_for_secrets(line) assert '1.0.0.147' in false_positives assert '123e4567-e89b-12d3-a456-426655440000' in false_positives assert '199.199.178.199' in secrets def test_calculate_shannon_entropy(self): test_string = 'SADE' entropy = self.validator.calculate_shannon_entropy(test_string) assert entropy == 2.0 def test_get_packs_white_list(self): final_white_list, ioc_white_list, files_while_list = \ self.validator.get_packs_white_list(self.TEST_WHITELIST_FILE_PACKS) assert ioc_white_list == [] assert files_while_list == [] assert final_white_list == [ 'boop', 'sade', 'sade.txt', '*****@*****.**', '' ] def test_get_generic_white_list(self): final_white_list, ioc_white_list, files_while_list = \ self.validator.get_generic_white_list(self.TEST_WHITELIST_FILE) assert ioc_white_list == ['*****@*****.**'] assert files_while_list == ['sade.txt'] assert final_white_list == ['*****@*****.**', 'aboop', 'asade'] def test_remove_false_positives(self): line = '[I AM MISTER MEESEEKS LOOK AT ME] sade' line = self.validator.remove_false_positives(line) assert line == ' sade' def test_is_secrets_disabled(self): line1 = 'disable-secrets-detection' skip_secrets = {'skip_once': False, 'skip_multi': False} skip_secrets = self.validator.is_secrets_disabled(line1, skip_secrets) assert skip_secrets['skip_once'] and not skip_secrets['skip_multi'] skip_secrets['skip_once'] = False line2 = 'disable-secrets-detection-start' skip_secrets = self.validator.is_secrets_disabled(line2, skip_secrets) assert not skip_secrets['skip_once'] and skip_secrets['skip_multi'] line3 = 'disable-secrets-detection-end' skip_secrets = self.validator.is_secrets_disabled(line3, skip_secrets) assert not skip_secrets['skip_once'] and not skip_secrets['skip_multi'] def test_ignore_base64(self): file_contents = self.TEST_BASE_64_STRING file_contents = self.validator.ignore_base64(file_contents) assert file_contents.lstrip() == 'sade'