def import_plugins():
"""
:rtype: Dict[str, Type[TypeVar('Plugin', bound=BasePlugin)]]
"""
modules = []
for root, _, files in os.walk(
os.path.join(get_root_directory(), 'detect_secrets/plugins'), ):
for filename in files:
if not filename.startswith('_'):
modules.append(os.path.splitext(filename)[0])
# Only want to import top level files
break
plugins = {}
for module_name in modules:
module = import_module('detect_secrets.plugins.{}'.format(module_name))
for name in filter(lambda x: not x.startswith('_'), dir(module)):
plugin = getattr(module, name)
try:
if not issubclass(plugin, BasePlugin):
continue
except TypeError:
# Occurs when plugin is not a class type.
continue
# Use this as a heuristic to determine abstract classes
if isinstance(plugin.secret_type, abstractproperty):
continue
plugins[name] = plugin
return plugins
def get_arguments():
plugins = [info.classname for info in PluginOptions.all_plugins]
parser = argparse.ArgumentParser(description='Run some benchmarks.')
parser.add_argument(
'filenames',
nargs=argparse.REMAINDER,
help='Filenames to check or detect-secrets compatible arguments.',
)
parser.add_argument(
'--pretty',
action='store_true',
help='Human readable output.',
)
parser.add_argument(
'--plugin',
default=
None, # needs to be None, otherwise append won't work as expected
choices=plugins,
action='append',
help=('Specifies a plugin to test. May provide multiple values. '
'Defaults to all.'),
)
parser.add_argument(
'--harakiri',
default=5,
type=assert_positive(float),
help=('Specifies an upper bound for the number of seconds to wait '
'per execution.'),
)
parser.add_argument(
'-n',
'--num-iterations',
default=1,
type=assert_positive(int),
help=('Specifies the number of times to run the test. '
'Results will be averaged over this value.'),
)
parser.add_argument(
'--baseline',
type=assert_valid_file,
help=('If provided, will compare performance with provided baseline. '
'Assumes pretty output (otherwise, you can do the comparison '
'yourself).'),
)
args = parser.parse_args()
if not args.filenames:
if args.baseline:
args.filenames = args.baseline['filenames']
else:
args.filenames = [get_root_directory()]
if not args.plugin:
args.plugin = plugins
return args
def generate_content(separator, length):
"""
:type secret: str
:type separator: str
:type length: int
"""
valid_secrets = {
'AWSKeyDetector': 'AKIATESTTESTTESTTEST', # pragma: whitelist secret
'ArtifactoryDetector': ':AKCtestTESTte', # pragma: whitelist secret
'Base64HighEntropyString':
'Y29uZ3JhdHVsYXRpb25zISB0aGlzIGlzIGEgaGlkZGVuIG1lc3NhZ2U=', # pragma: whitelist secret
'BasicAuthDetector':
'http://*****:*****@example.com', # pragma: whitelist secret
'HexHighEntropyString': '123456abcd', # pragma: whitelist secret
'KeywordDetector': 'api_key = foobar', # pragma: whitelist secret
'PrivateKeyDetector': 'BEGIN PRIVATE KEY', # pragma: whitelist secret
'SlackDetector': 'xoxb-1-test', # pragma: whitelist secret
'StripeDetector':
'rk_live_TESTtestTESTtestTESTtest', # pragma: whitelist secret
}
with open(
os.path.join(
get_root_directory(),
'test_data/performance/best-songs.txt',
), ) as f:
source_material = f.read().splitlines()
indexes = {}
for key in valid_secrets:
index = random.randint(0, length - 1)
indexes[index] = key
content = []
for line_number in range(length):
if line_number in indexes:
content.append(valid_secrets[indexes[line_number]])
else:
random_line = random.randint(0, len(source_material) - 1)
content.append(source_material[random_line])
return separator.join(content)
def scan_content(content, timeout, baseline=None):
"""
:type content: str
:type timeout: float
:type baseline: dict|None
"""
args = [
'python',
os.path.join(
get_root_directory(),
'scripts/benchmark.py',
),
'--harakiri',
str(timeout),
]
with tempfile.NamedTemporaryFile('w') as f:
f.write(content)
print('Running checks...', file=sys.stderr)
if not baseline:
args.append(f.name)
return subprocess.check_output(
args,
stderr=subprocess.DEVNULL,
).decode('utf-8')
with tempfile.NamedTemporaryFile('w') as b:
b.write(
json.dumps({
'filenames': [f.name],
'timings': baseline['timings'],
}), )
b.seek(0)
args.append('--baseline')
args.append(b.name)
return subprocess.check_output(
args,
stderr=subprocess.DEVNULL,
).decode('utf-8')
def import_plugins(plugin_filenames=None):
"""
:type plugin_filenames: tuple
:param plugin_filenames: the plugin filenames.
:rtype: Dict[str, Type[TypeVar('Plugin', bound=BasePlugin)]]
"""
modules = []
for root, _, files in os.walk(
os.path.join(get_root_directory(), 'detect_secrets/plugins'), ):
for filename in files:
if not filename.startswith('_'):
modules.append(os.path.splitext(filename)[0])
# Only want to import top level files
break
plugins = {}
for module_name in modules:
# If plugin_filenames is None, all of the plugins will get imported.
# Normal runs of this will have plugin_filenames set.
# plugin_filenames will be None if we are testing a method and don't pass it in.
if plugin_filenames is None or module_name in plugin_filenames:
module = import_module(
'detect_secrets.plugins.{}'.format(module_name))
for name in filter(lambda x: not x.startswith('_'), dir(module)):
plugin = getattr(module, name)
try:
if not issubclass(plugin, BasePlugin):
continue
except TypeError:
# Occurs when plugin is not a class type.
continue
# Use this as a heuristic to determine abstract classes
if isinstance(plugin.secret_type, abstractproperty):
continue
plugins[name] = plugin
return plugins
def import_plugins(custom_plugin_paths):
"""
:type custom_plugin_paths: tuple(str,)
:param custom_plugin_paths: possibly empty tuple of paths that have custom plugins.
:rtype: Dict[str, Type[TypeVar('Plugin', bound=BasePlugin)]]
"""
path_and_module_name_pairs = []
# Handle files
for path_to_import in custom_plugin_paths:
if os.path.isfile(path_to_import):
# [:-3] for removing '.py'
module_name = path_to_import[:-3].replace('/', '.')
path_and_module_name_pairs.append((
path_to_import,
module_name,
), )
# Handle directories
regular_plugins_dir = os.path.join(
get_root_directory(),
'detect_secrets/plugins',
)
plugin_dirs = (
[regular_plugins_dir] +
list(filter(
lambda path: (os.path.isdir(path)),
custom_plugin_paths,
), ))
for plugin_dir in plugin_dirs:
for filename in os.listdir(plugin_dir, ):
if (filename.startswith('_') or not filename.endswith('.py')):
continue
path_to_import = os.path.join(
plugin_dir,
filename,
)
# [:-3] for removing '.py'
if plugin_dir == regular_plugins_dir:
module_name = 'detect_secrets.plugins.{}'.format(filename[:-3])
else:
module_name = path_to_import[:-3].replace('/', '.')
path_and_module_name_pairs.append((
path_to_import,
module_name,
), )
# Do the importing
plugins = {}
for path_to_import, module_name in path_and_module_name_pairs:
module = _dynamically_import_module(
path_to_import,
module_name,
)
for attr_name in filter(
lambda attr_name: not attr_name.startswith('_'),
dir(module),
):
attr = getattr(module, attr_name)
if _is_valid_concrete_plugin_class(attr):
plugins[attr_name] = attr
return plugins
def import_plugins(plugin_filenames=None):
"""
:type plugin_filenames: tuple
:param plugin_filenames: the plugin filenames.
:rtype: Dict[str, Type[TypeVar('Plugin', bound=BasePlugin)]]
"""
modules = []
for root, _, files in os.walk(
os.path.join(get_root_directory(), 'detect_secrets/plugins'), ):
for filename in files:
if not filename.startswith('_'):
modules.append(os.path.splitext(filename)[0])
# Only want to import top level files
break
plugins = {}
for module_name in modules:
# If plugin_filenames is None, all of the plugins will get imported.
# Normal runs of this will have plugin_filenames set.
# plugin_filenames will be None if we are testing a method and don't pass it in.
if plugin_filenames is None or module_name in plugin_filenames:
try:
module = import_module(
'detect_secrets.plugins.{}'.format(module_name))
except ModuleNotFoundError as err: # pragma: no cover
if hasattr(err, 'msg'):
message = err.msg
if "No module named 'ibm_db'" not in message:
raise err
else:
yellow = '\033[93m'
end_yellow = '\033[0m'
print(
yellow,
'Warning: ibm_db is not installed, the DB2 plugin will not run.\n',
'To enable the optional DB2 plugin, install ibm_db with:\n',
'pip install ibm_db\n',
'and run detect secrets with',
'detect-secrets scan --update .secrets.baseline --db2-scan',
end_yellow,
file=sys.stderr,
flush=True,
)
continue
for name in filter(lambda x: not x.startswith('_'), dir(module)):
plugin = getattr(module, name)
try:
if not issubclass(plugin, BasePlugin):
continue
except TypeError:
# Occurs when plugin is not a class type.
continue
# Use this as a heuristic to determine abstract classes
if isinstance(plugin.secret_type, abstractproperty):
continue
plugins[name] = plugin
return plugins