def test_config_enabled_discovery_plugins():
"""
Test that the Config module identifies enabled discovery plugins for a given level.
Expected result: plugins listed in example config file are returned
"""
config_file = os.path.join(os.path.dirname(__file__), "rsc", "config.yaml")
config = Config(config_file)
plugins = config.get_enabled_discovery_plugins("example")
assert "cmake" in plugins
class Statick:
"""Code analysis front-end."""
def __init__(self, user_paths: List[str]) -> None:
"""Initialize Statick."""
self.resources = Resources(user_paths)
self.manager = PluginManager()
self.manager.setPluginPlaces(self.resources.get_plugin_paths())
self.manager.setCategoriesFilter({
"Discovery": DiscoveryPlugin,
"Tool": ToolPlugin,
"Reporting": ReportingPlugin,
})
self.manager.collectPlugins()
self.discovery_plugins: Dict[str, Any] = {}
for plugin_info in self.manager.getPluginsOfCategory("Discovery"):
self.discovery_plugins[plugin_info.plugin_object.get_name(
)] = plugin_info.plugin_object
self.tool_plugins: Dict[str, Any] = {}
for plugin_info in self.manager.getPluginsOfCategory("Tool"):
self.tool_plugins[plugin_info.plugin_object.get_name(
)] = plugin_info.plugin_object
self.reporting_plugins: Dict[str, Any] = {}
for plugin_info in self.manager.getPluginsOfCategory("Reporting"):
self.reporting_plugins[plugin_info.plugin_object.get_name(
)] = plugin_info.plugin_object
self.config: Optional[Config] = None
self.exceptions: Optional[Exceptions] = None
@staticmethod
def set_logging_level(args: argparse.Namespace) -> None:
"""Set the logging level to use for output.
Valid levels are: DEBUG, INFO, WARNING, ERROR, CRITICAL. Specifying the level is
case-insensitive (both upper-case and lower-case are allowed).
"""
valid_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
log_level = args.log_level.upper()
if log_level not in valid_levels:
log_level = "WARNING"
args.log_level = log_level
logging.basicConfig(level=log_level)
logging.root.setLevel(log_level)
logging.info("Log level set to %s", args.log_level.upper())
def get_config(self, args: argparse.Namespace) -> None:
"""Get Statick configuration."""
base_config_filename = "config.yaml"
user_config_filename = ""
if args.config is not None:
user_config_filename = args.config
try:
self.config = Config(
self.resources.get_file(base_config_filename),
self.resources.get_file(user_config_filename),
)
except OSError as ex:
logging.error(
"Failed to access configuration file %s or %s: %s",
base_config_filename,
user_config_filename,
ex,
)
except ValueError as ex:
logging.error(
"Configuration file %s or %s has errors: %s",
base_config_filename,
user_config_filename,
ex,
)
def get_exceptions(self, args: argparse.Namespace) -> None:
"""Get Statick exceptions."""
exceptions_filename = "exceptions.yaml"
if args.exceptions is not None:
exceptions_filename = args.exceptions
try:
self.exceptions = Exceptions(
self.resources.get_file(exceptions_filename))
except OSError as ex:
logging.error("Failed to access exceptions file %s: %s",
exceptions_filename, ex)
except ValueError as ex:
logging.error("Exceptions file %s has errors: %s",
exceptions_filename, ex)
def get_ignore_packages(self) -> List[str]:
"""Get packages to ignore during scan process."""
if self.exceptions is None:
return []
return self.exceptions.get_ignore_packages()
def gather_args(self, args: argparse.ArgumentParser) -> None:
"""Gather arguments."""
args.add_argument(
"--output-directory",
dest="output_directory",
type=str,
help="Directory to write output files to",
)
args.add_argument(
"--log",
dest="log_level",
type=str,
default="WARNING",
help="Verbosity level of output to show (DEBUG, INFO, WARNING, ERROR"
", CRITICAL)",
)
args.add_argument(
"--check",
dest="check",
action="store_true",
help="Return the status. Return code 0 means there were no issues. \
Return code 1 means there were issues.",
)
args.add_argument("--config",
dest="config",
type=str,
help="Name of config yaml file")
args.add_argument("--profile",
dest="profile",
type=str,
help="Name of profile yaml file")
args.add_argument(
"--exceptions",
dest="exceptions",
type=str,
help="Name of exceptions yaml file",
)
args.add_argument(
"--force-tool-list",
dest="force_tool_list",
type=str,
help="Force only the given list of tools to run",
)
args.add_argument(
"--version",
action="version",
version=f"%(prog)s {__version__}",
)
args.add_argument(
"--mapping-file-suffix",
dest="mapping_file_suffix",
type=str,
help="Suffix to use when searching for CERT mapping files",
)
# statick workspace arguments
args.add_argument(
"-ws",
dest="workspace",
action="store_true",
help="Treat the path argument as a workspace of multiple packages",
)
args.add_argument(
"--max-procs",
dest="max_procs",
type=int,
default=int(multiprocessing.cpu_count() / 2),
help=
"Maximum number of CPU cores to use, only used when running on a"
"workspace",
)
args.add_argument(
"--packages-file",
dest="packages_file",
type=str,
help=
"File listing packages to scan, only used when running on a workspace",
)
args.add_argument(
"--list-packages",
dest="list_packages",
action="store_true",
help=
"List packages and levels, only used when running on a workspace",
)
for _, plugin in list(self.discovery_plugins.items()):
plugin.gather_args(args)
for _, plugin in list(self.tool_plugins.items()):
plugin.gather_args(args)
for _, plugin in list(self.reporting_plugins.items()):
plugin.gather_args(args)
def get_level(self, path: str, args: argparse.Namespace) -> Optional[str]:
"""Get level to scan package at."""
path = os.path.abspath(path)
profile_filename = "profile.yaml"
if args.profile is not None:
profile_filename = args.profile
profile_resource = self.resources.get_file(profile_filename)
if profile_resource is None:
logging.error("Could not find profile file %s!", profile_filename)
return None
try:
profile = Profile(profile_resource)
except OSError as ex:
# This isn't quite redundant with the profile_resource check: it's possible
# that something else triggers an OSError, like permissions.
logging.error("Failed to access profile file %s: %s",
profile_filename, ex)
return None
except ValueError as ex:
logging.error("Profile file %s has errors: %s", profile_filename,
ex)
return None
package = Package(os.path.basename(path), path)
level = profile.get_package_level(package)
return level
# pylint: disable=too-many-locals, too-many-return-statements, too-many-branches
# pylint: disable=too-many-statements
def run(
self, path: str, args: argparse.Namespace
) -> Tuple[Optional[Dict[str, List[Issue]]], bool]:
"""Run scan tools against targets on path."""
success = True
path = os.path.abspath(path)
if not os.path.exists(path):
logging.error("No package found at %s!", path)
return None, False
package = Package(os.path.basename(path), path)
level: Optional[str] = self.get_level(path, args)
logging.info("level: %s", level)
if level is None:
logging.error("Level is not valid.")
return None, False
if not self.config or not self.config.has_level(level):
logging.error("Can't find specified level %s in config!", level)
return None, False
orig_path = os.getcwd()
if args.output_directory:
if not os.path.isdir(args.output_directory):
try:
os.mkdir(args.output_directory)
except OSError as ex:
logging.error(
"Unable to create output directory at %s: %s",
args.output_directory,
ex,
)
return None, False
output_dir = os.path.join(args.output_directory,
package.name + "-" + level)
if not os.path.isdir(output_dir):
try:
os.mkdir(output_dir)
except OSError as ex:
logging.error(
"Unable to create output directory at %s: %s",
output_dir, ex)
return None, False
logging.info("Writing output to: %s", output_dir)
os.chdir(output_dir)
logging.info("------")
logging.info("Scanning package %s (%s) at level %s", package.name,
package.path, level)
issues: Dict[str, List[Issue]] = {}
ignore_packages = self.get_ignore_packages()
if package.name in ignore_packages:
logging.info("Package %s is configured to be ignored by Statick.",
package.name)
return issues, True
plugin_context = PluginContext(args, self.resources, self.config)
logging.info("---Discovery---")
if not DiscoveryPlugin.file_command_exists():
logging.info(
"file command isn't available, discovery plugins will be less effective"
)
discovery_plugins = self.config.get_enabled_discovery_plugins(level)
if not discovery_plugins:
discovery_plugins = list(self.discovery_plugins.keys())
plugins_ran: List[Any] = []
for plugin_name in discovery_plugins:
if plugin_name not in self.discovery_plugins:
logging.error("Can't find specified discovery plugin %s!",
plugin_name)
return None, False
plugin = self.discovery_plugins[plugin_name]
dependencies = plugin.get_discovery_dependencies()
for dependency_name in dependencies:
dependency_plugin = self.discovery_plugins[dependency_name]
if dependency_plugin.get_name() in plugins_ran:
continue
dependency_plugin.set_plugin_context(plugin_context)
logging.info("Running %s discovery plugin...",
dependency_plugin.get_name())
dependency_plugin.scan(package, level, self.exceptions)
logging.info("%s discovery plugin done.",
dependency_plugin.get_name())
plugins_ran.append(dependency_plugin.get_name())
if plugin.get_name() not in plugins_ran:
plugin.set_plugin_context(plugin_context)
logging.info("Running %s discovery plugin...",
plugin.get_name())
plugin.scan(package, level, self.exceptions)
logging.info("%s discovery plugin done.", plugin.get_name())
plugins_ran.append(plugin.get_name())
logging.info("---Discovery---")
logging.info("---Tools---")
enabled_plugins = self.config.get_enabled_tool_plugins(level)
plugins_to_run = copy.copy(enabled_plugins)
plugins_ran = []
plugin_dependencies: List[str] = []
while plugins_to_run:
plugin_name = plugins_to_run[0]
if plugin_name not in self.tool_plugins:
logging.error("Can't find specified tool plugin %s!",
plugin_name)
return None, False
if args.force_tool_list is not None:
force_tool_list = args.force_tool_list.split(",")
if (plugin_name not in force_tool_list
and plugin_name not in plugin_dependencies):
logging.info("Skipping plugin not in force list %s!",
plugin_name)
plugins_to_run.remove(plugin_name)
continue
plugin = self.tool_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
dependencies = plugin.get_tool_dependencies()
dependencies_met = True
for dependency_name in dependencies:
if dependency_name not in plugins_ran:
if dependency_name not in enabled_plugins:
logging.error(
"Plugin %s depends on plugin %s which isn't enabled!",
plugin_name,
dependency_name,
)
return None, False
plugin_dependencies.append(dependency_name)
if dependency_name in plugins_to_run:
plugins_to_run.remove(dependency_name)
plugins_to_run.insert(0, dependency_name)
dependencies_met = False
if not dependencies_met:
continue
logging.info("Running %s tool plugin...", plugin.get_name())
tool_issues = plugin.scan(package, level)
if tool_issues is not None:
issues[plugin_name] = tool_issues
logging.info("%s tool plugin done.", plugin.get_name())
else:
logging.error("%s tool plugin failed", plugin.get_name())
success = False
plugins_to_run.remove(plugin_name)
plugins_ran.append(plugin_name)
logging.info("---Tools---")
if self.exceptions is not None:
issues = self.exceptions.filter_issues(package, issues)
os.chdir(orig_path)
logging.info("---Reporting---")
reporting_plugins = self.config.get_enabled_reporting_plugins(level)
if not reporting_plugins:
reporting_plugins = self.reporting_plugins.keys() # type: ignore
for plugin_name in reporting_plugins:
if plugin_name not in self.reporting_plugins:
logging.error("Can't find specified reporting plugin %s!",
plugin_name)
return None, False
plugin = self.reporting_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
logging.info("Running %s reporting plugin...", plugin.get_name())
plugin.report(package, issues, level)
logging.info("%s reporting plugin done.", plugin.get_name())
logging.info("---Reporting---")
logging.info("Done!")
return issues, success
# pylint: enable=too-many-locals, too-many-return-statements, too-many-branches
# pylint: enable=too-many-statements
def run_workspace(
self, parsed_args: argparse.Namespace
) -> Tuple[Optional[Dict[str, List[Issue]]], bool]: # pylint: disable=too-many-locals, too-many-branches, too-many-statements
"""Run statick on a workspace.
--max-procs can be set to the desired number of CPUs to use for processing a
workspace.
This defaults to half the available CPUs.
Setting this to -1 will cause statick.run_workspace to use all available CPUs.
"""
max_cpus = multiprocessing.cpu_count()
if parsed_args.max_procs > max_cpus or parsed_args.max_procs == -1:
parsed_args.max_procs = max_cpus
elif parsed_args.max_procs <= 0:
parsed_args.max_procs = 1
if parsed_args.output_directory:
out_dir = parsed_args.output_directory
if not os.path.isdir(out_dir):
try:
os.mkdir(out_dir)
except OSError as ex:
logging.error(
"Unable to create output directory at %s: %s", out_dir,
ex)
return None, False
ignore_packages = self.get_ignore_packages()
ignore_files = ["AMENT_IGNORE", "CATKIN_IGNORE", "COLCON_IGNORE"]
package_indicators = ["package.xml", "setup.py", "pyproject.toml"]
packages = []
for root, dirs, files in os.walk(parsed_args.path):
if any(item in files for item in ignore_files):
dirs.clear()
continue
for sub_dir in dirs:
full_dir = os.path.join(root, sub_dir)
files = os.listdir(full_dir)
if any(item in package_indicators
for item in files) and not any(
item in files for item in ignore_files):
if ignore_packages and sub_dir in ignore_packages:
continue
packages.append(Package(sub_dir, full_dir))
if parsed_args.packages_file is not None:
packages_file_list = []
try:
packages_file = os.path.abspath(parsed_args.packages_file)
with open(packages_file, "r", encoding="utf8") as fname:
packages_file_list = [
package.strip() for package in fname.readlines()
if package.strip() and package[0] != "#"
]
except OSError:
logging.error("Packages file not found")
return None, False
packages = [
package for package in packages
if package.name in packages_file_list
]
if parsed_args.list_packages:
for package in packages:
logging.info("%s: %s", package.name,
self.get_level(package.path, parsed_args))
return None, True
count = 0
total_issues = []
num_packages = len(packages)
mp_args = []
if multiprocessing.get_start_method() == "fork":
logging.info("-- Scanning %d packages --", num_packages)
for package in packages:
count += 1
mp_args.append((parsed_args, count, package, num_packages))
with multiprocessing.Pool(parsed_args.max_procs) as pool:
total_issues = pool.starmap(self.scan_package, mp_args)
else:
logging.warning(
"Statick's plugin manager does not currently support multiprocessing"
" without UNIX's fork function. Falling back to a single process."
)
logging.info("-- Scanning %d packages --", num_packages)
for package in packages:
count += 1
pkg_issues = self.scan_package(parsed_args, count, package,
num_packages)
total_issues.append(pkg_issues)
logging.info("-- All packages run --")
logging.info("-- overall report --")
success = True
issues: Dict[str, List[Issue]] = {}
for issue in total_issues:
if issue is not None:
for key, value in list(issue.items()):
if key in issues:
issues[key] += value
if value:
success = False
else:
issues[key] = value
if value:
success = False
enabled_reporting_plugins: List[str] = []
available_reporting_plugins = {}
for plugin_info in self.manager.getPluginsOfCategory("Reporting"):
available_reporting_plugins[plugin_info.plugin_object.get_name(
)] = plugin_info.plugin_object
# Make a fake 'all' package for reporting
dummy_all_package = Package("all_packages", parsed_args.path)
level = self.get_level(dummy_all_package.path, parsed_args)
if level is not None and self.config is not None:
if not self.config or not self.config.has_level(level):
logging.error("Can't find specified level %s in config!",
level)
enabled_reporting_plugins = list(available_reporting_plugins)
else:
enabled_reporting_plugins = self.config.get_enabled_reporting_plugins(
level)
if not enabled_reporting_plugins:
enabled_reporting_plugins = list(available_reporting_plugins)
plugin_context = PluginContext(parsed_args, self.resources,
self.config) # type: ignore
plugin_context.args.output_directory = parsed_args.output_directory
for plugin_name in enabled_reporting_plugins:
if plugin_name not in available_reporting_plugins:
logging.error("Can't find specified reporting plugin %s!",
plugin_name)
continue
plugin = self.reporting_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
logging.info("Running %s reporting plugin...", plugin.get_name())
plugin.report(dummy_all_package, issues, level)
logging.info("%s reporting plugin done.", plugin.get_name())
return issues, success
def scan_package(
self,
parsed_args: argparse.Namespace,
count: int,
package: Package,
num_packages: int,
) -> Optional[Dict[str, List[Issue]]]:
"""Scan each package in a separate process while buffering output."""
logger = logging.getLogger()
old_handler = None
if logger.handlers[0]:
old_handler = logger.handlers[0]
handler = MemoryHandler(10000,
flushLevel=logging.ERROR,
target=old_handler)
logger.removeHandler(old_handler)
logger.addHandler(handler)
logging.info("-- Scanning package %s (%d of %d) --", package.name,
count, num_packages)
sio = io.StringIO()
old_stdout = sys.stdout
old_stderr = sys.stderr
sys.stdout = sio
sys.stderr = sio
issues, dummy = self.run(package.path, parsed_args)
sys.stdout = old_stdout
sys.stderr = old_stderr
logging.info(sio.getvalue())
if issues is not None:
logging.info(
"-- Done scanning package %s (%d of %d) --",
package.name,
count,
num_packages,
)
else:
logging.error("Failed to run statick on package %s!", package.name)
if old_handler is not None:
handler.flush()
logger.removeHandler(handler)
logger.addHandler(old_handler)
return issues
@staticmethod
def print_no_issues() -> None:
"""Print that no information about issues was found."""
logging.error("Something went wrong, no information about issues."
" Statick exiting with errors.")
@staticmethod
def print_exit_status(status: bool) -> None:
"""Print Statick exit status."""
if status:
logging.info("Statick exiting with success.")
else:
logging.error("Statick exiting with errors.")
class Statick:
"""Code analysis front-end."""
def __init__(self, user_paths: List[str]) -> None:
"""Initialize Statick."""
self.resources = Resources(user_paths)
self.manager = PluginManager()
self.manager.setPluginPlaces(self.resources.get_plugin_paths())
self.manager.setCategoriesFilter(
{
"Discovery": DiscoveryPlugin,
"Tool": ToolPlugin,
"Reporting": ReportingPlugin,
}
)
self.manager.collectPlugins()
self.discovery_plugins = {} # type: Dict
for plugin_info in self.manager.getPluginsOfCategory("Discovery"):
self.discovery_plugins[
plugin_info.plugin_object.get_name()
] = plugin_info.plugin_object
self.tool_plugins = {} # type: Dict
for plugin_info in self.manager.getPluginsOfCategory("Tool"):
self.tool_plugins[
plugin_info.plugin_object.get_name()
] = plugin_info.plugin_object
self.reporting_plugins = {} # type: Dict
for plugin_info in self.manager.getPluginsOfCategory("Reporting"):
self.reporting_plugins[
plugin_info.plugin_object.get_name()
] = plugin_info.plugin_object
self.config = None # type: Optional[Config]
self.exceptions = None # type: Optional[Exceptions]
def get_config(self, args: argparse.Namespace) -> None:
"""Get Statick configuration."""
config_filename = "config.yaml"
if args.config is not None:
config_filename = args.config
self.config = Config(self.resources.get_file(config_filename))
def get_exceptions(self, args: argparse.Namespace) -> None:
"""Get Statick exceptions."""
exceptions_filename = "exceptions.yaml"
if args.exceptions is not None:
exceptions_filename = args.exceptions
self.exceptions = Exceptions(self.resources.get_file(exceptions_filename))
def get_ignore_packages(self) -> List[str]:
"""Get packages to ignore during scan process."""
assert self.exceptions
return self.exceptions.get_ignore_packages()
def gather_args(self, args: argparse.Namespace) -> None:
"""Gather arguments."""
args.add_argument(
"--output-directory",
dest="output_directory",
type=str,
help="Directory to write output files to",
)
args.add_argument(
"--show-tool-output",
dest="show_tool_output",
action="store_true",
help="Show tool output",
)
args.add_argument(
"--check",
dest="check",
action="store_true",
help="Return the status. Return code 0 means there were no issues. \
Return code 1 means there were issues.",
)
args.add_argument(
"--config", dest="config", type=str, help="Name of config yaml file"
)
args.add_argument(
"--profile", dest="profile", type=str, help="Name of profile yaml file"
)
args.add_argument(
"--exceptions",
dest="exceptions",
type=str,
help="Name of exceptions yaml file",
)
args.add_argument(
"--force-tool-list",
dest="force_tool_list",
type=str,
help="Force only the given list of tools to run",
)
args.add_argument(
"--version",
action="version",
version="%(prog)s {version}".format(version=__version__),
)
args.add_argument(
"--mapping-file-suffix",
dest="mapping_file_suffix",
type=str,
help="Suffix to use when searching for CERT mapping files",
)
for _, plugin in list(self.discovery_plugins.items()):
plugin.gather_args(args)
for _, plugin in list(self.tool_plugins.items()):
plugin.gather_args(args)
for _, plugin in list(self.reporting_plugins.items()):
plugin.gather_args(args)
def get_level(self, path: str, args: argparse.Namespace) -> Optional[str]:
"""Get level to scan package at."""
path = os.path.abspath(path)
profile_filename = "profile.yaml"
if args.profile is not None:
profile_filename = args.profile
profile_resource = self.resources.get_file(profile_filename)
if profile_resource is None:
print("Could not find profile file {}!".format(profile_filename))
return None
try:
profile = Profile(profile_resource)
except OSError as ex:
# This isn't quite redundant with the profile_resource check: it's possible
# that something else triggers an OSError, like permissions.
print("Failed to access profile file {}: {}".format(profile_filename, ex))
return None
except ValueError as ex:
print("Profile file {} has errors: {}".format(profile_filename, ex))
return None
package = Package(os.path.basename(path), path)
level = profile.get_package_level(package)
return level
def run( # pylint: disable=too-many-locals, too-many-return-statements, too-many-branches, too-many-statements
self, path: str, args: argparse.Namespace
) -> Tuple[Optional[Dict], bool]:
"""Run scan tools against targets on path."""
success = True
path = os.path.abspath(path)
if not os.path.exists(path):
print("No package found at {}!".format(path))
return None, False
package = Package(os.path.basename(path), path)
level = self.get_level(path, args) # type: Optional[str]
assert level
if not self.config or not self.config.has_level(level):
print("Can't find specified level {} in config!".format(level))
return None, False
orig_path = os.getcwd()
if args.output_directory:
if not os.path.isdir(args.output_directory):
print("Output directory not found at {}!".format(args.output_directory))
return None, False
output_dir = os.path.join(args.output_directory, package.name + "-" + level)
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
if not os.path.isdir(output_dir):
print("Unable to create output directory at {}!".format(output_dir))
return None, False
print("Writing output to: {}".format(output_dir))
os.chdir(output_dir)
print("------")
print(
"Scanning package {} ({}) at level {}".format(
package.name, package.path, level
)
)
issues = {} # type: Dict[str, List[Issue]]
ignore_packages = self.get_ignore_packages()
if package.name in ignore_packages:
print(
"Package {} is configured to be ignored by Statick.".format(
package.name
)
)
return issues, True
plugin_context = PluginContext(args, self.resources, self.config)
print("---Discovery---")
if not DiscoveryPlugin.file_command_exists():
print(
"file command isn't available, discovery plugins will be less effective"
)
discovery_plugins = self.config.get_enabled_discovery_plugins(level)
if not discovery_plugins:
discovery_plugins = list(self.discovery_plugins.keys())
for plugin_name in discovery_plugins:
if plugin_name not in self.discovery_plugins:
print("Can't find specified discovery plugin {}!".format(plugin_name))
return None, False
plugin = self.discovery_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
print("Running {} discovery plugin...".format(plugin.get_name()))
plugin.scan(package, level, self.exceptions)
print("{} discovery plugin done.".format(plugin.get_name()))
print("---Discovery---")
print("---Tools---")
enabled_plugins = self.config.get_enabled_tool_plugins(level)
plugins_to_run = copy.copy(enabled_plugins)
plugins_ran = [] # type: List[Any]
plugin_dependencies = [] # type: List[str]
while plugins_to_run:
plugin_name = plugins_to_run[0]
if plugin_name not in self.tool_plugins:
print("Can't find specified tool plugin {}!".format(plugin_name))
return None, False
if args.force_tool_list is not None:
force_tool_list = args.force_tool_list.split(",")
if (
plugin_name not in force_tool_list
and plugin_name not in plugin_dependencies
):
print("Skipping plugin not in force list {}!".format(plugin_name))
plugins_to_run.remove(plugin_name)
continue
plugin = self.tool_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
dependencies = plugin.get_tool_dependencies()
dependencies_met = True
for dependency_name in dependencies:
if dependency_name not in plugins_ran:
if dependency_name not in enabled_plugins:
print(
"Plugin {} depends on plugin {} which isn't "
"enabled!".format(plugin_name, dependency_name)
)
return None, False
plugin_dependencies.append(dependency_name)
plugins_to_run.remove(dependency_name)
plugins_to_run.insert(0, dependency_name)
dependencies_met = False
if not dependencies_met:
continue
print("Running {} tool plugin...".format(plugin.get_name()))
tool_issues = plugin.scan(package, level)
if tool_issues is not None:
issues[plugin_name] = tool_issues
print("{} tool plugin done.".format(plugin.get_name()))
else:
print("{} tool plugin failed".format(plugin.get_name()))
success = False
plugins_to_run.remove(plugin_name)
plugins_ran.append(plugin_name)
print("---Tools---")
if self.exceptions is not None:
issues = self.exceptions.filter_issues(package, issues)
os.chdir(orig_path)
print("---Reporting---")
reporting_plugins = self.config.get_enabled_reporting_plugins(level)
if not reporting_plugins:
reporting_plugins = self.reporting_plugins.keys() # type: ignore
for plugin_name in reporting_plugins:
if plugin_name not in self.reporting_plugins.keys():
print("Can't find specified reporting plugin {}!".format(plugin_name))
return None, False
plugin = self.reporting_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
print("Running {} reporting plugin...".format(plugin.get_name()))
plugin.report(package, issues, level)
print("{} reporting plugin done.".format(plugin.get_name()))
print("---Reporting---")
print("Done!")
return issues, success
class Statick(object):
"""Code analysis front-end."""
def __init__(self, user_paths):
"""Initialize Statick."""
self.resources = Resources(user_paths)
self.manager = PluginManager()
self.manager.setPluginPlaces(self.resources.get_plugin_paths())
self.manager.setCategoriesFilter({
"Discovery": DiscoveryPlugin,
"Tool": ToolPlugin,
"Reporting": ReportingPlugin
})
self.manager.collectPlugins()
self.discovery_plugins = {}
for plugin_info in self.manager.getPluginsOfCategory("Discovery"):
self.discovery_plugins[plugin_info.plugin_object.get_name()] = \
plugin_info.plugin_object
self.tool_plugins = {}
for plugin_info in self.manager.getPluginsOfCategory("Tool"):
self.tool_plugins[plugin_info.plugin_object.get_name()] = \
plugin_info.plugin_object
self.reporting_plugins = {}
for plugin_info in self.manager.getPluginsOfCategory("Reporting"):
self.reporting_plugins[plugin_info.plugin_object.get_name()] = \
plugin_info.plugin_object
self.config = Config(self.resources.get_file("config.yaml"))
self.exceptions = Exceptions(
self.resources.get_file("exceptions.yaml"))
def get_ignore_packages(self):
"""Get packages to ignore during scan process."""
return self.exceptions.get_ignore_packages()
def gather_args(self, args):
"""Gather arguments."""
args.add_argument("output_directory", help="Output directory")
args.add_argument("--show-tool-output",
dest="show_tool_output",
action="store_true",
help="Show tool output")
args.add_argument("--profile",
dest="profile",
type=str,
help="Name of profile yaml file")
args.add_argument("--force-tool-list",
dest="force_tool_list",
type=str,
help="Force only the given list of tools to run")
args.add_argument(
'--version',
action='version',
version='%(prog)s {version}'.format(version=__version__))
args.add_argument(
'--mapping-file-suffix',
dest="mapping_file_suffix",
type=str,
help="Suffix to use when searching for CERT mapping files")
for _, plugin in list(self.discovery_plugins.items()):
plugin.gather_args(args)
for _, plugin in list(self.tool_plugins.items()):
plugin.gather_args(args)
for _, plugin in iteritems(self.reporting_plugins):
plugin.gather_args(args)
def get_level(self, path, args):
"""Get level to scan package at."""
path = os.path.abspath(path)
profile_filename = "profile.yaml"
if args.profile is not None:
profile_filename = args.profile
try:
profile = Profile(self.resources.get_file(profile_filename))
except TypeError:
print("Could not find profile file {}!".format(profile_filename))
return None
package = Package(os.path.basename(path), path)
level = profile.get_package_level(package)
return level
def run(self, path, args): # pylint: disable=too-many-locals, too-many-return-statements, too-many-branches, too-many-statements
"""Run scan tools against targets on path."""
success = True
path = os.path.abspath(path)
if not os.path.exists(path):
print("No package found at {}!".format(path))
return None, False
package = Package(os.path.basename(path), path)
level = self.get_level(path, args)
if not self.config.has_level(level):
print("Can't find specified level {} in config!".format(level))
return None, False
orig_path = os.getcwd()
if not os.path.isdir(args.output_directory):
print("Output directory not found at {}!".format(
args.output_directory))
return None, False
output_dir = os.path.join(args.output_directory,
package.name + "-" + level)
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
if not os.path.isdir(output_dir):
print(
"Unable to create output directory at {}!".format(output_dir))
return None, False
print("Writing output to: {}".format(output_dir))
os.chdir(output_dir)
print("------")
print("Scanning package {} ({}) at level {}".format(
package.name, package.path, level))
issues = {}
ignore_packages = self.get_ignore_packages()
if package.name in ignore_packages:
print("Package {} is configured to be ignored by Statick.".format(
package.name))
return issues, True
plugin_context = PluginContext(args, self.resources, self.config)
print("---Discovery---")
if not DiscoveryPlugin.file_command_exists():
print(
"file command isn't available, discovery plugins will be less effective"
)
discovery_plugins = self.config.get_enabled_discovery_plugins(level)
if not discovery_plugins:
discovery_plugins = list(self.discovery_plugins.keys())
for plugin_name in discovery_plugins:
if plugin_name not in self.discovery_plugins:
print("Can't find specified discovery plugin {}!".format(
plugin_name))
return None, False
plugin = self.discovery_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
print("Running {} discovery plugin...".format(plugin.get_name()))
plugin.scan(package, level, self.exceptions)
print("{} discovery plugin done.".format(plugin.get_name()))
print("---Discovery---")
print("---Tools---")
enabled_plugins = self.config.get_enabled_tool_plugins(level)
plugins_to_run = copy.copy(enabled_plugins)
plugins_ran = []
plugin_dependencies = []
while plugins_to_run:
plugin_name = plugins_to_run[0]
if plugin_name not in self.tool_plugins:
if plugin_name == "pep257":
plugin_name = "pydocstyle"
plugins_to_run.remove("pep257")
plugins_to_run.insert(0, plugin_name)
print(
"DEPRECATION WARNING: The pep257 tool has been renamed "
"as the pydocstyle tool. Please update your configuration."
)
elif plugin_name == "pep8":
plugin_name = "pycodestyle"
plugins_to_run.remove("pep8")
plugins_to_run.insert(0, plugin_name)
print(
"DEPRECATION WARNING: The pep8 tool has been renamed "
"as the pycodestyle tool. Please update your configuration."
)
else:
print("Can't find specified tool plugin {}!".format(
plugin_name))
return None, False
if args.force_tool_list is not None:
force_tool_list = args.force_tool_list.split(",")
if plugin_name not in force_tool_list and plugin_name not in plugin_dependencies:
print("Skipping plugin not in force list {}!".format(
plugin_name))
plugins_to_run.remove(plugin_name)
continue
plugin = self.tool_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
dependencies = plugin.get_tool_dependencies()
dependencies_met = True
for dependency_name in dependencies:
if dependency_name not in plugins_ran:
if dependency_name not in enabled_plugins:
print("Plugin {} depends on plugin {} which isn't "
"enabled!".format(plugin_name, dependency_name))
return None, False
plugin_dependencies.append(dependency_name)
plugins_to_run.remove(dependency_name)
plugins_to_run.insert(0, dependency_name)
dependencies_met = False
if not dependencies_met:
continue
print("Running {} tool plugin...".format(plugin.get_name()))
tool_issues = plugin.scan(package, level)
if tool_issues is not None:
issues[plugin_name] = tool_issues
print("{} tool plugin done.".format(plugin.get_name()))
else:
print("{} tool plugin failed".format(plugin.get_name()))
success = False
plugins_to_run.remove(plugin_name)
plugins_ran.append(plugin_name)
print("---Tools---")
issues = self.exceptions.filter_issues(package, issues)
os.chdir(orig_path)
print("---Reporting---")
reporting_plugins = self.config.get_enabled_reporting_plugins(level)
if not reporting_plugins:
reporting_plugins = self.reporting_plugins.keys()
for plugin_name in reporting_plugins:
if plugin_name not in self.reporting_plugins.keys():
print("Can't find specified reporting plugin {}!".format(
plugin_name))
return None, False
plugin = self.reporting_plugins[plugin_name]
plugin.set_plugin_context(plugin_context)
print("Running {} reporting plugin...".format(plugin.get_name()))
plugin.report(package, issues, level)
print("{} reporting plugin done.".format(plugin.get_name()))
print("---Reporting---")
print("Done!")
return issues, success
