def exec_tool(args, cwd=None, stdout=subprocess.PIPE):
"""
Convenience method to invoke cli tools
Args:
args cli command and args
"""
try:
LOG.debug('⚡︎ Executing "{}"'.format(" ".join(args)))
if os.environ.get("FETCH_LICENSE"):
LOG.debug(
"License information would be fetched from the registry. This would take several minutes ..."
)
subprocess.run(
args,
stdout=stdout,
stderr=subprocess.STDOUT,
cwd=cwd,
env=os.environ.copy(),
check=False,
shell=False,
encoding="utf-8",
)
except Exception as e:
LOG.exception(e)
def scan(db, project_type, pkg_list, suggest_mode):
"""
Method to search packages in our vulnerability database
:param db: Reference to db
:param project_type: Project Type
:param pkg_list: List of packages
:param suggest_mode: True if package fix version should be normalized across findings
"""
if not pkg_list:
LOG.debug("Empty package search attempted!")
else:
LOG.info("Scanning {} oss dependencies for issues".format(
len(pkg_list)))
results, pkg_aliases = utils.search_pkgs(db, project_type, pkg_list)
# pkg_aliases is a dict that can be used to find the original vendor and package name
# This way we consistently use the same names used by the caller irrespective of how
# the result was obtained
sug_version_dict = {}
if suggest_mode:
# From the results identify optimal max version
sug_version_dict = suggest_version(results, pkg_aliases)
if sug_version_dict:
LOG.debug(
"Adjusting fix version based on the initial suggestion {}".
format(sug_version_dict))
# Recheck packages
sug_pkg_list = []
for k, v in sug_version_dict.items():
if not v:
continue
vendor = ""
name = None
version = v
tmpA = k.split(":")
if len(tmpA) == 2:
vendor = tmpA[0]
name = tmpA[1]
else:
name = tmpA[0]
# De-alias the vendor and package name
full_pkg = "{}:{}".format(vendor, name)
full_pkg = pkg_aliases.get(full_pkg, full_pkg)
vendor, name = full_pkg.split(":")
sug_pkg_list.append({
"vendor": vendor,
"name": name,
"version": version
})
LOG.debug(
"Re-checking our suggestion to ensure there are no further vulnerabilities"
)
override_results, _ = utils.search_pkgs(db, project_type,
sug_pkg_list)
if override_results:
new_sug_dict = suggest_version(override_results)
LOG.debug("Received override results: {}".format(new_sug_dict))
for nk, nv in new_sug_dict.items():
sug_version_dict[nk] = nv
return results, pkg_aliases, sug_version_dict
def exec_tool(args, cwd=None, stdout=subprocess.PIPE):
"""
Convenience method to invoke cli tools
Args:
args cli command and args
"""
try:
LOG.debug('⚡︎ Executing "{}"'.format(" ".join(args)))
subprocess.run(
args,
stdout=stdout,
stderr=subprocess.STDOUT,
cwd=cwd,
env=os.environ,
check=False,
shell=False,
encoding="utf-8",
)
except Exception as e:
LOG.exception(e)
def get_pkg_list(xmlfile):
"""Method to parse the bom xml file and convert into packages list
:param xmlfile: BOM xml file to parse
:return list of package dict
"""
if xmlfile.endswith(".json"):
return get_pkg_list_json(xmlfile)
pkgs = []
try:
et = parse(xmlfile)
root = et.getroot()
for child in root:
if child.tag.endswith("components"):
for ele in child.iter():
if ele.tag.endswith("component"):
licenses = get_licenses(ele)
pkgs.append(get_package(ele, licenses))
except xml.etree.ElementTree.ParseError as pe:
LOG.debug("Unable to parse {} {}".format(xmlfile, pe))
LOG.warning(
"Unable to produce Software Bill-of-Materials for this project. Execute the scan after installing the dependencies!"
)
return pkgs
def main():
args = build_args()
if not args.no_banner:
print(at_logo)
src_dir = args.src_dir_image
if not src_dir:
src_dir = os.getcwd()
reports_base_dir = src_dir
# Detect the project types and perform the right type of scan
if args.project_type:
project_types_list = args.project_type.split(",")
elif args.bom:
project_types_list = ["bom"]
else:
project_types_list = utils.detect_project_type(src_dir)
if ("docker" in project_types_list or "podman" in project_types_list
or "container" in project_types_list
or "binary" in project_types_list):
reports_base_dir = os.getcwd()
db = dbLib.get()
run_cacher = args.cache
areport_file = (args.report_file if args.report_file else os.path.join(
reports_base_dir, "reports", "depscan.json"))
reports_dir = os.path.dirname(areport_file)
# Create reports directory
if not os.path.exists(reports_dir):
os.makedirs(reports_dir)
if len(project_types_list) > 1:
LOG.debug(
"Multiple project types found: {}".format(project_types_list))
# Enable license scanning
if "license" in project_types_list:
os.environ["FETCH_LICENSE"] = "true"
project_types_list.remove("license")
console.print(
Panel(
"License audit is enabled for this scan. This would increase the time by up to 10 minutes.",
title="License Audit",
expand=False,
))
for project_type in project_types_list:
sug_version_dict = {}
pkg_aliases = {}
results = []
report_file = areport_file.replace(".json",
"-{}.json".format(project_type))
risk_report_file = areport_file.replace(
".json", "-risk.{}.json".format(project_type))
LOG.info("=" * 80)
creation_status = False
if args.bom and os.path.exists(args.bom):
bom_file = args.bom
creation_status = True
else:
bom_file = os.path.join(reports_dir,
"bom-" + project_type + ".json")
creation_status = create_bom(project_type, bom_file, src_dir)
if not creation_status:
LOG.debug(
"Bom file {} was not created successfully".format(bom_file))
continue
LOG.debug("Scanning using the bom file {}".format(bom_file))
pkg_list = get_pkg_list(bom_file)
if not pkg_list:
LOG.debug("No packages found in the project!")
continue
scoped_pkgs = {}
if project_type in ["python"]:
all_imports = utils.get_all_imports(src_dir)
LOG.debug(f"Identified {len(all_imports)} imports in your project")
scoped_pkgs = utils.get_scope_from_imports(project_type, pkg_list,
all_imports)
else:
scoped_pkgs = utils.get_pkgs_by_scope(project_type, pkg_list)
if os.getenv("FETCH_LICENSE", "") in (True, "1", "true"):
licenses_results = bulk_lookup(
build_license_data(license_data_dir, spdx_license_list),
pkg_list=pkg_list,
)
license_report_file = os.path.join(
reports_dir, "license-" + project_type + ".json")
analyse_licenses(project_type, licenses_results,
license_report_file)
if project_type in risk_audit_map.keys():
if args.risk_audit:
console.print(
Panel(
f"Performing OSS Risk Audit for packages from {src_dir}\nNo of packages [bold]{len(pkg_list)}[/bold]. This will take a while ...",
title="OSS Risk Audit",
expand=False,
))
try:
risk_results = risk_audit(
project_type,
scoped_pkgs,
args.private_ns,
pkg_list,
risk_report_file,
)
analyse_pkg_risks(
project_type,
scoped_pkgs,
args.private_ns,
risk_results,
risk_report_file,
)
except Exception as e:
LOG.error(e)
LOG.error("Risk audit was not successful")
else:
console.print(
Panel(
"Depscan supports OSS Risk audit for this project.\nTo enable set the environment variable [bold]ENABLE_OSS_RISK=true[/bold]",
title="New Feature",
expand=False,
))
if project_type in type_audit_map.keys():
LOG.info("Performing remote audit for {} of type {}".format(
src_dir, project_type))
LOG.debug(f"No of packages {len(pkg_list)}")
try:
audit_results = audit(project_type, pkg_list, report_file)
if audit_results:
LOG.debug(
f"Remote audit yielded {len(audit_results)} results")
results = results + audit_results
except Exception as e:
LOG.error("Remote audit was not successful")
LOG.error(e)
results = None
# In case of docker, check if there are any npm packages that can be audited remotely
if project_type in ("podman", "docker"):
npm_pkg_list = get_pkg_by_type(pkg_list, "npm")
if npm_pkg_list:
LOG.debug(f"No of packages {len(npm_pkg_list)}")
try:
audit_results = audit("nodejs", npm_pkg_list, report_file)
if audit_results:
LOG.debug(
f"Remote audit yielded {len(audit_results)} results"
)
results = results + audit_results
except Exception as e:
LOG.error("Remote audit was not successful")
LOG.error(e)
if not dbLib.index_count(db["index_file"]):
run_cacher = True
else:
LOG.debug("Vulnerability database loaded from {}".format(
config.vdb_bin_file))
sources_list = [OSVSource(), NvdSource()]
if os.environ.get("GITHUB_TOKEN"):
sources_list.insert(0, GitHubSource())
if run_cacher:
for s in sources_list:
LOG.debug("Refreshing {}".format(s.__class__.__name__))
s.refresh()
run_cacher = False
elif args.sync:
for s in sources_list:
LOG.debug("Syncing {}".format(s.__class__.__name__))
s.download_recent()
run_cacher = False
LOG.debug("Vulnerability database contains {} records".format(
dbLib.index_count(db["index_file"])))
LOG.info("Performing regular scan for {} using plugin {}".format(
src_dir, project_type))
vdb_results, pkg_aliases, sug_version_dict = scan(
db, project_type, pkg_list, args.suggest)
if vdb_results:
results = results + vdb_results
# Summarise and print results
summary = summarise(
project_type,
results,
pkg_aliases,
sug_version_dict,
scoped_pkgs,
report_file,
True,
)
if summary and not args.noerror and len(project_types_list) == 1:
# Hard coded build break logic for now
if summary.get("CRITICAL") > 0:
sys.exit(1)
def main():
args = build_args()
if not args.no_banner:
print(at_logo)
src_dir = args.src_dir
if not args.src_dir:
src_dir = os.getcwd()
db = dbLib.get()
run_cacher = args.cache
areport_file = (
args.report_file
if args.report_file
else os.path.join(src_dir, "reports", "depscan.json")
)
reports_dir = os.path.dirname(areport_file)
# Create reports directory
if not os.path.exists(reports_dir):
os.makedirs(reports_dir)
# Detect the project types and perform the right type of scan
if args.project_type:
project_types_list = args.project_type.split(",")
else:
project_types_list = utils.detect_project_type(src_dir)
if len(project_types_list) > 1:
LOG.debug("Multiple project types found: {}".format(project_types_list))
for project_type in project_types_list:
sug_version_dict = {}
pkg_aliases = {}
report_file = areport_file.replace(".json", "-{}.json".format(project_type))
risk_report_file = areport_file.replace(
".json", "-risk.{}.json".format(project_type)
)
LOG.info("=" * 80)
creation_status = False
if args.bom and os.path.exists(args.bom):
bom_file = args.bom
creation_status = True
else:
bom_file = os.path.join(reports_dir, "bom-" + project_type + ".json")
creation_status = create_bom(project_type, bom_file, src_dir)
if not creation_status:
LOG.debug("Bom file {} was not created successfully".format(bom_file))
continue
LOG.debug("Scanning using the bom file {}".format(bom_file))
pkg_list = get_pkg_list(bom_file)
if not pkg_list:
LOG.debug("No packages found in the project!")
continue
scoped_pkgs = utils.get_pkgs_by_scope(pkg_list)
if not args.no_license_scan:
licenses_results = bulk_lookup(
build_license_data(license_data_dir), pkg_list=pkg_list
)
license_report_file = os.path.join(
reports_dir, "license-" + project_type + ".json"
)
analyse_licenses(project_type, licenses_results, license_report_file)
if project_type in risk_audit_map.keys():
if args.risk_audit:
console.print(
Panel(
f"Performing OSS Risk Audit for packages from {src_dir}\nNo of packages [bold]{len(pkg_list)}[/bold]. This will take a while ...",
title="OSS Risk Audit",
expand=False,
)
)
try:
risk_results = risk_audit(
project_type, args.private_ns, pkg_list, risk_report_file
)
analyse_pkg_risks(
project_type, args.private_ns, risk_results, risk_report_file
)
except Exception as e:
LOG.error(e)
LOG.error("Risk audit was not successful")
risk_results = None
else:
console.print(
Panel(
"Depscan supports OSS Risk audit for this project.\nTo enable set the environment variable [bold]ENABLE_OSS_RISK=true[/bold]",
title="New Feature",
expand=False,
)
)
if project_type in type_audit_map.keys():
LOG.info(
"Performing remote audit for {} of type {}".format(
src_dir, project_type
)
)
LOG.debug(f"No of packages {len(pkg_list)}")
try:
results = audit(project_type, pkg_list, report_file)
except Exception as e:
LOG.error("Remote audit was not successful")
LOG.error(e)
results = None
else:
if not dbLib.index_count(db["index_file"]):
run_cacher = True
else:
LOG.debug(
"Vulnerability database loaded from {}".format(config.vdb_bin_file)
)
sources_list = [NvdSource()]
if os.environ.get("GITHUB_TOKEN"):
sources_list.insert(0, GitHubSource())
else:
LOG.info(
"To use GitHub advisory source please set the environment variable GITHUB_TOKEN!"
)
if run_cacher:
for s in sources_list:
LOG.debug("Refreshing {}".format(s.__class__.__name__))
s.refresh()
elif args.sync:
for s in sources_list:
LOG.debug("Syncing {}".format(s.__class__.__name__))
s.download_recent()
LOG.debug(
"Vulnerability database contains {} records".format(
dbLib.index_count(db["index_file"])
)
)
LOG.info(
"Performing regular scan for {} using plugin {}".format(
src_dir, project_type
)
)
results, pkg_aliases, sug_version_dict = scan(
db, project_type, pkg_list, args.suggest
)
# Summarise and print results
summary = summarise(
project_type,
results,
pkg_aliases,
sug_version_dict,
scoped_pkgs,
report_file,
True,
)
if summary and not args.noerror and len(project_types_list) == 1:
# Hard coded build break logic for now
if summary.get("CRITICAL") > 0:
sys.exit(1)
def metadata_from_registry(registry_type, pkg_list, private_ns=None):
"""
Method to query registry for the package metadata
:param pkg_list: List of packages
:param private_ns: Private namespace
"""
metadata_dict = {}
task = None
# Circuit breaker flag to break the risk audit in case of many api errors
circuit_breaker = False
# Track the api failures count
failure_count = 0
done_count = 0
with Progress(
console=console,
transient=True,
redirect_stderr=False,
redirect_stdout=False,
refresh_per_second=1,
) as progress:
task = progress.add_task("[green] Auditing packages",
total=len(pkg_list),
start=True)
for pkg in pkg_list:
if circuit_breaker:
LOG.info(
"Risk audited has been interrupted due to frequent api errors. Please try again later."
)
progress.stop()
return {}
scope = pkg.get("scope", "").lower()
key, lookup_url = get_lookup_url(registry_type, pkg)
if not key or key.startswith("https://"):
progress.advance(task)
continue
progress.update(task, description=f"Checking {key}")
try:
r = requests.get(url=lookup_url,
timeout=config.request_timeout_sec)
json_data = r.json()
# Npm returns this error if the package is not found
if (json_data.get("code") == "MethodNotAllowedError"
or r.status_code > 400):
continue
is_private_pkg = False
if private_ns:
namespace_prefixes = private_ns.split(",")
for ns in namespace_prefixes:
if key.lower().startswith(ns.lower()) or key.lower(
).startswith("@" + ns.lower()):
is_private_pkg = True
break
risk_metrics = {}
if registry_type == "npm":
risk_metrics = npm_pkg_risk(json_data, is_private_pkg,
scope)
elif registry_type == "pypi":
risk_metrics = pypi_pkg_risk(json_data, is_private_pkg,
scope)
metadata_dict[key] = {
"scope": scope,
"pkg_metadata": json_data,
"risk_metrics": risk_metrics,
"is_private_pkg": is_private_pkg,
}
except Exception as e:
LOG.debug(e)
failure_count = failure_count + 1
progress.advance(task)
done_count = done_count + 1
if failure_count >= config.max_request_failures:
circuit_breaker = True
LOG.debug(
f"Retrieved package metadata for {done_count}/{len(pkg_list)} packages. Failures count {failure_count}"
)
return metadata_dict