def cli(
click_ctx: click.Context,
output: str,
package_name: str,
package_version: str,
no_pretty: bool = False,
verbose: bool = False,
) -> None:
"""Obtain dependent packages for the given package, respect registered solvers in Thoth.
The result of reverse solver is a list of dependencies that depend on the given package.
"""
if click_ctx:
click_ctx.auto_envvar_prefix = "THOTH_REVSOLVER"
if verbose:
_LOGGER.setLevel(logging.DEBUG)
_LOGGER.debug("Debug mode is on")
_LOGGER.debug("Version: %s", __component_version__)
start_time = time.monotonic()
result = do_solve(package_name=package_name,
package_version=package_version)
print_command_result(
click_ctx,
result,
analyzer=__title__,
analyzer_version=__component_version__,
output=output,
duration=time.monotonic() - start_time,
pretty=not no_pretty,
)
def dependencies(
click_ctx,
log: str,
ceph_document_id: str = None,
output: str = "plain",
report_output: str = "-",
pretty: bool = False,
):
"""Process dependencies from the log file."""
if ceph_document_id:
_get_document(ceph_document_id, log)
with open(log, "r") as f:
build_log: str = f.read()
df = build_log_to_dependency_table(build_log)
result = _format_table(df, output=output, pretty=pretty)
if ceph_document_id:
print_command_result(
click_ctx=click_ctx,
result=result,
analyzer=analyzer_name,
analyzer_version=analyzer_version,
output=report_output,
pretty=pretty,
)
else:
click.echo(result)
sys.exit(0)
def cli_extract_image(
click_ctx,
image,
timeout=None,
no_pretty=False,
output=None,
registry_credentials=None,
no_tls_verify=False,
):
"""Extract installed packages from an image."""
start_time = time.monotonic()
result = extract_image(
image,
timeout,
registry_credentials=registry_credentials,
tls_verify=not no_tls_verify,
)
print_command_result(
click_ctx,
result,
analyzer=analyzer,
analyzer_version=analyzer_version,
output=output or "-",
duration=time.monotonic() - start_time,
pretty=not no_pretty,
)
def analyze(
click_ctx,
log: str,
ceph_document_id: str = None,
output: str = "plain",
report_output: str = "-",
pretty: bool = False,
):
"""Analyze raw build log and produce tabular output."""
if ceph_document_id:
_get_document(ceph_document_id, log)
with open(log, "r") as f:
build_log: str = f.read()
_, df = build_breaker_analyze(build_log)
result = _format_table(df, output=output, pretty=pretty)
if ceph_document_id:
print_command_result(
click_ctx=click_ctx,
result=result,
analyzer=analyzer_name,
analyzer_version=analyzer_version,
output=report_output,
pretty=pretty,
)
else:
click.echo(result)
sys.exit(0)
def report(
click_ctx,
log: str,
ceph_document_id: str = None,
limit: int = 5,
report_output: str = "-",
handler: str = None,
colorize: bool = False,
pretty: bool = False,
) -> str:
"""Analyze raw build log and produce a report."""
start_time = time.monotonic()
if ceph_document_id:
_get_document(ceph_document_id, log)
with open(log, "r") as f:
build_log: str = f.read()
result: dict = build_breaker_report(log=build_log, handler=handler, top=limit, colorize=colorize)
if ceph_document_id:
print_command_result(
click_ctx=click_ctx,
result=result,
analyzer=analyzer_name,
analyzer_version=analyzer_version,
output=report_output,
duration=time.monotonic() - start_time,
pretty=pretty,
)
else:
if pretty:
result: str = pformat(result)
click.echo(result)
sys.exit(0)
def parse(
click_ctx: click.Context,
*,
output: str,
no_pretty: bool = False,
input_stream: str,
):
"""Parse the given build log and extract relevant information out of it."""
parameters = locals()
parameters.pop("click_ctx")
if input_stream == "-":
input_text = sys.stdin.read()
else:
with open(input_stream, "r") as input_f:
input_text = input_f.read()
duration = time.monotonic()
result = do_parse(input_text)
duration = time.monotonic() - duration
print_command_result(
click_ctx=click_ctx,
analyzer=analyzer_name,
analyzer_version=analyzer_version,
output=output,
pretty=not no_pretty,
duration=duration,
result=result,
)
click_ctx.exit(0)
def si_bandit(
click_ctx,
output: Optional[str],
from_directory: Optional[str],
package_name: str,
package_version: Optional[str],
package_index: Optional[str],
no_pretty: bool,
):
"""Run the cli for si-bandit."""
if from_directory is None:
with tempfile.TemporaryDirectory() as d:
command = (
f"pip download --no-binary=:all: --no-deps -d {d} -i {package_index} "
f"{package_name}==={package_version}")
run_command(command)
for f in os.listdir(d):
if f.endswith(".tar.gz"):
full_path = os.path.join(d, f)
tar = tarfile.open(full_path, "r:gz")
tar.extractall(os.path.join(d, "package"))
from_directory = os.path.join(d, "package")
break
else:
raise FileNotFoundError(
f"No source distribution found for {package_name}==={package_version} "
f"on {package_index}")
out = _run_bandit(from_directory)
else:
out = _run_bandit(from_directory)
if out is None:
raise RuntimeError("output of bandit is empty")
source_dict = {
"url": package_index,
"name": "foo",
"verify_ssl": False,
"warehouse": True
}
source = Source.from_dict(source_dict)
try:
out["time_of_release"] = source.get_package_release_date(
package_name=package_name, package_version=package_version)
except Exception:
out["time_of_release"] = None
print_command_result(
click_ctx=click_ctx,
result=out,
analyzer=si_bandit_title,
analyzer_version=si_bandit_version,
output=output,
duration=None,
pretty=not no_pretty,
)
def cli_extract_buildlog(click_ctx, input_file, no_pretty=False, output=None):
"""Extract installed packages from a build log."""
result = extract_buildlog(input_file.read())
print_command_result(click_ctx,
result,
analyzer=analyzer,
analyzer_version=analyzer_version,
output=output or '-',
pretty=not no_pretty)
def provenance(
click_ctx: click.Context,
requirements: str,
requirements_locked: str,
output: str,
whitelisted_sources: Optional[str] = None,
no_pretty: bool = False,
):
"""Check provenance of packages based on configuration."""
parameters = locals()
parameters.pop("click_ctx")
start_time = time.monotonic()
_LOGGER.debug("Passed arguments: %s", parameters)
whitelisted_sources = whitelisted_sources.split(",") if whitelisted_sources else []
result = {
"error": None,
"report": [],
"parameters": {"whitelisted_indexes": whitelisted_sources, "project": None},
"input": None,
}
try:
project = _instantiate_project(requirements, requirements_locked)
result["parameters"]["project"] = project.to_dict()
report = project.check_provenance(
whitelisted_sources=whitelisted_sources,
digests_fetcher=GraphDigestsFetcher(),
)
except (AdviserException, UnsupportedConfiguration) as exc:
if isinstance(exc, InternalError):
# Re-raise internal exceptions that shouldn't occur here.
raise
_LOGGER.exception("Error during checking provenance: %s", str(exc))
result["error"] = True
result["error_msg"] = str(exc)
result["report"] = [
{"type": "ERROR", "justification": f"{str(exc)} ({type(exc).__name__})"}
]
else:
result["error"] = False
result["error_msg"] = None
result["report"] = report
print_command_result(
click_ctx,
result,
analyzer=analyzer_name,
analyzer_version=analyzer_version,
output=output,
duration=time.monotonic() - start_time,
pretty=not no_pretty,
)
click_ctx.exit(int(result["error"] is True))
def provenance(
click_ctx,
requirements,
requirements_locked=None,
whitelisted_sources=None,
output=None,
files=False,
no_pretty=False,
):
"""Check provenance of packages based on configuration."""
start_time = time.monotonic()
_LOGGER.debug("Passed arguments: %s", locals())
whitelisted_sources = whitelisted_sources.split(",") if whitelisted_sources else []
result = {
"error": None,
"report": [],
"parameters": {"whitelisted_indexes": whitelisted_sources},
"input": None,
}
try:
project = _instantiate_project(requirements, requirements_locked, files)
result["input"] = project.to_dict()
report = project.check_provenance(
whitelisted_sources=whitelisted_sources,
digests_fetcher=GraphDigestsFetcher(),
)
except ThothAdviserException as exc:
# TODO: we should extend exceptions so they are capable of storing more info.
if isinstance(exc, InternalError):
# Re-raise internal exceptions that shouldn't occur here.
raise
_LOGGER.exception("Error during checking provenance: %s", str(exc))
result["error"] = True
result["report"] = [
{"type": "ERROR", "justification": f"{str(exc)} ({type(exc).__name__})"}
]
else:
result["error"] = False
result["report"] = report
print_command_result(
click_ctx,
result,
analyzer=analyzer_name,
analyzer_version=analyzer_version,
output=output,
duration=time.monotonic() - start_time,
pretty=not no_pretty,
)
return int(result["error"] is True)
def cli_extract_buildlog(click_ctx, input_file, no_pretty=False, output=None):
"""Extract installed packages from a build log."""
start_time = time.monotonic()
result = extract_buildlog(input_file.read())
print_command_result(
click_ctx,
result,
analyzer=analyzer,
analyzer_version=analyzer_version,
output=output or "-",
duration=time.monotonic() - start_time,
pretty=not no_pretty,
)
def si_cloc(
click_ctx,
output: Optional[str],
from_directory: Optional[str],
package_name: str,
package_version: Optional[str],
package_index: Optional[str],
no_pretty: bool,
):
"""Run the cli for si-cloc."""
if from_directory is None:
with tempfile.TemporaryDirectory() as d:
command = (
f"pip download --no-binary=:all: --no-deps -d {d} -i {package_index} "
f"{package_name}==={package_version}")
run_command(command)
for f in os.listdir(d):
if f.endswith(".tar.gz"):
full_path = os.path.join(d, f)
break
else:
raise FileNotFoundError(
f"No source distribution found for {package_name}==={package_version} "
f"on {package_index}")
out = run_command(
f"cloc --extract-with='gzip -dc >FILE< | tar xf -' {full_path} --json",
is_json=True)
else:
out = run_command(f"cloc {from_directory} --json", is_json=True)
results = out.stdout
if results is None:
results = {"error": True, "error_messages": [out.stderr]}
_LOGGER.warning(
"cloc output is empty with the following in stderr:\n%s",
out.stderr)
else:
results["error"] = False
print_command_result(
click_ctx=click_ctx,
result=results,
analyzer=__title__,
analyzer_version=__version__,
output=output,
duration=None,
pretty=not no_pretty,
)
def parse(
click_ctx: click.Context,
*,
output: str,
no_pretty: bool = False,
input_stream: str,
jsonpath: Optional[str] = None,
):
"""Parse the given build log and extract relevant information out of it."""
parameters = locals()
parameters.pop("click_ctx")
if input_stream == "-":
input_content = sys.stdin.read()
else:
with open(input_stream, "r") as input_f:
input_content = input_f.read()
input_text = input_content
if jsonpath:
expr = Path.parse_str(jsonpath)
match = [m.current_value for m in expr.match(json.loads(input_content))]
if len(match) > 1:
_LOGGER.error("Matched multiple entries based on the jsonpath (%r): %r", jsonpath, match)
sys.exit(1)
if not match:
_LOGGER.error("No match for the provided jsonpath: %r", jsonpath)
sys.exit(1)
input_text = match[0]
duration = time.monotonic()
result = do_parse(input_text)
duration = time.monotonic() - duration
print_command_result(
click_ctx=click_ctx,
analyzer=analyzer_name,
analyzer_version=analyzer_version,
output=output,
pretty=not no_pretty,
duration=duration,
result=result,
)
click_ctx.exit(0)
def pypi(
click_ctx,
requirements,
index=None,
python_version=3,
exclude_packages=None,
output=None,
subgraph_check_api=None,
no_transitive=True,
no_pretty=False,
):
"""Manipulate with dependency requirements using PyPI."""
start_time = time.monotonic()
requirements = [
requirement.strip() for requirement in requirements.split("\\n")
if requirement
]
if not requirements:
_LOG.error("No requirements specified, exiting")
sys.exit(1)
if not subgraph_check_api:
_LOG.info(
"No subgraph check API provided, no queries will be done for dependency subgraphs that should be avoided"
) # Ignore PycodestyleBear (E501)
result = resolve_python(
requirements,
index_urls=index.split(",") if index else
("https://pypi.org/simple", ),
python_version=int(python_version),
transitive=not no_transitive,
exclude_packages=set(
map(str.strip, (exclude_packages or "").split(","))),
subgraph_check_api=subgraph_check_api,
)
print_command_result(
click_ctx,
result,
analyzer=analyzer_name,
analyzer_version=analyzer_version,
output=output or "-",
duration=time.monotonic() - start_time,
pretty=not no_pretty,
)
def si_bandit(
click_ctx,
output: Optional[str],
from_directory: Optional[str],
package_name: str,
package_version: Optional[str],
package_index: Optional[str],
no_pretty: bool,
):
"""Run the cli for si-bandit."""
if from_directory is None:
with tempfile.TemporaryDirectory() as d:
command = (
f"pip download --no-binary=:all: --no-deps -d {d} -i {package_index} "
f"{package_name}==={package_version}")
run_command(command)
for f in os.listdir(d):
if f.endswith(".tar.gz"):
full_path = os.path.join(d, f)
tar = tarfile.open(full_path, "r:gz")
tar.extractall(os.path.join(d, "package"))
from_directory = os.path.join(d, "package")
break
else:
raise FileNotFoundError(
f"No source distribution found for {package_name}==={package_version} "
f"on {package_index}")
out = _run_bandit(from_directory)
else:
out = _run_bandit(from_directory)
out["package_name"] = package_name
out["package_version"] = package_version
out["bandit_version"] = bandit_version
out["package_index"] = package_index
print_command_result(
click_ctx=click_ctx,
result=out,
analyzer=si_bandit_title,
analyzer_version=si_bandit_version,
output=output,
duration=None,
pretty=not no_pretty,
)
def cli_extract_image(click_ctx,
image,
timeout=None,
no_pretty=False,
output=None,
registry_credentials=None,
no_tls_verify=False):
"""Extract installed packages from an image."""
result = extract_image(image,
timeout,
registry_credentials=registry_credentials,
tls_verify=not no_tls_verify)
print_command_result(click_ctx,
result,
analyzer=analyzer,
analyzer_version=analyzer_version,
output=output or '-',
pretty=not no_pretty)
def python(
click_ctx,
requirements,
index=None,
python_version=3,
exclude_packages=None,
output=None,
no_transitive=True,
no_pretty=False,
virtualenv=None,
limited_output=False,
):
"""Manipulate with dependency requirements using PyPI."""
start_time = time.monotonic()
requirements = [
requirement.strip() for requirement in requirements.split("\\n")
if requirement
]
if not requirements:
_LOG.error("No requirements specified, exiting")
sys.exit(1)
result = resolve_python(
requirements,
index_urls=index.split(",") if index else
("https://pypi.org/simple", ),
python_version=int(python_version),
transitive=not no_transitive,
exclude_packages=set(
map(str.strip, (exclude_packages or "").split(","))),
virtualenv=virtualenv,
limited_output=limited_output,
)
print_command_result(
click_ctx,
result,
analyzer=analyzer_name,
analyzer_version=analyzer_version,
output=output or "-",
duration=time.monotonic() - start_time,
pretty=not no_pretty,
)
def python(
click_ctx,
package_name: str,
package_version: str,
index_url: str = None,
no_pretty: bool = True,
output: str = None,
):
"""Fetch digests for packages in Python ecosystem."""
result = {}
python_fetcher = PythonDigestsFetcher(index_url)
result[index_url] = python_fetcher.fetch(package_name, package_version)
print_command_result(
click_ctx,
result,
analyzer=analyzer_name,
analyzer_version=analyzer_version,
output=output or "-",
pretty=not no_pretty,
)
def si_aggregator(
click_ctx,
output: Optional[str],
package_name: str,
package_version: Optional[str],
package_index: Optional[str],
si_bandit_results: str,
si_cloc_results: str,
aggregation_func: str,
no_pretty: bool,
) -> None:
"""Run the cli for si-aggregator."""
agg_func = getattr(aggregators, aggregation_func)
if agg_func is None:
raise NotImplementedError(
f"{aggregation_func} aggregation function not implemented yet.")
with open(si_bandit_results, "r") as f:
si_bandit_results = json.load(f)
with open(si_cloc_results, "r") as f:
si_cloc_results = json.load(f)
out = agg_func(si_bandit_results=si_bandit_results,
si_cloc_results=si_cloc_results)
if out.get("error", False):
out["metadata"] = _gen_metadata_on_err()
out["bandit_results"] = si_bandit_results
out["cloc_results"] = si_cloc_results
print_command_result(
click_ctx=click_ctx,
result=out,
analyzer=__title__,
analyzer_version=__version__,
output=output,
duration=None,
pretty=not no_pretty,
)
def python(
click_ctx,
package_name: str,
package_version: str,
index_url: str = None,
no_pretty: bool = True,
output: str = None,
dry_run: bool = False,
):
"""Fetch digests for packages in Python ecosystem."""
start_time = time.monotonic()
python_fetcher = PythonDigestsFetcher(index_url)
result = python_fetcher.fetch(package_name, package_version)
print_command_result(
click_ctx,
result,
analyzer=analyzer_name,
analyzer_version=analyzer_version,
output=output or "-",
duration=time.monotonic() - start_time,
pretty=not no_pretty,
dry_run=dry_run,
)
def advise(
click_ctx,
requirements,
requirements_format=None,
requirements_locked=None,
recommendation_type=None,
runtime_environment=None,
output=None,
no_pretty=False,
files=False,
count=None,
limit=None,
library_usage=None,
limit_latest_versions=None,
):
"""Advise package and package versions in the given stack or on solely package only."""
_LOGGER.debug("Passed arguments: %s", locals())
limit = int(limit) if limit else None
count = int(count) if count else None
# A special value of -1 signalizes no limit/count, this is a workaround for Click's option parser.
if count == -1:
count = None
if limit == -1:
limit = None
if limit_latest_versions == -1:
limit_latest_versions = None
if library_usage:
if os.path.isfile(library_usage):
try:
library_usage = json.loads(Path(library_usage).read_text())
except Exception as exc:
_LOGGER.error("Failed to load library usage file %r", library_usage)
raise
else:
library_usage = json.loads(library_usage)
runtime_environment = RuntimeEnvironment.load(runtime_environment)
recommendation_type = RecommendationType.by_name(recommendation_type)
requirements_format = PythonRecommendationOutput.by_name(requirements_format)
result = {
"error": None,
"report": [],
"stack_info": None,
"advised_configuration": None,
"pipeline_configuration": None,
"parameters": {
"runtime_environment": runtime_environment.to_dict(),
"recommendation_type": recommendation_type.name,
"library_usage": library_usage,
"requirements_format": requirements_format.name,
"limit": limit,
"limit_latest_versions": limit_latest_versions,
"count": count,
"no_pretty": no_pretty,
},
"input": None,
}
try:
project = _instantiate_project(
requirements, requirements_locked, files, runtime_environment
)
result["input"] = project.to_dict()
if runtime_environment:
_LOGGER.info(
"Runtime environment configuration:\n%s",
json.dumps(runtime_environment.to_dict(), sort_keys=True, indent=2),
)
else:
_LOGGER.info("No runtime environment configuration supplied")
if library_usage:
_LOGGER.info(
"Library usage:\n%s",
json.dumps(library_usage, sort_keys=True, indent=2),
)
else:
_LOGGER.info("No library usage supplied")
stack_info, advised_configuration, report, pipeline_configuration = Adviser.compute_on_project(
project,
recommendation_type=recommendation_type,
library_usage=library_usage,
count=count,
limit=limit,
limit_latest_versions=limit_latest_versions,
)
except ThothAdviserException as exc:
# TODO: we should extend exceptions so they are capable of storing more info.
if isinstance(exc, InternalError):
# Re-raise internal exceptions that shouldn't occur here.
raise
_LOGGER.exception("Error during computing recommendation: %s", str(exc))
result["error"] = True
result["report"] = [([{"justification": f"{str(exc)}", "type": "ERROR"}], None)]
except NoReleasesFound as exc:
result["error"] = True
result["report"] = [([{
"justification": f"{str(exc)}; analysis of the missing package will be "
f"automatically scheduled by the system",
"type": "ERROR"
}], None)]
except (SolverException, UnableLock) as exc:
result["error"] = True
result["report"] = [([{"justification": str(exc), "type": "ERROR"}], None)]
else:
# Convert report to a dict so its serialized.
result["report"] = [
(justification, project.to_dict(), overall_score)
for justification, project, overall_score in report
]
# Report error if we did not find any recommendation to the user, the
# stack_info carries information on why it hasn't been found.
result["error"] = len(result["report"]) == 0
result["stack_info"] = stack_info
if result["error"]:
result["stack_info"].append({
"type": "ERROR",
"justification": "Recommendation engine did not produce any stacks"
})
result["advised_configuration"] = advised_configuration
result["pipeline_configuration"] = pipeline_configuration
print_command_result(
click_ctx,
result,
analyzer=analyzer_name,
analyzer_version=analyzer_version,
output=output,
pretty=not no_pretty,
)
return int(result["error"] is True)
def dependency_monkey(
click_ctx,
requirements: str,
stack_output: str,
report_output: str,
files: bool,
seed: int = None,
decision_type: str = None,
dry_run: bool = False,
context: str = None,
no_pretty: bool = False,
count: int = None,
runtime_environment: dict = None,
limit_latest_versions: int = None,
library_usage: str = None,
):
"""Generate software stacks based on all valid resolutions that conform version ranges."""
# We cannot have these as ints in click because they are optional and we
# cannot pass empty string as an int as env variable.
seed = int(seed) if seed else None
count = int(count) if count else None
limit_latest_versions = (
int(limit_latest_versions) if limit_latest_versions else None
)
# A special value of -1 signalizes no limit, this is a workaround for Click's option parser.
if count == -1:
count = None
if limit_latest_versions == -1:
limit_latest_versions = None
runtime_environment = RuntimeEnvironment.load(runtime_environment)
decision_type = DecisionType.by_name(decision_type)
project = _instantiate_project(
requirements,
requirements_locked=None,
files=files,
runtime_environment=runtime_environment,
)
if library_usage:
if os.path.isfile(library_usage):
try:
library_usage = json.loads(Path(library_usage).read_text())
except Exception as exc:
_LOGGER.error("Failed to load library usage file %r: %s", library_usage, str(exc))
raise
else:
library_usage = json.loads(library_usage)
if runtime_environment:
_LOGGER.info(
"Runtime environment configuration:\n%s",
json.dumps(runtime_environment.to_dict(), sort_keys=True, indent=2),
)
else:
_LOGGER.info("No runtime environment configuration supplied")
if library_usage:
_LOGGER.info(
"Library usage:\n%s", json.dumps(library_usage, sort_keys=True, indent=2)
)
else:
_LOGGER.info("No library usage supplied")
result = {
"error": None,
"parameters": {
"requirements": project.pipfile.to_dict(),
"runtime_environment": runtime_environment.to_dict(),
"seed": seed,
"decision_type": decision_type.name,
"library_usage": library_usage,
"context": deepcopy(
context
), # We reuse context later, perform deepcopy to report the one on input.
"stack_output": stack_output,
"report_output": report_output,
"files": files,
"dry_run": dry_run,
"no_pretty": no_pretty,
"count": count,
},
"input": None,
"output": [],
"computed": None,
}
try:
report = run_dependency_monkey(
project,
stack_output,
seed=seed,
decision_type=decision_type,
dry_run=dry_run,
context=context,
count=count,
limit_latest_versions=limit_latest_versions,
library_usage=library_usage,
)
# Place report into result.
result.update(report)
except SolverException:
_LOGGER.exception("An error occurred during dependency monkey run")
result["error"] = traceback.format_exc()
print_command_result(
click_ctx,
result,
analyzer=analyzer_name,
analyzer_version=analyzer_version,
output=report_output,
pretty=not no_pretty,
)
return sys.exit(result["error"] is not None)
