def cli( click_ctx: click.Context, output: str, package_name: str, package_version: str, no_pretty: bool = False, verbose: bool = False, ) -> None: """Obtain dependent packages for the given package, respect registered solvers in Thoth. The result of reverse solver is a list of dependencies that depend on the given package. """ if click_ctx: click_ctx.auto_envvar_prefix = "THOTH_REVSOLVER" if verbose: _LOGGER.setLevel(logging.DEBUG) _LOGGER.debug("Debug mode is on") _LOGGER.debug("Version: %s", __component_version__) start_time = time.monotonic() result = do_solve(package_name=package_name, package_version=package_version) print_command_result( click_ctx, result, analyzer=__title__, analyzer_version=__component_version__, output=output, duration=time.monotonic() - start_time, pretty=not no_pretty, )
def dependencies( click_ctx, log: str, ceph_document_id: str = None, output: str = "plain", report_output: str = "-", pretty: bool = False, ): """Process dependencies from the log file.""" if ceph_document_id: _get_document(ceph_document_id, log) with open(log, "r") as f: build_log: str = f.read() df = build_log_to_dependency_table(build_log) result = _format_table(df, output=output, pretty=pretty) if ceph_document_id: print_command_result( click_ctx=click_ctx, result=result, analyzer=analyzer_name, analyzer_version=analyzer_version, output=report_output, pretty=pretty, ) else: click.echo(result) sys.exit(0)
def cli_extract_image( click_ctx, image, timeout=None, no_pretty=False, output=None, registry_credentials=None, no_tls_verify=False, ): """Extract installed packages from an image.""" start_time = time.monotonic() result = extract_image( image, timeout, registry_credentials=registry_credentials, tls_verify=not no_tls_verify, ) print_command_result( click_ctx, result, analyzer=analyzer, analyzer_version=analyzer_version, output=output or "-", duration=time.monotonic() - start_time, pretty=not no_pretty, )
def analyze( click_ctx, log: str, ceph_document_id: str = None, output: str = "plain", report_output: str = "-", pretty: bool = False, ): """Analyze raw build log and produce tabular output.""" if ceph_document_id: _get_document(ceph_document_id, log) with open(log, "r") as f: build_log: str = f.read() _, df = build_breaker_analyze(build_log) result = _format_table(df, output=output, pretty=pretty) if ceph_document_id: print_command_result( click_ctx=click_ctx, result=result, analyzer=analyzer_name, analyzer_version=analyzer_version, output=report_output, pretty=pretty, ) else: click.echo(result) sys.exit(0)
def report( click_ctx, log: str, ceph_document_id: str = None, limit: int = 5, report_output: str = "-", handler: str = None, colorize: bool = False, pretty: bool = False, ) -> str: """Analyze raw build log and produce a report.""" start_time = time.monotonic() if ceph_document_id: _get_document(ceph_document_id, log) with open(log, "r") as f: build_log: str = f.read() result: dict = build_breaker_report(log=build_log, handler=handler, top=limit, colorize=colorize) if ceph_document_id: print_command_result( click_ctx=click_ctx, result=result, analyzer=analyzer_name, analyzer_version=analyzer_version, output=report_output, duration=time.monotonic() - start_time, pretty=pretty, ) else: if pretty: result: str = pformat(result) click.echo(result) sys.exit(0)
def parse( click_ctx: click.Context, *, output: str, no_pretty: bool = False, input_stream: str, ): """Parse the given build log and extract relevant information out of it.""" parameters = locals() parameters.pop("click_ctx") if input_stream == "-": input_text = sys.stdin.read() else: with open(input_stream, "r") as input_f: input_text = input_f.read() duration = time.monotonic() result = do_parse(input_text) duration = time.monotonic() - duration print_command_result( click_ctx=click_ctx, analyzer=analyzer_name, analyzer_version=analyzer_version, output=output, pretty=not no_pretty, duration=duration, result=result, ) click_ctx.exit(0)
def si_bandit( click_ctx, output: Optional[str], from_directory: Optional[str], package_name: str, package_version: Optional[str], package_index: Optional[str], no_pretty: bool, ): """Run the cli for si-bandit.""" if from_directory is None: with tempfile.TemporaryDirectory() as d: command = ( f"pip download --no-binary=:all: --no-deps -d {d} -i {package_index} " f"{package_name}==={package_version}") run_command(command) for f in os.listdir(d): if f.endswith(".tar.gz"): full_path = os.path.join(d, f) tar = tarfile.open(full_path, "r:gz") tar.extractall(os.path.join(d, "package")) from_directory = os.path.join(d, "package") break else: raise FileNotFoundError( f"No source distribution found for {package_name}==={package_version} " f"on {package_index}") out = _run_bandit(from_directory) else: out = _run_bandit(from_directory) if out is None: raise RuntimeError("output of bandit is empty") source_dict = { "url": package_index, "name": "foo", "verify_ssl": False, "warehouse": True } source = Source.from_dict(source_dict) try: out["time_of_release"] = source.get_package_release_date( package_name=package_name, package_version=package_version) except Exception: out["time_of_release"] = None print_command_result( click_ctx=click_ctx, result=out, analyzer=si_bandit_title, analyzer_version=si_bandit_version, output=output, duration=None, pretty=not no_pretty, )
def cli_extract_buildlog(click_ctx, input_file, no_pretty=False, output=None): """Extract installed packages from a build log.""" result = extract_buildlog(input_file.read()) print_command_result(click_ctx, result, analyzer=analyzer, analyzer_version=analyzer_version, output=output or '-', pretty=not no_pretty)
def provenance( click_ctx: click.Context, requirements: str, requirements_locked: str, output: str, whitelisted_sources: Optional[str] = None, no_pretty: bool = False, ): """Check provenance of packages based on configuration.""" parameters = locals() parameters.pop("click_ctx") start_time = time.monotonic() _LOGGER.debug("Passed arguments: %s", parameters) whitelisted_sources = whitelisted_sources.split(",") if whitelisted_sources else [] result = { "error": None, "report": [], "parameters": {"whitelisted_indexes": whitelisted_sources, "project": None}, "input": None, } try: project = _instantiate_project(requirements, requirements_locked) result["parameters"]["project"] = project.to_dict() report = project.check_provenance( whitelisted_sources=whitelisted_sources, digests_fetcher=GraphDigestsFetcher(), ) except (AdviserException, UnsupportedConfiguration) as exc: if isinstance(exc, InternalError): # Re-raise internal exceptions that shouldn't occur here. raise _LOGGER.exception("Error during checking provenance: %s", str(exc)) result["error"] = True result["error_msg"] = str(exc) result["report"] = [ {"type": "ERROR", "justification": f"{str(exc)} ({type(exc).__name__})"} ] else: result["error"] = False result["error_msg"] = None result["report"] = report print_command_result( click_ctx, result, analyzer=analyzer_name, analyzer_version=analyzer_version, output=output, duration=time.monotonic() - start_time, pretty=not no_pretty, ) click_ctx.exit(int(result["error"] is True))
def provenance( click_ctx, requirements, requirements_locked=None, whitelisted_sources=None, output=None, files=False, no_pretty=False, ): """Check provenance of packages based on configuration.""" start_time = time.monotonic() _LOGGER.debug("Passed arguments: %s", locals()) whitelisted_sources = whitelisted_sources.split(",") if whitelisted_sources else [] result = { "error": None, "report": [], "parameters": {"whitelisted_indexes": whitelisted_sources}, "input": None, } try: project = _instantiate_project(requirements, requirements_locked, files) result["input"] = project.to_dict() report = project.check_provenance( whitelisted_sources=whitelisted_sources, digests_fetcher=GraphDigestsFetcher(), ) except ThothAdviserException as exc: # TODO: we should extend exceptions so they are capable of storing more info. if isinstance(exc, InternalError): # Re-raise internal exceptions that shouldn't occur here. raise _LOGGER.exception("Error during checking provenance: %s", str(exc)) result["error"] = True result["report"] = [ {"type": "ERROR", "justification": f"{str(exc)} ({type(exc).__name__})"} ] else: result["error"] = False result["report"] = report print_command_result( click_ctx, result, analyzer=analyzer_name, analyzer_version=analyzer_version, output=output, duration=time.monotonic() - start_time, pretty=not no_pretty, ) return int(result["error"] is True)
def cli_extract_buildlog(click_ctx, input_file, no_pretty=False, output=None): """Extract installed packages from a build log.""" start_time = time.monotonic() result = extract_buildlog(input_file.read()) print_command_result( click_ctx, result, analyzer=analyzer, analyzer_version=analyzer_version, output=output or "-", duration=time.monotonic() - start_time, pretty=not no_pretty, )
def si_cloc( click_ctx, output: Optional[str], from_directory: Optional[str], package_name: str, package_version: Optional[str], package_index: Optional[str], no_pretty: bool, ): """Run the cli for si-cloc.""" if from_directory is None: with tempfile.TemporaryDirectory() as d: command = ( f"pip download --no-binary=:all: --no-deps -d {d} -i {package_index} " f"{package_name}==={package_version}") run_command(command) for f in os.listdir(d): if f.endswith(".tar.gz"): full_path = os.path.join(d, f) break else: raise FileNotFoundError( f"No source distribution found for {package_name}==={package_version} " f"on {package_index}") out = run_command( f"cloc --extract-with='gzip -dc >FILE< | tar xf -' {full_path} --json", is_json=True) else: out = run_command(f"cloc {from_directory} --json", is_json=True) results = out.stdout if results is None: results = {"error": True, "error_messages": [out.stderr]} _LOGGER.warning( "cloc output is empty with the following in stderr:\n%s", out.stderr) else: results["error"] = False print_command_result( click_ctx=click_ctx, result=results, analyzer=__title__, analyzer_version=__version__, output=output, duration=None, pretty=not no_pretty, )
def parse( click_ctx: click.Context, *, output: str, no_pretty: bool = False, input_stream: str, jsonpath: Optional[str] = None, ): """Parse the given build log and extract relevant information out of it.""" parameters = locals() parameters.pop("click_ctx") if input_stream == "-": input_content = sys.stdin.read() else: with open(input_stream, "r") as input_f: input_content = input_f.read() input_text = input_content if jsonpath: expr = Path.parse_str(jsonpath) match = [m.current_value for m in expr.match(json.loads(input_content))] if len(match) > 1: _LOGGER.error("Matched multiple entries based on the jsonpath (%r): %r", jsonpath, match) sys.exit(1) if not match: _LOGGER.error("No match for the provided jsonpath: %r", jsonpath) sys.exit(1) input_text = match[0] duration = time.monotonic() result = do_parse(input_text) duration = time.monotonic() - duration print_command_result( click_ctx=click_ctx, analyzer=analyzer_name, analyzer_version=analyzer_version, output=output, pretty=not no_pretty, duration=duration, result=result, ) click_ctx.exit(0)
def pypi( click_ctx, requirements, index=None, python_version=3, exclude_packages=None, output=None, subgraph_check_api=None, no_transitive=True, no_pretty=False, ): """Manipulate with dependency requirements using PyPI.""" start_time = time.monotonic() requirements = [ requirement.strip() for requirement in requirements.split("\\n") if requirement ] if not requirements: _LOG.error("No requirements specified, exiting") sys.exit(1) if not subgraph_check_api: _LOG.info( "No subgraph check API provided, no queries will be done for dependency subgraphs that should be avoided" ) # Ignore PycodestyleBear (E501) result = resolve_python( requirements, index_urls=index.split(",") if index else ("https://pypi.org/simple", ), python_version=int(python_version), transitive=not no_transitive, exclude_packages=set( map(str.strip, (exclude_packages or "").split(","))), subgraph_check_api=subgraph_check_api, ) print_command_result( click_ctx, result, analyzer=analyzer_name, analyzer_version=analyzer_version, output=output or "-", duration=time.monotonic() - start_time, pretty=not no_pretty, )
def si_bandit( click_ctx, output: Optional[str], from_directory: Optional[str], package_name: str, package_version: Optional[str], package_index: Optional[str], no_pretty: bool, ): """Run the cli for si-bandit.""" if from_directory is None: with tempfile.TemporaryDirectory() as d: command = ( f"pip download --no-binary=:all: --no-deps -d {d} -i {package_index} " f"{package_name}==={package_version}") run_command(command) for f in os.listdir(d): if f.endswith(".tar.gz"): full_path = os.path.join(d, f) tar = tarfile.open(full_path, "r:gz") tar.extractall(os.path.join(d, "package")) from_directory = os.path.join(d, "package") break else: raise FileNotFoundError( f"No source distribution found for {package_name}==={package_version} " f"on {package_index}") out = _run_bandit(from_directory) else: out = _run_bandit(from_directory) out["package_name"] = package_name out["package_version"] = package_version out["bandit_version"] = bandit_version out["package_index"] = package_index print_command_result( click_ctx=click_ctx, result=out, analyzer=si_bandit_title, analyzer_version=si_bandit_version, output=output, duration=None, pretty=not no_pretty, )
def cli_extract_image(click_ctx, image, timeout=None, no_pretty=False, output=None, registry_credentials=None, no_tls_verify=False): """Extract installed packages from an image.""" result = extract_image(image, timeout, registry_credentials=registry_credentials, tls_verify=not no_tls_verify) print_command_result(click_ctx, result, analyzer=analyzer, analyzer_version=analyzer_version, output=output or '-', pretty=not no_pretty)
def python( click_ctx, requirements, index=None, python_version=3, exclude_packages=None, output=None, no_transitive=True, no_pretty=False, virtualenv=None, limited_output=False, ): """Manipulate with dependency requirements using PyPI.""" start_time = time.monotonic() requirements = [ requirement.strip() for requirement in requirements.split("\\n") if requirement ] if not requirements: _LOG.error("No requirements specified, exiting") sys.exit(1) result = resolve_python( requirements, index_urls=index.split(",") if index else ("https://pypi.org/simple", ), python_version=int(python_version), transitive=not no_transitive, exclude_packages=set( map(str.strip, (exclude_packages or "").split(","))), virtualenv=virtualenv, limited_output=limited_output, ) print_command_result( click_ctx, result, analyzer=analyzer_name, analyzer_version=analyzer_version, output=output or "-", duration=time.monotonic() - start_time, pretty=not no_pretty, )
def python( click_ctx, package_name: str, package_version: str, index_url: str = None, no_pretty: bool = True, output: str = None, ): """Fetch digests for packages in Python ecosystem.""" result = {} python_fetcher = PythonDigestsFetcher(index_url) result[index_url] = python_fetcher.fetch(package_name, package_version) print_command_result( click_ctx, result, analyzer=analyzer_name, analyzer_version=analyzer_version, output=output or "-", pretty=not no_pretty, )
def si_aggregator( click_ctx, output: Optional[str], package_name: str, package_version: Optional[str], package_index: Optional[str], si_bandit_results: str, si_cloc_results: str, aggregation_func: str, no_pretty: bool, ) -> None: """Run the cli for si-aggregator.""" agg_func = getattr(aggregators, aggregation_func) if agg_func is None: raise NotImplementedError( f"{aggregation_func} aggregation function not implemented yet.") with open(si_bandit_results, "r") as f: si_bandit_results = json.load(f) with open(si_cloc_results, "r") as f: si_cloc_results = json.load(f) out = agg_func(si_bandit_results=si_bandit_results, si_cloc_results=si_cloc_results) if out.get("error", False): out["metadata"] = _gen_metadata_on_err() out["bandit_results"] = si_bandit_results out["cloc_results"] = si_cloc_results print_command_result( click_ctx=click_ctx, result=out, analyzer=__title__, analyzer_version=__version__, output=output, duration=None, pretty=not no_pretty, )
def python( click_ctx, package_name: str, package_version: str, index_url: str = None, no_pretty: bool = True, output: str = None, dry_run: bool = False, ): """Fetch digests for packages in Python ecosystem.""" start_time = time.monotonic() python_fetcher = PythonDigestsFetcher(index_url) result = python_fetcher.fetch(package_name, package_version) print_command_result( click_ctx, result, analyzer=analyzer_name, analyzer_version=analyzer_version, output=output or "-", duration=time.monotonic() - start_time, pretty=not no_pretty, dry_run=dry_run, )
def advise( click_ctx, requirements, requirements_format=None, requirements_locked=None, recommendation_type=None, runtime_environment=None, output=None, no_pretty=False, files=False, count=None, limit=None, library_usage=None, limit_latest_versions=None, ): """Advise package and package versions in the given stack or on solely package only.""" _LOGGER.debug("Passed arguments: %s", locals()) limit = int(limit) if limit else None count = int(count) if count else None # A special value of -1 signalizes no limit/count, this is a workaround for Click's option parser. if count == -1: count = None if limit == -1: limit = None if limit_latest_versions == -1: limit_latest_versions = None if library_usage: if os.path.isfile(library_usage): try: library_usage = json.loads(Path(library_usage).read_text()) except Exception as exc: _LOGGER.error("Failed to load library usage file %r", library_usage) raise else: library_usage = json.loads(library_usage) runtime_environment = RuntimeEnvironment.load(runtime_environment) recommendation_type = RecommendationType.by_name(recommendation_type) requirements_format = PythonRecommendationOutput.by_name(requirements_format) result = { "error": None, "report": [], "stack_info": None, "advised_configuration": None, "pipeline_configuration": None, "parameters": { "runtime_environment": runtime_environment.to_dict(), "recommendation_type": recommendation_type.name, "library_usage": library_usage, "requirements_format": requirements_format.name, "limit": limit, "limit_latest_versions": limit_latest_versions, "count": count, "no_pretty": no_pretty, }, "input": None, } try: project = _instantiate_project( requirements, requirements_locked, files, runtime_environment ) result["input"] = project.to_dict() if runtime_environment: _LOGGER.info( "Runtime environment configuration:\n%s", json.dumps(runtime_environment.to_dict(), sort_keys=True, indent=2), ) else: _LOGGER.info("No runtime environment configuration supplied") if library_usage: _LOGGER.info( "Library usage:\n%s", json.dumps(library_usage, sort_keys=True, indent=2), ) else: _LOGGER.info("No library usage supplied") stack_info, advised_configuration, report, pipeline_configuration = Adviser.compute_on_project( project, recommendation_type=recommendation_type, library_usage=library_usage, count=count, limit=limit, limit_latest_versions=limit_latest_versions, ) except ThothAdviserException as exc: # TODO: we should extend exceptions so they are capable of storing more info. if isinstance(exc, InternalError): # Re-raise internal exceptions that shouldn't occur here. raise _LOGGER.exception("Error during computing recommendation: %s", str(exc)) result["error"] = True result["report"] = [([{"justification": f"{str(exc)}", "type": "ERROR"}], None)] except NoReleasesFound as exc: result["error"] = True result["report"] = [([{ "justification": f"{str(exc)}; analysis of the missing package will be " f"automatically scheduled by the system", "type": "ERROR" }], None)] except (SolverException, UnableLock) as exc: result["error"] = True result["report"] = [([{"justification": str(exc), "type": "ERROR"}], None)] else: # Convert report to a dict so its serialized. result["report"] = [ (justification, project.to_dict(), overall_score) for justification, project, overall_score in report ] # Report error if we did not find any recommendation to the user, the # stack_info carries information on why it hasn't been found. result["error"] = len(result["report"]) == 0 result["stack_info"] = stack_info if result["error"]: result["stack_info"].append({ "type": "ERROR", "justification": "Recommendation engine did not produce any stacks" }) result["advised_configuration"] = advised_configuration result["pipeline_configuration"] = pipeline_configuration print_command_result( click_ctx, result, analyzer=analyzer_name, analyzer_version=analyzer_version, output=output, pretty=not no_pretty, ) return int(result["error"] is True)
def dependency_monkey( click_ctx, requirements: str, stack_output: str, report_output: str, files: bool, seed: int = None, decision_type: str = None, dry_run: bool = False, context: str = None, no_pretty: bool = False, count: int = None, runtime_environment: dict = None, limit_latest_versions: int = None, library_usage: str = None, ): """Generate software stacks based on all valid resolutions that conform version ranges.""" # We cannot have these as ints in click because they are optional and we # cannot pass empty string as an int as env variable. seed = int(seed) if seed else None count = int(count) if count else None limit_latest_versions = ( int(limit_latest_versions) if limit_latest_versions else None ) # A special value of -1 signalizes no limit, this is a workaround for Click's option parser. if count == -1: count = None if limit_latest_versions == -1: limit_latest_versions = None runtime_environment = RuntimeEnvironment.load(runtime_environment) decision_type = DecisionType.by_name(decision_type) project = _instantiate_project( requirements, requirements_locked=None, files=files, runtime_environment=runtime_environment, ) if library_usage: if os.path.isfile(library_usage): try: library_usage = json.loads(Path(library_usage).read_text()) except Exception as exc: _LOGGER.error("Failed to load library usage file %r: %s", library_usage, str(exc)) raise else: library_usage = json.loads(library_usage) if runtime_environment: _LOGGER.info( "Runtime environment configuration:\n%s", json.dumps(runtime_environment.to_dict(), sort_keys=True, indent=2), ) else: _LOGGER.info("No runtime environment configuration supplied") if library_usage: _LOGGER.info( "Library usage:\n%s", json.dumps(library_usage, sort_keys=True, indent=2) ) else: _LOGGER.info("No library usage supplied") result = { "error": None, "parameters": { "requirements": project.pipfile.to_dict(), "runtime_environment": runtime_environment.to_dict(), "seed": seed, "decision_type": decision_type.name, "library_usage": library_usage, "context": deepcopy( context ), # We reuse context later, perform deepcopy to report the one on input. "stack_output": stack_output, "report_output": report_output, "files": files, "dry_run": dry_run, "no_pretty": no_pretty, "count": count, }, "input": None, "output": [], "computed": None, } try: report = run_dependency_monkey( project, stack_output, seed=seed, decision_type=decision_type, dry_run=dry_run, context=context, count=count, limit_latest_versions=limit_latest_versions, library_usage=library_usage, ) # Place report into result. result.update(report) except SolverException: _LOGGER.exception("An error occurred during dependency monkey run") result["error"] = traceback.format_exc() print_command_result( click_ctx, result, analyzer=analyzer_name, analyzer_version=analyzer_version, output=report_output, pretty=not no_pretty, ) return sys.exit(result["error"] is not None)