async def learn_using_solver(
openshift: OpenShift,
graph: GraphDatabase,
is_present: bool,
package_name: str,
index_url: str,
package_version: str,
solver: Optional[str] = None,
) -> int:
"""Learn using solver about Package Version Index dependencies."""
if not is_present:
# Package never seen (schedule all solver workflows to collect all knowledge for Thoth)
are_solvers_scheduled = await _schedule_all_solvers(
openshift=openshift,
package_name=package_name,
package_version=package_version,
indexes=[index_url])
return are_solvers_scheduled
# Select solvers
if not solver:
solvers: List[str] = openshift.get_solver_names()
else:
solvers = [solver]
dependency_indexes = graph.get_python_package_index_urls_all(enabled=True)
# Check for which solver has not been solved and schedule solver workflow
are_solvers_scheduled = 0
for solver_name in solvers:
is_solved = graph.python_package_version_exists(
package_name=package_name,
package_version=package_version,
index_url=index_url,
solver_name=solver_name)
if not is_solved:
is_solver_scheduled = _schedule_solver(
openshift=openshift,
package_name=package_name,
package_version=package_version,
indexes=[index_url],
dependency_indexes=dependency_indexes,
solver_name=solver_name,
)
are_solvers_scheduled += is_solver_scheduled
return are_solvers_scheduled
def do_solve(package_name: str, package_version: str) -> List[Dict[str, Any]]:
"""Obtain dependent packages for the given package, respect registered solvers in Thoth."""
graph = GraphDatabase()
graph.connect()
openshift = OpenShift()
result = []
for solver_name in openshift.get_solver_names():
_LOGGER.info(
"Obtaining dependencies for environment used by solver %r",
solver_name)
solver_info = openshift.parse_python_solver_name(solver_name)
start_offset = 0
while True:
dependents = graph.get_python_package_version_dependents_all(
package_name=package_name,
os_name=solver_info["os_name"],
os_version=solver_info["os_version"],
python_version=solver_info["python_version"],
start_offset=start_offset,
count=_QUERY_COUNT,
)
for dependent in dependents:
if (dependent["version_range"]
and dependent["version_range"] != "*"
and package_version
not in PackageVersion.parse_semantic_version(
dependent["version_range"])):
_LOGGER.info(
"Package %r in version %r from %r ignored, not matching version specifier %r in environment %r",
dependent["package_name"],
dependent["package_version"],
dependent["index_url"],
package_version,
solver_name,
)
continue
_LOGGER.info(
"Found dependent %r in version %r from %r in environment %r",
dependent["package_name"],
dependent["package_version"],
dependent["index_url"],
solver_name,
)
result.append({
"os_name": solver_info["os_name"],
"os_version": solver_info["os_version"],
"python_version": solver_info["python_version"],
"solver_name": solver_name,
**dependent,
})
if len(dependents) < int(_QUERY_COUNT):
break
# Adjust for the next round.
start_offset += int(_QUERY_COUNT)
return result
def graph_refresh() -> None:
"""Schedule refresh for packages that are not yet analyzed by solver."""
graph = GraphDatabase()
graph.connect()
indexes = list(graph.get_python_package_index_urls())
openshift = OpenShift()
packages = []
# Iterate over all registered solvers and gather packages which were not solved by them.
for solver_name in openshift.get_solver_names():
_LOGGER.info("Checking unsolved packages for solver %r", solver_name)
for package, versions in graph.retrieve_unsolved_pypi_packages(
solver_name).items():
for version in versions:
_LOGGER.info(
f"Adding new package {package} in version {version}")
_METRIC_PACKAGES_ADDED.inc()
packages.append((f"{package}=={version}", solver_name))
if not packages:
_LOGGER.info("No unsolved packages found")
return
count = 0
for package, solver in packages:
try:
analysis_id = openshift.schedule_solver(
solver=solver,
debug=_LOG_SOLVER,
packages=package,
indexes=indexes,
output=_SOLVER_OUTPUT,
subgraph_check_api=_SUBGRAPH_CHECK_API,
)
except Exception as ecx:
# If we get some errors from OpenShift master - do not retry. Rather schedule the remaining
# ones and try to schedule the given package in the next run.
_LOGGER.exception(
f"Failed to schedule new solver to solve package {package}, the graph refresh job will not "
"fail but will try to reschedule this in next run")
_METRIC_SOLVERS_UNSCHEDULED.labels(solver).inc()
continue
_LOGGER.info(
"Scheduled solver %r for package %r, analysis is %r",
solver,
package,
analysis_id,
)
_METRIC_SOLVERS_SCHEDULED.labels(solver).inc()
count += 1
if _THOTH_GRAPH_REFRESH_EAGER_STOP and count >= _THOTH_GRAPH_REFRESH_EAGER_STOP:
_LOGGER.info(
"Eager stop of scheduling new solver runs for unsolved package versions, packages scheduled: %d",
count,
)
return
