async def _check_package_availability(
package: Tuple[str, str, str],
sources: Dict[str, Any],
removed_packages: set,
missing_package: MissingPackageMessage,
) -> bool:
src = sources[package[1]]
if not package[0] in src["packages"]:
removed_packages.add((package[1], package[0]))
try:
await missing_package.publish_to_topic(missing_package.MessageContents(
index_url=package[1],
package_name=package[0],
component_name=COMPONENT_NAME,
service_version=__package_update_version__,
))
_LOGGER.info("%r no longer provides %r", package[1], package[0])
return False
except Exception as e:
_LOGGER.exception("Failed to publish with the following error message: %r", e)
return True
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Logic for handling missing_package message."""
from typing import Dict, Any
from .metrics_missing_package import missing_package_exceptions
from .metrics_missing_package import missing_package_success
from .metrics_missing_package import missing_package_in_progress
from ..common import register_handler
# from ..metrics import scheduled_workflows
from prometheus_async.aio import track_inprogress, count_exceptions
from thoth.messaging import MissingPackageMessage
@count_exceptions(missing_package_exceptions)
@track_inprogress(missing_package_in_progress)
@register_handler(MissingPackageMessage().topic_name, ["v1"])
async def parse_missing_package(package: Dict[str, Any], **kwargs):
"""Process a missing package message from package-update producer."""
# TODO call kebechet with the missing package name, which would call kebechet on the individual
# repositories with a different subcommand that would just create an issue.
# scheduled_workflows.labels(
# message_type=MissingPackageMessage.base_name, workflow_type="kebechet-administrator"
# ).inc()
missing_package_success.inc()
import logging
import faust
import os
import ssl
from urllib.parse import urlparse
from aiohttp import web
init_logging()
_LOGGER = logging.getLogger(__name__)
_LOGGER.info("Thoth Package Update consumer v%s", __service_version__)
app = MessageBase().app
hash_mismatch_topic = HashMismatchMessage().topic
missing_package_topic = MissingPackageMessage().topic
missing_version_topic = MissingVersionMessage().topic
@app.page("/metrics")
async def get_metrics(self, request):
"""Serve the metrics from the consumer registry."""
return web.Response(text=generate_latest().decode("utf-8"))
@app.page("/_health")
async def get_health(self, request):
"""Serve a readiness/liveness probe endpoint."""
data = {"status": "ready", "version": __service_version__}
return web.json_response(data)
async def main():
"""Run package-update."""
graph = GraphDatabase()
graph.connect()
removed_pkgs = set()
hash_mismatch = HashMismatchMessage()
missing_package = MissingPackageMessage()
missing_version = MissingVersionMessage()
indexes = {x["url"] for x in graph.get_python_package_index_all()}
sources = dict()
async_tasks = []
for i in indexes:
async_tasks.append(_gather_index_info(i, sources))
await asyncio.gather(*async_tasks, return_exceptions=True)
async_tasks.clear()
all_pkgs = graph.get_python_packages_all(count=None, distinct=True)
_LOGGER.info("Checking availability of %r package(s)", len(all_pkgs))
for pkg in all_pkgs:
async_tasks.append(
_check_package_availability(
package=pkg,
sources=sources,
removed_packages=removed_pkgs,
missing_package=missing_package,
))
await asyncio.gather(*async_tasks, return_exceptions=True)
async_tasks.clear()
all_pkg_vers = graph.get_python_package_versions_all(count=None,
distinct=True)
all_pkg_names = {(i[0], i[2]) for i in all_pkg_vers}
versions = dict.fromkeys(all_pkg_names)
for i in all_pkg_names:
async_tasks.append(
_get_all_versions(package_name=i[0],
source=i[1],
sources=sources,
accumulator=versions))
await asyncio.gather(*async_tasks, return_exceptions=True)
async_tasks.clear()
_LOGGER.info("Checking integrity of %r package(s)", len(all_pkg_vers))
for pkg_ver in all_pkg_vers:
# Skip because we have already marked the entire package as missing
if (pkg_ver[2], pkg_ver[0]) in removed_pkgs or versions[
(pkg_ver[0], pkg_ver[2])] is None: # in case of 404
continue
src = sources[pkg_ver[2]]["source"]
async_tasks.append(
_check_hashes(
package_version=pkg_ver,
package_versions=versions[(pkg_ver[0], pkg_ver[2])],
source=src,
removed_packages=removed_pkgs,
missing_version=missing_version,
hash_mismatch=hash_mismatch,
graph=graph,
))
await asyncio.gather(*async_tasks, return_exceptions=True)
async_tasks.clear()