def has_delete_permission(self, request, obj=None): publisher = get_publisher() if obj is None: return super().has_delete_permission(request, None) return not (KeptBuild.keep(obj) or publisher.published(obj.record()))
def get_build_summary(now: dt.datetime, machines: list[MachineInfo]) -> BuildSummary: """Update `context` with `latest_builds` and `build_recently`""" latest_builds = [] built_recently = [] build_packages = {} latest_published = set() publisher = get_publisher() for machine in machines: if not (latest_build := machine.latest_build): continue if publisher.published(latest_build): latest_published.add(latest_build) record = publisher.record(latest_build) if not record.completed: continue latest_builds.append(latest_build) build_id = latest_build.id try: build_packages[build_id] = [ i.cpv for i in publisher.storage.get_metadata( latest_build).packages.built ] except LookupError: build_packages[build_id] = [] if lapsed(record.completed, now) < 86400: built_recently.append(latest_build)
def handle(self, *args: Any, **options: Any) -> None: errors = 0 publisher = get_publisher() # Pass 1: check build content records = publisher.records.query(completed__isnull=False) for record in records: missing: list[Path] = [] for content in Content: path = publisher.storage.get_path(record, content) if not path.exists(): missing.append(path) if missing: self.stderr.write(f"Path missing for {record}: {missing}") errors += 1 # Pass 2: check for orphans for content in Content: directory = publisher.storage.path / content.value for path in directory.glob("*.*"): build = Build(path.name) try: publisher.records.get(build) except RecordNotFound: self.stderr.write(f"Record missing for {path}") errors += 1 if errors: self.stderr.write("Errors were encountered.") raise CommandError(errors)
def record(self) -> BuildRecord: publisher = get_publisher() if self._record is None: self._record = publisher.record(self.build) return self._record
def pull_build(build_id: str) -> None: """Pull the build into storage""" publisher = get_publisher() build = Build(build_id) try: publisher.pull(build) except Exception as error: logger.exception("Failed to pull build %s", build) # If this is an error due to 404 response don't retry if isinstance(error, requests.exceptions.HTTPError): response = getattr(error, "response", None) if response and response.status_code == 404: publisher.records.delete(build) raise if isinstance(error, PULL_RETRYABLE_EXCEPTIONS): pull_build.retry(exc=error) return publisher.records.delete(build) raise if Settings.from_environ().ENABLE_PURGE: purge_build.delay(build.machine)
def package_metadata(record: BuildRecord, context: DashboardContext) -> None: """Update `context` with `package_count` and `total_package_size`""" metadata: Optional[GBPMetadata] publisher = get_publisher() try: metadata = publisher.storage.get_metadata(record) except LookupError: metadata = None if metadata and record.completed: context["package_count"] += metadata.packages.total context["total_package_size"][record.machine] += metadata.packages.size if record.submitted and lapsed(record.submitted, context["now"]) < 86400: for package in metadata.packages.built: if (package.cpv in context["recent_packages"] or len(context["recent_packages"]) < 12): context["recent_packages"][package.cpv].add(record.machine) else: packages = get_packages(record) context["package_count"] += len(packages) context["total_package_size"][record.machine] += sum(i.size for i in packages)
def resolve_query_builds(_obj: Any, _info: GraphQLResolveInfo, machine: str) -> list[BuildType]: publisher = get_publisher() records = publisher.records.query(machine=machine, completed__isnull=False) return [BuildType(record) for record in records]
def packages_built(self) -> list[Package] | None: publisher = get_publisher() try: gbp_metadata = publisher.storage.get_metadata(self.build) except LookupError as error: raise GraphQLError("Packages built unknown") from error return gbp_metadata.packages.built
def delete_build(build_id: str) -> None: """Delete the given build from the db""" publisher = get_publisher() build = Build(build_id) logger.info("Deleting build: %s", build) publisher.delete(build) logger.info("Deleted build: %s", build)
def resolve_mutation_publish(_obj: Any, _info: GraphQLResolveInfo, id: str) -> MachineInfo: publisher = get_publisher() build = Build(id) if publisher.pulled(build): publisher.publish(build) else: publish_build.delay(build.id) return MachineInfo(build.machine)
def resolve_mutation_releasebuild(_obj: Any, _info: GraphQLResolveInfo, id: str) -> Optional[BuildType]: publisher = get_publisher() build = Build(id) if not publisher.records.exists(build): return None record = publisher.record(build) publisher.records.save(record, keep=False) return BuildType(record)
def packages(self) -> list[str] | None: publisher = get_publisher() if not publisher.pulled(self.build): return None try: return [ package.cpv for package in publisher.get_packages(self.build) ] except LookupError: return None
def publish_build(build_id: str) -> bool: """Publish the build""" publisher = get_publisher() try: pull_build.apply((build_id, ), throw=True) except PUBLISH_FATAL_EXCEPTIONS: logger.error("Build %s failed to pull. Not publishing", f"{build_id}") return False publisher.publish(Build(build_id)) return True
def response_change(self, request, obj): publisher = get_publisher() if "_publish" in request.POST: publisher.publish(obj.record()) if "_keep" in request.POST: try: kept_build = KeptBuild.objects.get(build_model=obj) kept_build.delete() except KeptBuild.DoesNotExist: KeptBuild.objects.create(build_model=obj) return super().response_change(request, obj)
def resolve_mutation_createnote( _obj: Any, _info: GraphQLResolveInfo, id: str, note: Optional[str] = None) -> Optional[BuildType]: publisher = get_publisher() build = Build(id) if not publisher.records.exists(build): return None record = publisher.record(build) publisher.records.save(record, note=note) return BuildType(record)
def get_packages(build: Build) -> list[Package]: """Return a list of packages from a build by looking up the index. This call may be cached for performance. """ publisher = get_publisher() cache_key = f"packages-{build}" try: packages: list[Package] = cache.get_or_set( cache_key, lambda: publisher.get_packages(build), timeout=None) except LookupError: return [] return packages
def buncha_builds( machines: list[str], end_date, num_days: int, per_day: int, ) -> defaultdict[str, list[Build]]: publisher = get_publisher() buildmap = defaultdict(list) for i in reversed(range(num_days)): day = end_date - dt.timedelta(days=i) for machine in machines: builds = BuildFactory.create_batch(per_day, machine=machine) for build in builds: publisher.records.save(publisher.record(build), submitted=day) buildmap[machine].extend(builds) return buildmap
def resolve_query_diff(_obj: Any, _info: GraphQLResolveInfo, left: str, right: str) -> Optional[Object]: publisher = get_publisher() left_build = Build(left) if not publisher.records.exists(left_build): return None right_build = Build(right) if not publisher.records.exists(right_build): return None items = publisher.diff_binpkgs(left_build, right_build) return { "left": BuildType(left_build), "right": BuildType(right_build), "items": [*items], }
def resolve_query_latest(_obj: Any, _info: GraphQLResolveInfo, machine: str) -> Optional[BuildType]: publisher = get_publisher() record = publisher.latest_build(machine, completed=True) return None if record is None else BuildType(record)
def purge_build(machine: str) -> None: """Purge old builds for machine""" publisher = get_publisher() publisher.purge(machine)
def dashboard(request: HttpRequest) -> HttpResponse: """Dashboard view""" publisher = get_publisher() now = timezone.localtime() current_timezone = timezone.get_current_timezone() bot_days: list[dt.date] = [ now.date() - dt.timedelta(days=days) for days in range(6, -1, -1) ] builds_over_time: dict[dt.date, defaultdict[str, int]] = { day: defaultdict(int) for day in bot_days } records = publisher.records.query() machines = publisher.machines() machines.sort(key=lambda m: m.build_count, reverse=True) color_start = Color(*GBP_SETTINGS.get("COLOR_START", (80, 69, 117))) color_end = Color(*GBP_SETTINGS.get("COLOR_END", (221, 218, 236))) context: DashboardContext = { "bot_days": [datetime.strftime("%A") for datetime in bot_days], "build_count": 0, "builds_to_do": [], "build_packages": {}, "builds_over_time": [], "built_recently": [], "latest_builds": [], "latest_published": set(), "machine_colors": [ str(color) for color in gradient(color_start, color_end, len(machines)) ], "machine_dist": [machine.build_count for machine in machines], "machines": [machine.machine for machine in machines], "now": now, "package_count": 0, "recent_packages": defaultdict(set), "total_package_size": defaultdict(int), "unpublished_builds_count": 0, } for record in records: context["build_count"] += 1 if not record.completed: context["builds_to_do"].append(record) package_metadata(record, context) if record.submitted is None: continue day_submitted = record.submitted.astimezone(current_timezone).date() if day_submitted >= bot_days[0]: builds_over_time[day_submitted][record.machine] += 1 ( context["latest_builds"], context["built_recently"], context["build_packages"], context["latest_published"], ) = astuple(get_build_summary(now, machines)) context["unpublished_builds_count"] = len([ build for build in context["latest_builds"] if not publisher.published(build) ]) context["builds_over_time"] = bot_to_list(builds_over_time, machines, bot_days) # https://stackoverflow.com/questions/4764110/django-template-cant-loop-defaultdict context["recent_packages"].default_factory = None return render(request, "gentoo_build_publisher/dashboard.html", context)
def resolve_query_searchnotes(_obj: Any, _info: GraphQLResolveInfo, machine: str, key: str) -> list[BuildType]: publisher = get_publisher() return [BuildType(i) for i in publisher.search_notes(machine, key)]
def resolve_query_working(_obj: Any, _info: GraphQLResolveInfo) -> list[BuildType]: publisher = get_publisher() records = publisher.records.query(completed=None) return [BuildType(record) for record in records]
def published(self, obj: BuildModel) -> bool: """Return the admin published field""" publisher = get_publisher() return publisher.published(obj.record())
def resolve_query_machines(_obj: Any, _info: GraphQLResolveInfo) -> list[MachineInfo]: publisher = get_publisher() return publisher.machines()
def resolve_query_build(_obj: Any, _info: GraphQLResolveInfo, id: str) -> Optional[BuildType]: publisher = get_publisher() build = Build(id) return None if not publisher.records.exists(build) else BuildType(build)
def pulled(self) -> bool: publisher = get_publisher() return publisher.pulled(self.build)
def resolve_mutation_schedule_build(_obj: Any, _info: GraphQLResolveInfo, machine: str) -> str: publisher = get_publisher() return publisher.schedule_build(machine)