def has_delete_permission(self, request, obj=None):
publisher = get_publisher()
if obj is None:
return super().has_delete_permission(request, None)
return not (KeptBuild.keep(obj) or publisher.published(obj.record()))
def get_build_summary(now: dt.datetime,
machines: list[MachineInfo]) -> BuildSummary:
"""Update `context` with `latest_builds` and `build_recently`"""
latest_builds = []
built_recently = []
build_packages = {}
latest_published = set()
publisher = get_publisher()
for machine in machines:
if not (latest_build := machine.latest_build):
continue
if publisher.published(latest_build):
latest_published.add(latest_build)
record = publisher.record(latest_build)
if not record.completed:
continue
latest_builds.append(latest_build)
build_id = latest_build.id
try:
build_packages[build_id] = [
i.cpv for i in publisher.storage.get_metadata(
latest_build).packages.built
]
except LookupError:
build_packages[build_id] = []
if lapsed(record.completed, now) < 86400:
built_recently.append(latest_build)
def handle(self, *args: Any, **options: Any) -> None:
errors = 0
publisher = get_publisher()
# Pass 1: check build content
records = publisher.records.query(completed__isnull=False)
for record in records:
missing: list[Path] = []
for content in Content:
path = publisher.storage.get_path(record, content)
if not path.exists():
missing.append(path)
if missing:
self.stderr.write(f"Path missing for {record}: {missing}")
errors += 1
# Pass 2: check for orphans
for content in Content:
directory = publisher.storage.path / content.value
for path in directory.glob("*.*"):
build = Build(path.name)
try:
publisher.records.get(build)
except RecordNotFound:
self.stderr.write(f"Record missing for {path}")
errors += 1
if errors:
self.stderr.write("Errors were encountered.")
raise CommandError(errors)
def record(self) -> BuildRecord:
publisher = get_publisher()
if self._record is None:
self._record = publisher.record(self.build)
return self._record
def pull_build(build_id: str) -> None:
"""Pull the build into storage"""
publisher = get_publisher()
build = Build(build_id)
try:
publisher.pull(build)
except Exception as error:
logger.exception("Failed to pull build %s", build)
# If this is an error due to 404 response don't retry
if isinstance(error, requests.exceptions.HTTPError):
response = getattr(error, "response", None)
if response and response.status_code == 404:
publisher.records.delete(build)
raise
if isinstance(error, PULL_RETRYABLE_EXCEPTIONS):
pull_build.retry(exc=error)
return
publisher.records.delete(build)
raise
if Settings.from_environ().ENABLE_PURGE:
purge_build.delay(build.machine)
def package_metadata(record: BuildRecord, context: DashboardContext) -> None:
"""Update `context` with `package_count` and `total_package_size`"""
metadata: Optional[GBPMetadata]
publisher = get_publisher()
try:
metadata = publisher.storage.get_metadata(record)
except LookupError:
metadata = None
if metadata and record.completed:
context["package_count"] += metadata.packages.total
context["total_package_size"][record.machine] += metadata.packages.size
if record.submitted and lapsed(record.submitted,
context["now"]) < 86400:
for package in metadata.packages.built:
if (package.cpv in context["recent_packages"]
or len(context["recent_packages"]) < 12):
context["recent_packages"][package.cpv].add(record.machine)
else:
packages = get_packages(record)
context["package_count"] += len(packages)
context["total_package_size"][record.machine] += sum(i.size
for i in packages)
def resolve_query_builds(_obj: Any, _info: GraphQLResolveInfo,
machine: str) -> list[BuildType]:
publisher = get_publisher()
records = publisher.records.query(machine=machine, completed__isnull=False)
return [BuildType(record) for record in records]
def packages_built(self) -> list[Package] | None:
publisher = get_publisher()
try:
gbp_metadata = publisher.storage.get_metadata(self.build)
except LookupError as error:
raise GraphQLError("Packages built unknown") from error
return gbp_metadata.packages.built
def delete_build(build_id: str) -> None:
"""Delete the given build from the db"""
publisher = get_publisher()
build = Build(build_id)
logger.info("Deleting build: %s", build)
publisher.delete(build)
logger.info("Deleted build: %s", build)
def resolve_mutation_publish(_obj: Any, _info: GraphQLResolveInfo,
id: str) -> MachineInfo:
publisher = get_publisher()
build = Build(id)
if publisher.pulled(build):
publisher.publish(build)
else:
publish_build.delay(build.id)
return MachineInfo(build.machine)
def resolve_mutation_releasebuild(_obj: Any, _info: GraphQLResolveInfo,
id: str) -> Optional[BuildType]:
publisher = get_publisher()
build = Build(id)
if not publisher.records.exists(build):
return None
record = publisher.record(build)
publisher.records.save(record, keep=False)
return BuildType(record)
def packages(self) -> list[str] | None:
publisher = get_publisher()
if not publisher.pulled(self.build):
return None
try:
return [
package.cpv for package in publisher.get_packages(self.build)
]
except LookupError:
return None
def publish_build(build_id: str) -> bool:
"""Publish the build"""
publisher = get_publisher()
try:
pull_build.apply((build_id, ), throw=True)
except PUBLISH_FATAL_EXCEPTIONS:
logger.error("Build %s failed to pull. Not publishing", f"{build_id}")
return False
publisher.publish(Build(build_id))
return True
def response_change(self, request, obj):
publisher = get_publisher()
if "_publish" in request.POST:
publisher.publish(obj.record())
if "_keep" in request.POST:
try:
kept_build = KeptBuild.objects.get(build_model=obj)
kept_build.delete()
except KeptBuild.DoesNotExist:
KeptBuild.objects.create(build_model=obj)
return super().response_change(request, obj)
def resolve_mutation_createnote(
_obj: Any,
_info: GraphQLResolveInfo,
id: str,
note: Optional[str] = None) -> Optional[BuildType]:
publisher = get_publisher()
build = Build(id)
if not publisher.records.exists(build):
return None
record = publisher.record(build)
publisher.records.save(record, note=note)
return BuildType(record)
def get_packages(build: Build) -> list[Package]:
"""Return a list of packages from a build by looking up the index.
This call may be cached for performance.
"""
publisher = get_publisher()
cache_key = f"packages-{build}"
try:
packages: list[Package] = cache.get_or_set(
cache_key, lambda: publisher.get_packages(build), timeout=None)
except LookupError:
return []
return packages
def buncha_builds(
machines: list[str],
end_date,
num_days: int,
per_day: int,
) -> defaultdict[str, list[Build]]:
publisher = get_publisher()
buildmap = defaultdict(list)
for i in reversed(range(num_days)):
day = end_date - dt.timedelta(days=i)
for machine in machines:
builds = BuildFactory.create_batch(per_day, machine=machine)
for build in builds:
publisher.records.save(publisher.record(build), submitted=day)
buildmap[machine].extend(builds)
return buildmap
def resolve_query_diff(_obj: Any, _info: GraphQLResolveInfo, left: str,
right: str) -> Optional[Object]:
publisher = get_publisher()
left_build = Build(left)
if not publisher.records.exists(left_build):
return None
right_build = Build(right)
if not publisher.records.exists(right_build):
return None
items = publisher.diff_binpkgs(left_build, right_build)
return {
"left": BuildType(left_build),
"right": BuildType(right_build),
"items": [*items],
}
def resolve_query_latest(_obj: Any, _info: GraphQLResolveInfo,
machine: str) -> Optional[BuildType]:
publisher = get_publisher()
record = publisher.latest_build(machine, completed=True)
return None if record is None else BuildType(record)
def purge_build(machine: str) -> None:
"""Purge old builds for machine"""
publisher = get_publisher()
publisher.purge(machine)
def dashboard(request: HttpRequest) -> HttpResponse:
"""Dashboard view"""
publisher = get_publisher()
now = timezone.localtime()
current_timezone = timezone.get_current_timezone()
bot_days: list[dt.date] = [
now.date() - dt.timedelta(days=days) for days in range(6, -1, -1)
]
builds_over_time: dict[dt.date, defaultdict[str, int]] = {
day: defaultdict(int)
for day in bot_days
}
records = publisher.records.query()
machines = publisher.machines()
machines.sort(key=lambda m: m.build_count, reverse=True)
color_start = Color(*GBP_SETTINGS.get("COLOR_START", (80, 69, 117)))
color_end = Color(*GBP_SETTINGS.get("COLOR_END", (221, 218, 236)))
context: DashboardContext = {
"bot_days": [datetime.strftime("%A") for datetime in bot_days],
"build_count":
0,
"builds_to_do": [],
"build_packages": {},
"builds_over_time": [],
"built_recently": [],
"latest_builds": [],
"latest_published":
set(),
"machine_colors": [
str(color)
for color in gradient(color_start, color_end, len(machines))
],
"machine_dist": [machine.build_count for machine in machines],
"machines": [machine.machine for machine in machines],
"now":
now,
"package_count":
0,
"recent_packages":
defaultdict(set),
"total_package_size":
defaultdict(int),
"unpublished_builds_count":
0,
}
for record in records:
context["build_count"] += 1
if not record.completed:
context["builds_to_do"].append(record)
package_metadata(record, context)
if record.submitted is None:
continue
day_submitted = record.submitted.astimezone(current_timezone).date()
if day_submitted >= bot_days[0]:
builds_over_time[day_submitted][record.machine] += 1
(
context["latest_builds"],
context["built_recently"],
context["build_packages"],
context["latest_published"],
) = astuple(get_build_summary(now, machines))
context["unpublished_builds_count"] = len([
build for build in context["latest_builds"]
if not publisher.published(build)
])
context["builds_over_time"] = bot_to_list(builds_over_time, machines,
bot_days)
# https://stackoverflow.com/questions/4764110/django-template-cant-loop-defaultdict
context["recent_packages"].default_factory = None
return render(request, "gentoo_build_publisher/dashboard.html", context)
def resolve_query_searchnotes(_obj: Any, _info: GraphQLResolveInfo,
machine: str, key: str) -> list[BuildType]:
publisher = get_publisher()
return [BuildType(i) for i in publisher.search_notes(machine, key)]
def resolve_query_working(_obj: Any,
_info: GraphQLResolveInfo) -> list[BuildType]:
publisher = get_publisher()
records = publisher.records.query(completed=None)
return [BuildType(record) for record in records]
def published(self, obj: BuildModel) -> bool:
"""Return the admin published field"""
publisher = get_publisher()
return publisher.published(obj.record())
def resolve_query_machines(_obj: Any,
_info: GraphQLResolveInfo) -> list[MachineInfo]:
publisher = get_publisher()
return publisher.machines()
def resolve_query_build(_obj: Any, _info: GraphQLResolveInfo,
id: str) -> Optional[BuildType]:
publisher = get_publisher()
build = Build(id)
return None if not publisher.records.exists(build) else BuildType(build)
def pulled(self) -> bool:
publisher = get_publisher()
return publisher.pulled(self.build)
def resolve_mutation_schedule_build(_obj: Any, _info: GraphQLResolveInfo,
machine: str) -> str:
publisher = get_publisher()
return publisher.schedule_build(machine)
