def get():
reps = []
github_status = reporters.GitHubStatusPush(
token=os.environ.get("BUILDBOT_ACCESS_TOKEN"),
context=get_context,
startDescription='Build started.',
endDescription='Build done.',
verbose=True)
reps.append(github_status)
return reps
def get_github_status():
from buildbot.process.properties import Interpolate
from buildbot.plugins import reporters
import private
return [
reporters.GitHubStatusPush(
token=private.github_status_token,
startDescription=Interpolate(
"Build #%(prop:buildnumber)s started."),
endDescription=Interpolate("Build #%(prop:buildnumber)s done."),
context=Interpolate("%(prop:buildername)s"),
verbose=True)
]
def setupReporters(self, _reporters, spawner_name, try_name, codebases):
name = "GitHubStatusPush"
reportersByName = dict([(r.name, r) for r in _reporters])
if name not in reportersByName and self.github_token:
token = self.github_token
if token.startswith("file:"):
with open(token.split(":", 2)[1]) as f:
token = f.read().strip()
if token.startswith("env:"):
token = os.environ[token.split(":", 2)[1]]
if not self.reporter_context:
self.reporter_context = self.default_reporter_context
_reporters.append(
reporters.GitHubStatusPush(token, context=util.Interpolate(self.reporter_context),
verbose=True))
from zorg.buildbot.util.InformativeMailNotifier import LLVMInformativeMailNotifier
# Should be a single e-mail address
status_email = str(config.options.get('Master Options',
'status_email')).split(',')
all = [
# Note: reporters.GitHubStatusPush requires txrequests package to allow
# interaction with GitHub REST API.
reporters.GitHubStatusPush(
str(config.options.get('GitHub Status', 'token')),
context=Interpolate("%(prop:buildername)s"),
verbose=
True, # TODO: Turn off the verbosity once this is working reliably.
builders=[
"llvm-clang-x86_64-expensive-checks-ubuntu",
"llvm-clang-x86_64-win-fast",
"clang-x86_64-debian-fast",
"llvm-clang-x86_64-expensive-checks-debian",
]),
reporters.IRC(
useColors=False,
host=str(config.options.get('IRC', 'host')),
nick=str(config.options.get('IRC', 'nick')),
channels=str(config.options.get('IRC', 'channels')).split(','),
#authz=... # TODO: Consider allowing "harmful" operations to authorizes users.
useRevisions=False, # FIXME: There is a bug in the buildbot
showBlameList=True,
notify_events=str(config.options.get('IRC',
'notify_events')).split(','),
for builder_name, properties in config.FLOW.get_prepared_builders().items():
if properties.get('add_triggerable_sheduler', True):
c["schedulers"].append(
schedulers.Triggerable(name=builder_name,
builderNames=[builder_name]))
c["builders"].append(
util.BuilderConfig(name=builder_name,
workernames=get_workers(properties.get("worker")),
factory=properties['factory']))
# Push status of build to the Github
c["services"] = [
reporters.GitHubStatusPush(
token=config.GITHUB_TOKEN,
context=util.Interpolate("buildbot/%(prop:buildername)s"),
startDescription="Started",
endDescription="Done",
verbose=True)
]
# Will be useful for implementing build notifications in the future
# reporters.GitHubCommentPush(token=config.GITHUB_TOKEN,
# startDescription="Started (comment)",
# endDescription="Done (comment)",
# verbose=True,
# debug=True)]
# Get changes
c["change_source"] = []
class MediasdkChangeChecker(bb.utils.ChangeChecker):
def build_config() -> dict[str, Any]:
c = {}
c["buildbotNetUsageData"] = None
# configure a janitor which will delete all logs older than one month, and will run on sundays at noon
c['configurators'] = [util.JanitorConfigurator(
logHorizon=timedelta(weeks=4),
hour=12,
dayOfWeek=6
)]
c["schedulers"] = [
# build all pushes to master
schedulers.SingleBranchScheduler(
name="master",
change_filter=util.ChangeFilter(branch="master"),
builderNames=["nix-eval"],
),
# build all pull requests
schedulers.SingleBranchScheduler(
name="prs",
change_filter=util.ChangeFilter(category="pull"),
builderNames=["nix-eval"],
),
# this is triggered from `nix-eval`
schedulers.Triggerable(
name="nix-build",
builderNames=["nix-build"],
),
# allow to manually trigger a nix-build
schedulers.ForceScheduler(name="force", builderNames=["nix-eval"]),
# allow to manually update flakes
schedulers.ForceScheduler(
name="update-flake",
builderNames=["nix-update-flake"],
buttonName="Update flakes",
),
# updates flakes once a weeek
schedulers.NightlyTriggerable(
name="update-flake-weekly",
builderNames=["nix-update-flake"],
hour=3,
minute=0,
dayOfWeek=6,
),
]
github_api_token = read_secret_file("github-token")
c["services"] = [
reporters.GitHubStatusPush(
token=github_api_token,
# Since we dynamically create build steps,
# we use `virtual_builder_name` in the webinterface
# so that we distinguish what has beeing build
context=Interpolate("buildbot/%(prop:virtual_builder_name)s"),
),
# Notify on irc
NotifyFailedBuilds("irc://buildbot|[email protected]:6667/#xxx"),
]
# Shape of this file:
# [ { "name": "<worker-name>", "pass": "******", "cores": "<cpu-cores>" } ]
worker_config = json.loads(read_secret_file("github-workers"))
credentials = os.environ.get("CREDENTIALS_DIRECTORY", ".")
enable_cachix = os.path.isfile(os.path.join(credentials, "cachix-token"))
systemd_secrets = secrets.SecretInAFile(dirname=credentials)
c["secretsProviders"] = [systemd_secrets]
c["workers"] = []
worker_names = []
for item in worker_config:
cores = item.get("cores", 0)
for i in range(cores):
worker_name = f"{item['name']}-{i}"
c["workers"].append(worker.Worker(worker_name, item["pass"]))
worker_names.append(worker_name)
c["builders"] = [
# Since all workers run on the same machine, we only assign one of them to do the evaluation.
# This should prevent exessive memory usage.
nix_eval_config([worker_names[0]], github_token_secret="github-token"),
nix_build_config(worker_names, enable_cachix),
nix_update_flake_config(
worker_names,
"TUM-DSE/doctor-cluster-config",
github_token_secret="github-token",
),
]
c["www"] = {
"port": int(os.environ.get("PORT", "1810")),
"auth": util.GitHubAuth(
os.environ.get("GITHUB_OAUTH_ID"), read_secret_file("github-oauth-secret")
),
"authz": util.Authz(
roleMatchers=[
util.RolesFromGroups(groupPrefix="") # so we can match on TUM-DSE
],
allowRules=[
util.AnyEndpointMatcher(role="TUM-DSE", defaultDeny=False),
util.AnyControlEndpointMatcher(role="TUM-DSE"),
],
),
"plugins": dict(waterfall_view={}, console_view={}, grid_view={}),
"change_hook_dialects": dict(
github={
"secret": read_secret_file("github-webhook-secret"),
"strict": True,
"token": github_api_token,
"github_property_whitelist": "*",
}
),
}
c["db"] = {"db_url": os.environ.get("DB_URL", "sqlite:///state.sqlite")}
c["protocols"] = {"pb": {"port": "tcp:9989:interface=\\:\\:"}}
c["buildbotURL"] = "https://buildbot.dse.in.tum.de/"
return c
def getReporters():
# Should be a single e-mail address
status_email = str(config.options.get('Master Options',
'status_email')).split(',')
return [
# Note: reporters.GitHubStatusPush requires txrequests package to allow
# interaction with GitHub REST API.
reporters.GitHubStatusPush(
str(config.options.get('GitHub Status', 'token')),
context=Interpolate("%(prop:buildername)s"),
verbose=
True, # TODO: Turn off the verbosity once this is working reliably.
builders=[
"llvm-clang-x86_64-expensive-checks-ubuntu",
"llvm-clang-x86_64-win-fast",
"clang-x86_64-debian-fast",
"llvm-clang-x86_64-expensive-checks-debian",
]),
reporters.IRC(
useColors=False,
host=str(config.options.get('IRC', 'host')),
nick=str(config.options.get('IRC', 'nick')),
channels=str(config.options.get('IRC', 'channels')).split(','),
#authz=... # TODO: Consider allowing "harmful" operations to authorizes users.
useRevisions=False, # FIXME: There is a bug in the buildbot
showBlameList=True,
notify_events=str(config.options.get('IRC',
'notify_events')).split(','),
),
reporters.MailNotifier(
mode=('problem', ),
fromaddr=
"*****@*****.**", # TODO: Change this to [email protected].
extraRecipients=status_email,
extraHeaders={"Reply-To":
status_email[0]}, # The first from the list.
lookup="lab.llvm.org",
messageFormatter=LLVMInformativeMailNotifier,
# TODO: For debug purposes only. Remove later.
dumpMailsToLog=True,
),
# In addition to that the following notifiers are defined for special
# cases.
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=[
"clang-aarch64-linux-build-cache",
"clang-armv7-linux-build-cache"
]),
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=["clang-x86_64-debian-fast"]),
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=["clang-x64-ninja-win7"]),
reporters.MailNotifier(
fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=["clang-hexagon-elf"]),
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=[
"clang-s390x-linux",
"clang-s390x-linux-multistage",
"clang-s390x-linux-lnt"
]),
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=[
"*****@*****.**",
"*****@*****.**"
],
subject="Build %(builder)s Failure",
mode="failing",
builders=["clang-x86_64-linux-abi-test"]),
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=["llvm-avr-linux"]),
reporters.MailNotifier(
fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=[
"lld-x86_64-win", "lld-x86_64-freebsd", "lld-x86_64-darwin",
"clang-x86_64-linux-abi-test", "clang-with-lto-ubuntu",
"clang-with-thin-lto-ubuntu",
"llvm-clang-x86_64-expensive-checks-win",
"llvm-clang-x86_64-expensive-checks-ubuntu"
]),
reporters.MailNotifier(
fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=[
"*****@*****.**", "*****@*****.**"
],
subject="Build %(builder)s Failure",
mode="failing",
builders=[
"polly-arm-linux",
"aosp-O3-polly-before-vectorizer-unprofitable"
]),
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=True,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="problem",
builders=["reverse-iteration"]),
reporters.MailNotifier(
fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=["clang-cuda-k80", "clang-cuda-p4", "clang-cuda-t4"]),
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=["llvm-riscv-linux"]),
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=[
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**"
],
subject="Build %(builder)s Failure",
mode="failing",
builders=["lldb-x64-windows-ninja"]),
reporters.MailNotifier(
fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=["*****@*****.**", "*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=["mlir-windows"]),
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=["fuchsia-x86_64-linux"]),
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=True,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=["lldb-x86_64-fedora"]),
reporters.MailNotifier(
fromaddr="*****@*****.**",
sendToInterestedUsers=True,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=["fedora-llvm-x86_64", "x86_64-fedora-clang"]),
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=True,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=["lldb-x86_64-debian"]),
reporters.MailNotifier(
fromaddr="*****@*****.**",
sendToInterestedUsers=True,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=["lldb-arm-ubuntu", "lldb-aarch64-ubuntu"]),
reporters.MailNotifier(
fromaddr="*****@*****.**",
sendToInterestedUsers=True,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=[
"llvm-clang-x86_64-win-fast", "lld-x86_64-ubuntu-fast",
"llvm-clang-x86_64-expensive-checks-ubuntu",
"llvm-clang-win-x-armv7l", "llvm-clang-win-x-aarch64"
]),
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=["*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=["clang-ve-ninja"]),
reporters.MailNotifier(
fromaddr="*****@*****.**",
sendToInterestedUsers=True,
extraRecipients=["*****@*****.**", "*****@*****.**"],
subject="Build %(builder)s Failure",
mode="failing",
builders=[
"libc-x86_64-debian", "libc-x86_64_debian-dbg",
"libc-x86_64-debian-dbg-asan"
]),
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=True,
extraRecipients=["*****@*****.**"],
subject="Sphinx build %(builder)s Failure",
mode="failing",
builders=["publish-sphinx-docs"]),
reporters.MailNotifier(
fromaddr="*****@*****.**",
sendToInterestedUsers=True,
extraRecipients=["*****@*****.**"],
subject="ML Compiler Opt Failure: %(builder)s",
mode="failing",
builders=[
"ml-opt-dev-x86-64", "ml-opt-rel-x86-64",
"ml-opt-devrel-x86-64"
]),
reporters.MailNotifier(fromaddr="*****@*****.**",
sendToInterestedUsers=False,
extraRecipients=[
"*****@*****.**",
"*****@*****.**"
],
subject="Build %(builder)s Failure",
mode="failing",
builders=[
"flang-aarch64-ubuntu",
"flang-aarch64-ubuntu-clang",
"flang-aarch64-ubuntu-gcc10"
]),
]