def add_buildbot_jobs(repo_name, revision, buildernames, metadata, dry_run):
# Cleaning mozci caches
buildjson.BUILDS_CACHE = {}
query_jobs.JOBS_CACHE = {}
if not buildernames:
return -1 # FAILURE
builders_graph, other_builders_to_schedule = buildbot_bridge.buildbot_graph_builder(
builders=buildernames,
revision=revision,
complete=False # XXX: This can be removed when BBB is in use
)
if builders_graph != {}:
mgr = TaskClusterBuildbotManager(dry_run=dry_run)
mgr.schedule_graph(repo_name=repo_name,
revision=revision,
metadata=metadata,
builders_graph=builders_graph,
dry_run=dry_run)
else:
LOG.info("We don't have anything to schedule through TaskCluster")
if other_builders_to_schedule:
# XXX: We should be able to replace this once all Buildbot jobs run through BBB
# XXX: There might be a work around with
# https://github.com/mozilla/mozilla_ci_tools/issues/424
LOG.info("We're going to schedule these builders via Buildapi.")
# This is used for test jobs which need an existing Buildbot job to be scheduled
for buildername in other_builders_to_schedule:
trigger_job(revision, buildername, dry_run=dry_run)
else:
LOG.info("We don't have anything to schedule through Buildapi")
def trigger_missing_jobs_for_revision(self, repo_name, revision, dry_run=False, trigger_build_if_missing=True):
"""
Trigger missing jobs for a given revision.
Jobs containing 'b2g' or 'pgo' in their buildername will not be triggered.
"""
builders_for_repo = BuildApi().determine_missing_jobs(repo_name=repo_name, revision=revision)
buildbot_graph = buildbot_bridge.buildbot_graph_builder(builders_for_repo, revision)[0]
self.schedule_graph(repo_name=repo_name, revision=revision, builders_graph=buildbot_graph)
def setUp(self):
self.buildernames = ['Android armv7 API 15+ try debug build']
self.revision = '1ab622ac1706a0f5dfaf7734a1c56aa9d3502eec'
self.repo_name = 'try'
self.builders_graph, _ = buildbot_bridge.buildbot_graph_builder(
builders=self.buildernames,
revision=self.revision,
complete=False
)
def trigger_range(self, buildername, repo_name, revisions, times, dry_run, files,
trigger_build_if_missing):
for revision in revisions:
builder_graph, trigger_with_buildapi = buildbot_bridge.buildbot_graph_builder(
builders=[buildername],
revision=revision,
complete=False # XXX: This can be removed when BBB is in use
)
self.schedule_graph(
repo_name=repo_name,
revision=revision,
builders_graph=builder_graph,
dry_run=dry_run
)
def trigger_missing_jobs_for_revision(self,
repo_name,
revision,
dry_run=False,
trigger_build_if_missing=True):
"""
Trigger missing jobs for a given revision.
Jobs containing 'b2g' or 'pgo' in their buildername will not be triggered.
"""
builders_for_repo = BuildApi().determine_missing_jobs(
repo_name=repo_name, revision=revision)
buildbot_graph = buildbot_bridge.buildbot_graph_builder(
builders_for_repo, revision)[0]
self.schedule_graph(repo_name=repo_name,
revision=revision,
builders_graph=buildbot_graph)
def on_runnable_job_event(data, message, dry_run, treeherder_host,
acknowledge):
# Cleaning mozci caches
buildjson.BUILDS_CACHE = {}
query_jobs.JOBS_CACHE = {}
treeherder_client = TreeherderClient(host='treeherder.allizom.org')
# XXX:
# Grabbing data received over pulse
repo_name = data["project"]
requester = data["requester"]
resultset_id = data["resultset_id"]
buildernames = data["buildernames"]
resultset = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0]
revision = resultset["revision"]
author = resultset["author"]
status = None
treeherder_link = TREEHERDER % {
'host': treeherder_host,
'repo': repo_name,
'revision': resultset['revision']
}
message_sender = MessageHandler()
if not (requester.endswith('@mozilla.com') or author == requester
or whitelisted_users(requester)):
# We want to see this in the alerts
LOG.error("Notice that we're letting %s schedule jobs for %s." %
(requester, treeherder_link))
'''
# Everyone can press the button, but only authorized users can trigger jobs
# TODO: remove this when proper LDAP identication is set up on TH
if not (requester.endswith('@mozilla.com') or author == requester or
whitelisted_users(requester)):
if acknowledge:
# Remove message from pulse queue
message.ack()
# We publish a message saying we will not trigger the job
pulse_message = {
'resultset_id': resultset_id,
'requester': requester,
'status': "Could not determine if the user is authorized, nothing was triggered."}
routing_key = '{}.{}'.format(repo_name, 'runnable')
try:
message_sender.publish_message(pulse_message, routing_key)
except:
LOG.warning("Failed to publish message over pulse stream.")
LOG.error("Requester %s is not allowed to trigger jobs on %s." %
(requester, treeherder_link))
return # Raising an exception adds too much noise
'''
LOG.info("New jobs requested by %s for %s" % (requester, treeherder_link))
LOG.info("List of builders:")
for b in buildernames:
LOG.info("- %s" % b)
buildernames = filter_invalid_builders(buildernames)
# Treeherder can send us invalid builder names
# https://bugzilla.mozilla.org/show_bug.cgi?id=1242038
if buildernames is None:
if acknowledge:
# We need to ack the message to remove it from our queue
message.ack()
return
builders_graph, other_builders_to_schedule = buildbot_bridge.buildbot_graph_builder(
builders=buildernames,
revision=revision,
complete=False # XXX: This can be removed when BBB is in use
)
if builders_graph != {}:
mgr = TaskClusterBuildbotManager()
mgr.schedule_graph(
repo_name=repo_name,
revision=revision,
metadata={
'name':
'pulse_actions_graph',
'description':
'Adding new jobs to push via pulse_actions/treeherder for %s.'
% requester,
'owner':
requester,
'source':
treeherder_link,
},
builders_graph=builders_graph,
dry_run=dry_run)
else:
LOG.info("We don't have anything to schedule through TaskCluster")
if other_builders_to_schedule:
# XXX: We should be able to replace this once all Buildbot jobs run through BBB
# XXX: There might be a work around with
# https://github.com/mozilla/mozilla_ci_tools/issues/424
LOG.info("We're going to schedule these builders via Buildapi: %s" %
str(other_builders_to_schedule))
# This is used for test jobs which need an existing Buildbot job to be scheduled
for buildername in other_builders_to_schedule:
trigger_job(revision, buildername, dry_run=dry_run)
else:
LOG.info("We don't have anything to schedule through Buildapi")
# Send a pulse message showing what we did
message_sender = MessageHandler()
pulse_message = {
'resultset_id': resultset_id,
'graph': builders_graph,
'requester': requester,
'status': status
}
routing_key = '{}.{}'.format(repo_name, 'runnable')
try:
message_sender.publish_message(pulse_message, routing_key)
except:
LOG.warning("Failed to publish message over pulse stream.")
if acknowledge:
# We need to ack the message to remove it from our queue
message.ack()
def on_runnable_job_event(data, message, dry_run, stage):
# Cleaning mozci caches
buildjson.BUILDS_CACHE = {}
query_jobs.JOBS_CACHE = {}
if stage:
treeherder_client = TreeherderClient(host='treeherder.allizom.org')
else:
treeherder_client = TreeherderClient()
# Grabbing data received over pulse
repo_name = data["project"]
requester = data["requester"]
resultset_id = data["resultset_id"]
buildernames = data["buildernames"]
resultset = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0]
revision = resultset["revision"]
author = resultset["author"]
status = None
treeherder_link = TREEHERDER % {'repo': repo_name, 'revision': resultset['revision']}
message_sender = MessageHandler()
# Everyone can press the button, but only authorized users can trigger jobs
# TODO: remove this when proper LDAP identication is set up on TH
if not (requester.endswith('@mozilla.com') or author == requester or
whitelisted_users(requester)):
if not dry_run:
# Remove message from pulse queue
message.ack()
# We publish a message saying we will not trigger the job
pulse_message = {
'resultset_id': resultset_id,
'requester': requester,
'status': "Could not determine if the user is authorized, nothing was triggered."}
routing_key = '{}.{}'.format(repo_name, 'runnable')
try:
message_sender.publish_message(pulse_message, routing_key)
except:
LOG.warning("Failed to publish message over pulse stream.")
LOG.error("Requester %s is not allowed to trigger jobs on %s." %
(requester, treeherder_link))
return # Raising an exception adds too much noise
LOG.info("New jobs requested by %s for %s" % (requester, treeherder_link))
LOG.info("List of builders:")
for b in buildernames:
LOG.info("- %s" % b)
buildernames = filter_invalid_builders(buildernames)
# Treeherder can send us invalid builder names
# https://bugzilla.mozilla.org/show_bug.cgi?id=1242038
if buildernames is None:
if not dry_run:
# We need to ack the message to remove it from our queue
message.ack()
return
builders_graph, other_builders_to_schedule = buildbot_bridge.buildbot_graph_builder(
builders=buildernames,
revision=revision,
complete=False # XXX: This can be removed when BBB is in use
)
if builders_graph != {}:
mgr = TaskClusterBuildbotManager()
mgr.schedule_graph(
repo_name=repo_name,
revision=revision,
metadata={
'name': 'pulse_actions_graph',
'description':
'Adding new jobs to push via pulse_actions/treeherder for %s.' % requester,
'owner': requester,
'source': treeherder_link,
},
builders_graph=builders_graph,
dry_run=dry_run)
else:
LOG.info("We don't have anything to schedule through TaskCluster")
if other_builders_to_schedule:
# XXX: We should be able to replace this once all Buildbot jobs run through BBB
# XXX: There might be a work around with
# https://github.com/mozilla/mozilla_ci_tools/issues/424
LOG.info("We're going to schedule these builders via Buildapi.")
# This is used for test jobs which need an existing Buildbot job to be scheduled
for buildername in other_builders_to_schedule:
trigger_job(revision, buildername, dry_run=dry_run)
else:
LOG.info("We don't have anything to schedule through Buildapi")
# Send a pulse message showing what we did
message_sender = MessageHandler()
pulse_message = {
'resultset_id': resultset_id,
'graph': builders_graph,
'requester': requester,
'status': status}
routing_key = '{}.{}'.format(repo_name, 'runnable')
try:
message_sender.publish_message(pulse_message, routing_key)
except:
LOG.warning("Failed to publish message over pulse stream.")
if not dry_run:
# We need to ack the message to remove it from our queue
message.ack()