def on_buildbot_event(data, message, dry_run): """Act upon buildbot events.""" # Pulse gives us a job_id and a job_guid, we need request_id. repo_name = data['project'] job_id = data['job_id'] request_id = get_request_id_from_job_id(repo_name, job_id) action = data['action'] status = None # Re-trigger action if action == 'retrigger': make_retrigger_request(repo_name, request_id, dry_run=dry_run) if not dry_run: status = 'Retrigger request sent' else: status = 'Dry-mode, nothing was retriggered' # Cancel action elif action == 'cancel': make_cancel_request(repo_name, request_id, dry_run=dry_run) if not dry_run: status = 'Cancel request sent' else: status = 'Dry-run mode, nothing was cancelled' # Send a pulse message showing what we did message_sender = MessageHandler() pulse_message = { 'job_id': job_id, 'request_id': request_id, 'action': action, 'requester': data['requester'], 'status': status} routing_key = '{}.{}'.format(repo_name, action) message_sender.publish_message(pulse_message, routing_key) # We need to ack the message to remove it from our queue message.ack()
def on_buildbot_event(data, message, dry_run, stage=False): """Act upon buildbot events.""" # Pulse gives us a job_id and a job_guid, we need request_id. LOG.info( "%s action requested by %s on repo_name %s with job_id: %s" % (data["action"], data["requester"], data["project"], data["job_id"]) ) # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} if stage: treeherder_client = TreeherderClient(host="treeherder.allizom.org") else: treeherder_client = TreeherderClient() repo_name = data["project"] job_id = data["job_id"] result = treeherder_client.get_jobs(repo_name, id=job_id) # If result not found, ignore if not result: LOG.info("We could not find any result for repo_name: %s and " "job_id: %s" % (repo_name, job_id)) message.ack() return result = result[0] buildername = result["ref_data_name"] resultset_id = result["result_set_id"] result_sets = treeherder_client.get_resultsets(repo_name, id=resultset_id) revision = result_sets[0]["revision"] action = data["action"] status = None buildername = filter_invalid_builders(buildername) # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 if buildername is None: status = "Builder %s was invalid." % buildername[0] # Backfill action elif action == "backfill": manual_backfill(revision, buildername, max_revisions=get_maxRevisions(buildername), dry_run=dry_run) if not dry_run: status = "Backfill request sent" else: status = "Dry-run mode, nothing was backfilled" # Send a pulse message showing what we did message_sender = MessageHandler() pulse_message = {"job_id": job_id, "action": action, "requester": data["requester"], "status": status} routing_key = "{}.{}".format(repo_name, action) try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") if not dry_run: # We need to ack the message to remove it from our queue message.ack()
def on_resultset_action_event(data, message, dry_run): # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} repo_name = data["project"] action = data["action"] times = data["times"] # Pulse gives us resultset_id, we need to get revision from it. resultset_id = data["resultset_id"] treeherder_client = TreeherderClient() # We do not handle 'cancel_all' action right now, so skip it. if action == "cancel_all": message.ack() return LOG.info("%s action requested by %s on repo_name %s with resultset_id: %s" % (data['action'], data["requester"], data["project"], data["resultset_id"])) revision = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0]["revision"] status = None if action == "trigger_missing_jobs": trigger_missing_jobs_for_revision(repo_name, revision, dry_run=dry_run) if not dry_run: status = 'trigger_missing_jobs request sent' else: status = 'Dry-mode, no request sent' elif action == "trigger_all_talos_jobs": trigger_all_talos_jobs(repo_name, revision, times, dry_run=dry_run) if not dry_run: status = 'trigger_all_talos_jobs %s times request sent' % times else: status = 'Dry-mode, no request sent' # Send a pulse message showing what we did message_sender = MessageHandler() pulse_message = { 'resultset_id': resultset_id, 'action': action, 'requester': data['requester'], 'status': status} routing_key = '{}.{}'.format(repo_name, action) message_sender.publish_message(pulse_message, routing_key) # We need to ack the message to remove it from our queue message.ack()
def on_runnable_job_event(data, message, dry_run, treeherder_host, acknowledge): # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} treeherder_client = TreeherderClient(host='treeherder.allizom.org') # XXX: # Grabbing data received over pulse repo_name = data["project"] requester = data["requester"] resultset_id = data["resultset_id"] buildernames = data["buildernames"] resultset = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0] revision = resultset["revision"] author = resultset["author"] status = None treeherder_link = TREEHERDER % { 'host': treeherder_host, 'repo': repo_name, 'revision': resultset['revision'] } message_sender = MessageHandler() if not (requester.endswith('@mozilla.com') or author == requester or whitelisted_users(requester)): # We want to see this in the alerts LOG.error("Notice that we're letting %s schedule jobs for %s." % (requester, treeherder_link)) ''' # Everyone can press the button, but only authorized users can trigger jobs # TODO: remove this when proper LDAP identication is set up on TH if not (requester.endswith('@mozilla.com') or author == requester or whitelisted_users(requester)): if acknowledge: # Remove message from pulse queue message.ack() # We publish a message saying we will not trigger the job pulse_message = { 'resultset_id': resultset_id, 'requester': requester, 'status': "Could not determine if the user is authorized, nothing was triggered."} routing_key = '{}.{}'.format(repo_name, 'runnable') try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") LOG.error("Requester %s is not allowed to trigger jobs on %s." % (requester, treeherder_link)) return # Raising an exception adds too much noise ''' LOG.info("New jobs requested by %s for %s" % (requester, treeherder_link)) LOG.info("List of builders:") for b in buildernames: LOG.info("- %s" % b) buildernames = filter_invalid_builders(buildernames) # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 if buildernames is None: if acknowledge: # We need to ack the message to remove it from our queue message.ack() return builders_graph, other_builders_to_schedule = buildbot_bridge.buildbot_graph_builder( builders=buildernames, revision=revision, complete=False # XXX: This can be removed when BBB is in use ) if builders_graph != {}: mgr = TaskClusterBuildbotManager() mgr.schedule_graph( repo_name=repo_name, revision=revision, metadata={ 'name': 'pulse_actions_graph', 'description': 'Adding new jobs to push via pulse_actions/treeherder for %s.' % requester, 'owner': requester, 'source': treeherder_link, }, builders_graph=builders_graph, dry_run=dry_run) else: LOG.info("We don't have anything to schedule through TaskCluster") if other_builders_to_schedule: # XXX: We should be able to replace this once all Buildbot jobs run through BBB # XXX: There might be a work around with # https://github.com/mozilla/mozilla_ci_tools/issues/424 LOG.info("We're going to schedule these builders via Buildapi: %s" % str(other_builders_to_schedule)) # This is used for test jobs which need an existing Buildbot job to be scheduled for buildername in other_builders_to_schedule: trigger_job(revision, buildername, dry_run=dry_run) else: LOG.info("We don't have anything to schedule through Buildapi") # Send a pulse message showing what we did message_sender = MessageHandler() pulse_message = { 'resultset_id': resultset_id, 'graph': builders_graph, 'requester': requester, 'status': status } routing_key = '{}.{}'.format(repo_name, 'runnable') try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") if acknowledge: # We need to ack the message to remove it from our queue message.ack()
def on_runnable_job_event(data, message, dry_run, stage): # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} if stage: treeherder_client = TreeherderClient(host='treeherder.allizom.org') else: treeherder_client = TreeherderClient() # Grabbing data received over pulse repo_name = data["project"] requester = data["requester"] resultset_id = data["resultset_id"] buildernames = data["buildernames"] resultset = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0] revision = resultset["revision"] author = resultset["author"] status = None treeherder_link = TREEHERDER % {'repo': repo_name, 'revision': resultset['revision']} message_sender = MessageHandler() # Everyone can press the button, but only authorized users can trigger jobs # TODO: remove this when proper LDAP identication is set up on TH if not (requester.endswith('@mozilla.com') or author == requester or whitelisted_users(requester)): if not dry_run: # Remove message from pulse queue message.ack() # We publish a message saying we will not trigger the job pulse_message = { 'resultset_id': resultset_id, 'requester': requester, 'status': "Could not determine if the user is authorized, nothing was triggered."} routing_key = '{}.{}'.format(repo_name, 'runnable') try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") LOG.error("Requester %s is not allowed to trigger jobs on %s." % (requester, treeherder_link)) return # Raising an exception adds too much noise LOG.info("New jobs requested by %s for %s" % (requester, treeherder_link)) LOG.info("List of builders:") for b in buildernames: LOG.info("- %s" % b) buildernames = filter_invalid_builders(buildernames) # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 if buildernames is None: if not dry_run: # We need to ack the message to remove it from our queue message.ack() return builders_graph, other_builders_to_schedule = buildbot_bridge.buildbot_graph_builder( builders=buildernames, revision=revision, complete=False # XXX: This can be removed when BBB is in use ) if builders_graph != {}: mgr = TaskClusterBuildbotManager() mgr.schedule_graph( repo_name=repo_name, revision=revision, metadata={ 'name': 'pulse_actions_graph', 'description': 'Adding new jobs to push via pulse_actions/treeherder for %s.' % requester, 'owner': requester, 'source': treeherder_link, }, builders_graph=builders_graph, dry_run=dry_run) else: LOG.info("We don't have anything to schedule through TaskCluster") if other_builders_to_schedule: # XXX: We should be able to replace this once all Buildbot jobs run through BBB # XXX: There might be a work around with # https://github.com/mozilla/mozilla_ci_tools/issues/424 LOG.info("We're going to schedule these builders via Buildapi.") # This is used for test jobs which need an existing Buildbot job to be scheduled for buildername in other_builders_to_schedule: trigger_job(revision, buildername, dry_run=dry_run) else: LOG.info("We don't have anything to schedule through Buildapi") # Send a pulse message showing what we did message_sender = MessageHandler() pulse_message = { 'resultset_id': resultset_id, 'graph': builders_graph, 'requester': requester, 'status': status} routing_key = '{}.{}'.format(repo_name, 'runnable') try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") if not dry_run: # We need to ack the message to remove it from our queue message.ack()
def on_resultset_action_event(data, message, dry_run, stage=False): # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} repo_name = data["project"] action = data["action"] times = data["times"] # Pulse gives us resultset_id, we need to get revision from it. resultset_id = data["resultset_id"] if stage: treeherder_client = TreeherderClient(host='treeherder.allizom.org') else: treeherder_client = TreeherderClient() # We do not handle 'cancel_all' action right now, so skip it. if action == "cancel_all": if not dry_run: message.ack() return LOG.info("%s action requested by %s on repo_name %s with resultset_id: %s" % ( data['action'], data["requester"], data["project"], data["resultset_id"]) ) revision = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0]["revision"] status = None if action == "trigger_missing_jobs": mgr = BuildAPIManager() mgr.trigger_missing_jobs_for_revision(repo_name, revision, dry_run=dry_run) if not dry_run: status = 'trigger_missing_jobs request sent' else: status = 'Dry-mode, no request sent' elif action == "trigger_all_talos_jobs": trigger_all_talos_jobs( repo_name=repo_name, revision=revision, times=times, priority=-1, dry_run=dry_run ) if not dry_run: status = 'trigger_all_talos_jobs: {0} times request sent with priority'\ 'lower then normal'.format(times) else: status = 'Dry-mode, no request sent' # Send a pulse message showing what we did message_sender = MessageHandler() pulse_message = { 'resultset_id': resultset_id, 'action': action, 'requester': data['requester'], 'status': status} routing_key = '{}.{}'.format(repo_name, action) try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") if not dry_run: # We need to ack the message to remove it from our queue message.ack()