def on_event(data, message, dry_run, **kwargs): """ Whenever PGO builds are completed in mozilla-inbound or fx-team, we trigger the corresponding talos jobs twice. """ if ignored(data): LOG.debug("'%s' with status %i. Nothing to be done.", data['payload']['buildername'], data['payload']['status']) return 0 # SUCCESS # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} payload = data["payload"] buildername = payload["buildername"] revision = payload["revision"] # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 buildername = filter_invalid_builders(buildername) if buildername is None: return -1 # FAILURE status = trigger_talos_jobs_for_build(buildername=buildername, revision=revision, times=2, dry_run=dry_run) LOG.info('We triggered talos jobs for the build.') return status
def on_buildbot_event(data, message, dry_run, stage=False): """Act upon buildbot events.""" # Pulse gives us a job_id and a job_guid, we need request_id. LOG.info( "%s action requested by %s on repo_name %s with job_id: %s" % (data["action"], data["requester"], data["project"], data["job_id"]) ) # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} if stage: treeherder_client = TreeherderClient(host="treeherder.allizom.org") else: treeherder_client = TreeherderClient() repo_name = data["project"] job_id = data["job_id"] result = treeherder_client.get_jobs(repo_name, id=job_id) # If result not found, ignore if not result: LOG.info("We could not find any result for repo_name: %s and " "job_id: %s" % (repo_name, job_id)) message.ack() return result = result[0] buildername = result["ref_data_name"] resultset_id = result["result_set_id"] result_sets = treeherder_client.get_resultsets(repo_name, id=resultset_id) revision = result_sets[0]["revision"] action = data["action"] status = None buildername = filter_invalid_builders(buildername) # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 if buildername is None: status = "Builder %s was invalid." % buildername[0] # Backfill action elif action == "backfill": manual_backfill(revision, buildername, max_revisions=get_maxRevisions(buildername), dry_run=dry_run) if not dry_run: status = "Backfill request sent" else: status = "Dry-run mode, nothing was backfilled" # Send a pulse message showing what we did message_sender = MessageHandler() pulse_message = {"job_id": job_id, "action": action, "requester": data["requester"], "status": status} routing_key = "{}.{}".format(repo_name, action) try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") if not dry_run: # We need to ack the message to remove it from our queue message.ack()
def on_event(data, message, dry_run): """Automatically backfill failed jobs.""" # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} payload = data["payload"] status = payload["status"] buildername = payload["buildername"] # Backfill a failed job if status in [FAILURE, WARNING]: buildername = filter_invalid_builders(buildername) # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 if buildername is None: if not dry_run: # We need to ack the message to remove it from our queue message.ack() return revision = payload["revision"] LOG.info("**") # visual separator LOG.info("Failed job found at revision %s. Buildername: %s", revision, buildername) try: # We want to ensure 1 appearance of the job on every revision revlist = find_backfill_revlist( revision=revision, max_revisions=get_maxRevisions(buildername), buildername=buildername) trigger_range( buildername=buildername, revisions=revlist[1:], times=1, dry_run=dry_run, trigger_build_if_missing=False ) if not dry_run: # We need to ack the message to remove it from our queue message.ack() except ConnectionError: # The message has not been acked so we will try again LOG.warning("Connection error. Trying again") except PushlogError, e: # Unable to retrieve pushlog data. Please check repo_url and revision specified. LOG.warning(str(e)) except Exception, e: # The message has not been acked so we will try again LOG.warning(str(e)) raise
def on_event(data, message, dry_run, acknowledge): """Automatically backfill failed jobs.""" # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} payload = data["payload"] status = payload["status"] buildername = payload["buildername"] # Backfill a failed job if status in [FAILURE, WARNING]: buildername = filter_invalid_builders(buildername) # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 if buildername is None: if acknowledge: # We need to ack the message to remove it from our queue message.ack() return revision = payload["revision"] LOG.info("**") # visual separator LOG.info("Failed job found at revision %s. Buildername: %s", revision, buildername) try: # We want to ensure 1 appearance of the job on every revision revlist = find_backfill_revlist(buildername=buildername, revision=revision) trigger_range(buildername=buildername, revisions=revlist[1:], times=1, dry_run=dry_run, trigger_build_if_missing=False) if acknowledge: # We need to ack the message to remove it from our queue message.ack() except ConnectionError: # The message has not been acked so we will try again LOG.warning("Connection error. Trying again") except PushlogError, e: # Unable to retrieve pushlog data. Please check repo_url and revision specified. LOG.warning(str(e)) except Exception, e: # The message has not been acked so we will try again LOG.warning(str(e)) raise
def on_event(data, message, dry_run, acknowledge, **kwargs): """ Whenever PGO builds are completed in mozilla-inbound or fx-team, we trigger the corresponding talos jobs twice. """ if ignored(data): if acknowledge: # We need to ack the message to remove it from our queue message.ack() LOG.debug("'%s' with status %i. Nothing to be done.", data["payload"]["buildername"], data["payload"]["status"]) return 0 # SUCCESS # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} payload = data["payload"] buildername = payload["buildername"] revision = payload["revision"] # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 buildername = filter_invalid_builders(buildername) if buildername is None: if acknowledge: # We need to ack the message to remove it from our queue message.ack() return -1 # FAILURE status = trigger_talos_jobs_for_build(buildername=buildername, revision=revision, times=2, dry_run=dry_run) if acknowledge: # We need to ack the message to remove it from our queue message.ack() LOG.info("We triggered talos jobs for the build.") return status
def on_event(data, message, dry_run, treeherder_server_url, acknowledge, **kwargs): """Act upon Treeherder job events. Return if the outcome was successful or not """ exit_code = 0 # SUCCESS if ignored(data): if acknowledge: # We need to ack the message to remove it from our queue message.ack() return exit_code # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} treeherder_client = TreeherderClient(server_url=treeherder_server_url) action = data["action"].capitalize() job_id = data["job_id"] repo_name = data["project"] status = None # We want to know the status of the job we're processing try: job_info = treeherder_client.get_jobs(repo_name, id=job_id)[0] except IndexError: LOG.info("We could not find any job_info for repo_name: %s and " "job_id: %s" % (repo_name, job_id)) return exit_code buildername = job_info["ref_data_name"] # We want to know the revision associated for this job result_sets = treeherder_client.get_resultsets(repo_name, id=job_info["result_set_id"]) revision = result_sets[0]["revision"] link_to_job = "{}/#/jobs?repo={}&revision={}&selectedJob={}".format( treeherder_server_url, repo_name, revision, job_id ) LOG.info("{} action requested by {} for '{}'".format(action, data["requester"], buildername)) LOG.info("Request for {}".format(link_to_job)) buildername = filter_invalid_builders(buildername) if buildername is None: LOG.info("Treeherder can send us invalid builder names.") LOG.info("See https://bugzilla.mozilla.org/show_bug.cgi?id=1242038.") LOG.warning('Requested job name "%s" is invalid.' % job_info["ref_data_name"]) exit_code = -1 # FAILURE # There are various actions that can be taken on a job, however, we currently # only process the backfill one elif action == "Backfill": exit_code = manual_backfill(revision=revision, buildername=buildername, dry_run=dry_run) if not dry_run: status = "Backfill request sent" else: status = "Dry-run mode, nothing was backfilled." LOG.debug(status) else: LOG.error('We were not aware of the "{}" action. Please file an issue'.format(action)) exit_code = -1 # FAILURE if acknowledge: # We need to ack the message to remove it from our queue message.ack() return exit_code
def on_event(data, message, dry_run, treeherder_server_url, acknowledge, **kwargs): """Act upon Treeherder job events. Return if the outcome was successful or not """ LOG.info('Acknowledge value: {}'.format(acknowledge)) exit_code = 0 # SUCCESS if ignored(data): if acknowledge: # We need to ack the message to remove it from our queue LOG.info('Message acknowledged') message.ack() return exit_code # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} treeherder_client = TreeherderClient(server_url=treeherder_server_url) action = data['action'].capitalize() job_id = data['job_id'] repo_name = data['project'] status = None # We want to know the status of the job we're processing try: job_info = treeherder_client.get_jobs(repo_name, id=job_id)[0] except IndexError: LOG.info("We could not find any job_info for repo_name: %s and " "job_id: %s" % (repo_name, job_id)) return exit_code buildername = job_info["ref_data_name"] # We want to know the revision associated for this job result_sets = treeherder_client.get_resultsets( repo_name, id=job_info["result_set_id"]) revision = result_sets[0]["revision"] link_to_job = '{}/#/jobs?repo={}&revision={}&selectedJob={}'.format( treeherder_server_url, repo_name, revision, job_id) LOG.info("{} action requested by {} for '{}'".format( action, data['requester'], buildername, )) LOG.info('Request for {}'.format(link_to_job)) buildername = filter_invalid_builders(buildername) if buildername is None: LOG.info('Treeherder can send us invalid builder names.') LOG.info('See https://bugzilla.mozilla.org/show_bug.cgi?id=1242038.') LOG.warning('Requested job name "%s" is invalid.' % job_info['ref_data_name']) exit_code = -1 # FAILURE # There are various actions that can be taken on a job, however, we currently # only process the backfill one elif action == "Backfill": exit_code = manual_backfill( revision=revision, buildername=buildername, dry_run=dry_run, ) if not dry_run: status = 'Backfill request sent' else: status = 'Dry-run mode, nothing was backfilled.' LOG.debug(status) else: LOG.error('We were not aware of the "{}" action. Please file an issue'. format(action)) exit_code = -1 # FAILURE if acknowledge: # We need to ack the message to remove it from our queue LOG.info('Message acknowledged') message.ack() return exit_code
def on_runnable_job_event(data, message, dry_run, treeherder_host, acknowledge): # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} treeherder_client = TreeherderClient(host='treeherder.allizom.org') # XXX: # Grabbing data received over pulse repo_name = data["project"] requester = data["requester"] resultset_id = data["resultset_id"] buildernames = data["buildernames"] resultset = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0] revision = resultset["revision"] author = resultset["author"] status = None treeherder_link = TREEHERDER % { 'host': treeherder_host, 'repo': repo_name, 'revision': resultset['revision'] } message_sender = MessageHandler() if not (requester.endswith('@mozilla.com') or author == requester or whitelisted_users(requester)): # We want to see this in the alerts LOG.error("Notice that we're letting %s schedule jobs for %s." % (requester, treeherder_link)) ''' # Everyone can press the button, but only authorized users can trigger jobs # TODO: remove this when proper LDAP identication is set up on TH if not (requester.endswith('@mozilla.com') or author == requester or whitelisted_users(requester)): if acknowledge: # Remove message from pulse queue message.ack() # We publish a message saying we will not trigger the job pulse_message = { 'resultset_id': resultset_id, 'requester': requester, 'status': "Could not determine if the user is authorized, nothing was triggered."} routing_key = '{}.{}'.format(repo_name, 'runnable') try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") LOG.error("Requester %s is not allowed to trigger jobs on %s." % (requester, treeherder_link)) return # Raising an exception adds too much noise ''' LOG.info("New jobs requested by %s for %s" % (requester, treeherder_link)) LOG.info("List of builders:") for b in buildernames: LOG.info("- %s" % b) buildernames = filter_invalid_builders(buildernames) # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 if buildernames is None: if acknowledge: # We need to ack the message to remove it from our queue message.ack() return builders_graph, other_builders_to_schedule = buildbot_bridge.buildbot_graph_builder( builders=buildernames, revision=revision, complete=False # XXX: This can be removed when BBB is in use ) if builders_graph != {}: mgr = TaskClusterBuildbotManager() mgr.schedule_graph( repo_name=repo_name, revision=revision, metadata={ 'name': 'pulse_actions_graph', 'description': 'Adding new jobs to push via pulse_actions/treeherder for %s.' % requester, 'owner': requester, 'source': treeherder_link, }, builders_graph=builders_graph, dry_run=dry_run) else: LOG.info("We don't have anything to schedule through TaskCluster") if other_builders_to_schedule: # XXX: We should be able to replace this once all Buildbot jobs run through BBB # XXX: There might be a work around with # https://github.com/mozilla/mozilla_ci_tools/issues/424 LOG.info("We're going to schedule these builders via Buildapi: %s" % str(other_builders_to_schedule)) # This is used for test jobs which need an existing Buildbot job to be scheduled for buildername in other_builders_to_schedule: trigger_job(revision, buildername, dry_run=dry_run) else: LOG.info("We don't have anything to schedule through Buildapi") # Send a pulse message showing what we did message_sender = MessageHandler() pulse_message = { 'resultset_id': resultset_id, 'graph': builders_graph, 'requester': requester, 'status': status } routing_key = '{}.{}'.format(repo_name, 'runnable') try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") if acknowledge: # We need to ack the message to remove it from our queue message.ack()
def on_event(data, message, dry_run, treeherder_server_url, **kwargs): """Act upon Treeherder job events. Return if the outcome was successful or not """ exit_code = 0 # SUCCESS if ignored(data): return exit_code # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} treeherder_client = TreeherderClient(server_url=treeherder_server_url) action = data['action'].capitalize() job_id = data['job_id'] repo_name = data['project'] status = None # We want to know the status of the job we're processing try: job_info = treeherder_client.get_jobs(repo_name, id=job_id)[0] except IndexError: LOG.info("We could not find any job_info for repo_name: %s and " "job_id: %s" % (repo_name, job_id)) return exit_code # We want to know the revision associated for this job result_set = treeherder_client.get_resultsets( repo_name, id=job_info["result_set_id"])[0] revision = result_set["revision"] link_to_job = '{}/#/jobs?repo={}&revision={}&selectedJob={}'.format( treeherder_server_url, repo_name, revision, job_id) # There are various actions that can be taken on a job, however, we currently # only process the backfill one if action == "Backfill": if job_info["build_system_type"] == "taskcluster": jobs = [] jobs_per_call = 250 offset = 0 while True: results = treeherder_client.get_jobs( repo_name, push_id=job_info["result_set_id"], count=jobs_per_call, offset=offset) jobs += results if (len(results) < jobs_per_call): break offset += jobs_per_call decision = [ t for t in jobs if t["job_type_name"] == "Gecko Decision Task" ][0] details = treeherder_client.get_job_details( job_guid=decision["job_guid"]) inspect = [ detail["url"] for detail in details if detail["value"] == "Inspect Task" ][0] # Pull out the taskId from the URL e.g. # oN1NErz_Rf2DZJ1hi7YVfA from <tc_tools_site>/task-inspector/#oN1NErz_Rf2DZJ1hi7YVfA/ decision_id = inspect.partition("#")[-1].rpartition("/")[0] mgr = TaskClusterManager(dry_run=dry_run) mgr.schedule_action_task(decision_id=decision_id, action="backfill", action_args={ "project": repo_name, "job": job_info["id"] }) else: buildername = job_info["ref_data_name"] LOG.info("{} action requested by {} for '{}'".format( action, data['requester'], buildername, )) LOG.info('Request for {}'.format(link_to_job)) buildername = filter_invalid_builders(buildername) if buildername is None: LOG.info('Treeherder can send us invalid builder names.') LOG.info( 'See https://bugzilla.mozilla.org/show_bug.cgi?id=1242038.' ) LOG.warning('Requested job name "%s" is invalid.' % job_info['ref_data_name']) exit_code = -1 # FAILURE else: exit_code = manual_backfill( revision=revision, buildername=buildername, dry_run=dry_run, ) if not dry_run: status = 'Backfill request sent' else: status = 'Dry-run mode, nothing was backfilled.' LOG.debug(status) else: LOG.error('We were not aware of the "{}" action. Please file an issue'. format(action)) exit_code = -1 # FAILURE return exit_code
def on_runnable_job_event(data, message, dry_run, stage): # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} if stage: treeherder_client = TreeherderClient(host='treeherder.allizom.org') else: treeherder_client = TreeherderClient() # Grabbing data received over pulse repo_name = data["project"] requester = data["requester"] resultset_id = data["resultset_id"] buildernames = data["buildernames"] resultset = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0] revision = resultset["revision"] author = resultset["author"] status = None treeherder_link = TREEHERDER % {'repo': repo_name, 'revision': resultset['revision']} message_sender = MessageHandler() # Everyone can press the button, but only authorized users can trigger jobs # TODO: remove this when proper LDAP identication is set up on TH if not (requester.endswith('@mozilla.com') or author == requester or whitelisted_users(requester)): if not dry_run: # Remove message from pulse queue message.ack() # We publish a message saying we will not trigger the job pulse_message = { 'resultset_id': resultset_id, 'requester': requester, 'status': "Could not determine if the user is authorized, nothing was triggered."} routing_key = '{}.{}'.format(repo_name, 'runnable') try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") LOG.error("Requester %s is not allowed to trigger jobs on %s." % (requester, treeherder_link)) return # Raising an exception adds too much noise LOG.info("New jobs requested by %s for %s" % (requester, treeherder_link)) LOG.info("List of builders:") for b in buildernames: LOG.info("- %s" % b) buildernames = filter_invalid_builders(buildernames) # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 if buildernames is None: if not dry_run: # We need to ack the message to remove it from our queue message.ack() return builders_graph, other_builders_to_schedule = buildbot_bridge.buildbot_graph_builder( builders=buildernames, revision=revision, complete=False # XXX: This can be removed when BBB is in use ) if builders_graph != {}: mgr = TaskClusterBuildbotManager() mgr.schedule_graph( repo_name=repo_name, revision=revision, metadata={ 'name': 'pulse_actions_graph', 'description': 'Adding new jobs to push via pulse_actions/treeherder for %s.' % requester, 'owner': requester, 'source': treeherder_link, }, builders_graph=builders_graph, dry_run=dry_run) else: LOG.info("We don't have anything to schedule through TaskCluster") if other_builders_to_schedule: # XXX: We should be able to replace this once all Buildbot jobs run through BBB # XXX: There might be a work around with # https://github.com/mozilla/mozilla_ci_tools/issues/424 LOG.info("We're going to schedule these builders via Buildapi.") # This is used for test jobs which need an existing Buildbot job to be scheduled for buildername in other_builders_to_schedule: trigger_job(revision, buildername, dry_run=dry_run) else: LOG.info("We don't have anything to schedule through Buildapi") # Send a pulse message showing what we did message_sender = MessageHandler() pulse_message = { 'resultset_id': resultset_id, 'graph': builders_graph, 'requester': requester, 'status': status} routing_key = '{}.{}'.format(repo_name, 'runnable') try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") if not dry_run: # We need to ack the message to remove it from our queue message.ack()
def on_event(data, message, dry_run, treeherder_server_url, **kwargs): if ignored(data): return 0 # SUCCESS # Grabbing data received over pulse repo_name = data["project"] requester = data["requester"] resultset_id = data["resultset_id"] if "requested_jobs" in data: requested_jobs = data["requested_jobs"] else: LOG.error("Appropriate job requests not found in the pulse message.") return -1 treeherder_client = TreeherderClient(server_url=treeherder_server_url) resultset = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0] revision = resultset["revision"] author = resultset["author"] treeherder_link = TREEHERDER % { 'treeherder_server_url': treeherder_server_url, 'repo': repo_name, 'revision': resultset['revision'] } metadata = { 'name': 'pulse_actions_graph', 'description': 'Adding new jobs to push via pulse_actions/treeherder for %s.' % requester, 'owner': requester, 'source': treeherder_link, } if not (requester.endswith('@mozilla.com') or author == requester or whitelisted_users(requester)): # We want to see this in the alerts LOG.warning( "Notice that we're letting {} schedule jobs for {}.".format( requester, treeherder_link)) LOG.info("New jobs requested by %s for %s" % (requester, treeherder_link)) LOG.info("List of requested jobs:") for job in requested_jobs: LOG.info("- {}".format(job)) # This is empty strings in non-try pulse messages # Remove support for `decisionTaskID` once bug 1286897 fixed. decision_task_id = data.get('decision_task_id', data.get('decisionTaskID')) # Separate Buildbot buildernames from TaskCluster task labels if decision_task_id: task_labels = [ x for x in requested_jobs if is_taskcluster_label(x, decision_task_id) ] else: task_labels = [] # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 buildernames = filter_invalid_builders( list(set(requested_jobs) - set(task_labels))) # XXX: In the future handle return codes add_taskcluster_jobs(task_labels, decision_task_id, repo_name, dry_run) add_buildbot_jobs(repo_name, revision, buildernames, metadata, dry_run) return 0 # SUCCESS