def main(): parser = ArgumentParser() parser.add_argument("--debug", action="store_true", dest="debug", help="set debug for logging.") parser.add_argument("--dry-run", action="store_true", dest="dry_run", help="Dry run. No real actions are taken.") parser.add_argument("--repo-name", action="store", dest="repo_name", type=str, help="Repository name, e.g. mozilla-inbound.") parser.add_argument("--revision", action="store", dest="revision", type=str, help="12-char representing a push.") parser.add_argument('builders_graph', metavar='builders_graph', help='Graph of builders in the form of: ' 'dict(builder: [dep_builders].') options = parser.parse_args() if options.debug: setup_logging(logging.DEBUG) else: setup_logging() mgr = TaskClusterBuildbotManager() # XXX: test what happens when we have a bad graph mgr.schedule_graph( repo_name=options.repo_name, revision=options.revision, builders_graph=ast.literal_eval(options.builders_graph), # dry_run=options.dry_run )
def main(): parser = ArgumentParser() parser.add_argument("--debug", action="store_true", dest="debug", help="set debug for logging.") parser.add_argument("--dry-run", action="store_true", dest="dry_run", help="Dry run. No real actions are taken.") parser.add_argument("--repo-name", action="store", dest="repo_name", type=str, help="Repository name, e.g. mozilla-inbound.") parser.add_argument("--revision", action="store", dest="revision", type=str, help="12-char representing a push.") parser.add_argument("--trigger-from-task-id", action="store", dest="trigger_from_task_id", type=str, help="Trigger builders based on build task (use with " "--builders).") parser.add_argument("--builders", action="store", dest="builders", type=str, help="Use this if you want to pass a list of builders " "(e.g. \"['builder 1']\".") parser.add_argument( "--children-of", action="store", dest="children_of", type=str, help="This allows you to request a list of all the associated " "test jobs to a build job.") parser.add_argument("-g", "--graph", action="store", dest="builders_graph", help='Graph of builders in the form of: ' 'dict(builder: [dep_builders].') options = parser.parse_args() if options.debug: setup_logging(logging.DEBUG) else: setup_logging() assert options.repo_name and options.revision, \ "Make sure you specify --repo-name and --revion" if not options.dry_run and not credentials_available(): sys.exit(1) mgr = TaskClusterBuildbotManager() builders = None if options.builders: builders = ast.literal_eval(options.builders) else: builders = get_downstream_jobs(options.children_of) if options.trigger_from_task_id and builders: trigger_builders_based_on_task_id(repo_name=options.repo_name, revision=options.revision, task_id=options.trigger_from_task_id, builders=builders, dry_run=options.dry_run) elif builders and len(builders) == 1: mgr.schedule_arbitrary_job(repo_name=options.repo_name, revision=options.revision, uuid=builders[0], dry_run=options.dry_run) elif options.builders_graph: mgr.schedule_graph(repo_name=options.repo_name, revision=options.revision, builders_graph=ast.literal_eval( options.builders_graph), dry_run=options.dry_run) else: print "Please read the help menu to know what options are available to you."
def on_runnable_job_event(data, message, dry_run, treeherder_host, acknowledge): # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} treeherder_client = TreeherderClient(host='treeherder.allizom.org') # XXX: # Grabbing data received over pulse repo_name = data["project"] requester = data["requester"] resultset_id = data["resultset_id"] buildernames = data["buildernames"] resultset = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0] revision = resultset["revision"] author = resultset["author"] status = None treeherder_link = TREEHERDER % { 'host': treeherder_host, 'repo': repo_name, 'revision': resultset['revision'] } message_sender = MessageHandler() if not (requester.endswith('@mozilla.com') or author == requester or whitelisted_users(requester)): # We want to see this in the alerts LOG.error("Notice that we're letting %s schedule jobs for %s." % (requester, treeherder_link)) ''' # Everyone can press the button, but only authorized users can trigger jobs # TODO: remove this when proper LDAP identication is set up on TH if not (requester.endswith('@mozilla.com') or author == requester or whitelisted_users(requester)): if acknowledge: # Remove message from pulse queue message.ack() # We publish a message saying we will not trigger the job pulse_message = { 'resultset_id': resultset_id, 'requester': requester, 'status': "Could not determine if the user is authorized, nothing was triggered."} routing_key = '{}.{}'.format(repo_name, 'runnable') try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") LOG.error("Requester %s is not allowed to trigger jobs on %s." % (requester, treeherder_link)) return # Raising an exception adds too much noise ''' LOG.info("New jobs requested by %s for %s" % (requester, treeherder_link)) LOG.info("List of builders:") for b in buildernames: LOG.info("- %s" % b) buildernames = filter_invalid_builders(buildernames) # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 if buildernames is None: if acknowledge: # We need to ack the message to remove it from our queue message.ack() return builders_graph, other_builders_to_schedule = buildbot_bridge.buildbot_graph_builder( builders=buildernames, revision=revision, complete=False # XXX: This can be removed when BBB is in use ) if builders_graph != {}: mgr = TaskClusterBuildbotManager() mgr.schedule_graph( repo_name=repo_name, revision=revision, metadata={ 'name': 'pulse_actions_graph', 'description': 'Adding new jobs to push via pulse_actions/treeherder for %s.' % requester, 'owner': requester, 'source': treeherder_link, }, builders_graph=builders_graph, dry_run=dry_run) else: LOG.info("We don't have anything to schedule through TaskCluster") if other_builders_to_schedule: # XXX: We should be able to replace this once all Buildbot jobs run through BBB # XXX: There might be a work around with # https://github.com/mozilla/mozilla_ci_tools/issues/424 LOG.info("We're going to schedule these builders via Buildapi: %s" % str(other_builders_to_schedule)) # This is used for test jobs which need an existing Buildbot job to be scheduled for buildername in other_builders_to_schedule: trigger_job(revision, buildername, dry_run=dry_run) else: LOG.info("We don't have anything to schedule through Buildapi") # Send a pulse message showing what we did message_sender = MessageHandler() pulse_message = { 'resultset_id': resultset_id, 'graph': builders_graph, 'requester': requester, 'status': status } routing_key = '{}.{}'.format(repo_name, 'runnable') try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") if acknowledge: # We need to ack the message to remove it from our queue message.ack()
def on_runnable_job_event(data, message, dry_run, stage): # Cleaning mozci caches buildjson.BUILDS_CACHE = {} query_jobs.JOBS_CACHE = {} if stage: treeherder_client = TreeherderClient(host='treeherder.allizom.org') else: treeherder_client = TreeherderClient() # Grabbing data received over pulse repo_name = data["project"] requester = data["requester"] resultset_id = data["resultset_id"] buildernames = data["buildernames"] resultset = treeherder_client.get_resultsets(repo_name, id=resultset_id)[0] revision = resultset["revision"] author = resultset["author"] status = None treeherder_link = TREEHERDER % {'repo': repo_name, 'revision': resultset['revision']} message_sender = MessageHandler() # Everyone can press the button, but only authorized users can trigger jobs # TODO: remove this when proper LDAP identication is set up on TH if not (requester.endswith('@mozilla.com') or author == requester or whitelisted_users(requester)): if not dry_run: # Remove message from pulse queue message.ack() # We publish a message saying we will not trigger the job pulse_message = { 'resultset_id': resultset_id, 'requester': requester, 'status': "Could not determine if the user is authorized, nothing was triggered."} routing_key = '{}.{}'.format(repo_name, 'runnable') try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") LOG.error("Requester %s is not allowed to trigger jobs on %s." % (requester, treeherder_link)) return # Raising an exception adds too much noise LOG.info("New jobs requested by %s for %s" % (requester, treeherder_link)) LOG.info("List of builders:") for b in buildernames: LOG.info("- %s" % b) buildernames = filter_invalid_builders(buildernames) # Treeherder can send us invalid builder names # https://bugzilla.mozilla.org/show_bug.cgi?id=1242038 if buildernames is None: if not dry_run: # We need to ack the message to remove it from our queue message.ack() return builders_graph, other_builders_to_schedule = buildbot_bridge.buildbot_graph_builder( builders=buildernames, revision=revision, complete=False # XXX: This can be removed when BBB is in use ) if builders_graph != {}: mgr = TaskClusterBuildbotManager() mgr.schedule_graph( repo_name=repo_name, revision=revision, metadata={ 'name': 'pulse_actions_graph', 'description': 'Adding new jobs to push via pulse_actions/treeherder for %s.' % requester, 'owner': requester, 'source': treeherder_link, }, builders_graph=builders_graph, dry_run=dry_run) else: LOG.info("We don't have anything to schedule through TaskCluster") if other_builders_to_schedule: # XXX: We should be able to replace this once all Buildbot jobs run through BBB # XXX: There might be a work around with # https://github.com/mozilla/mozilla_ci_tools/issues/424 LOG.info("We're going to schedule these builders via Buildapi.") # This is used for test jobs which need an existing Buildbot job to be scheduled for buildername in other_builders_to_schedule: trigger_job(revision, buildername, dry_run=dry_run) else: LOG.info("We don't have anything to schedule through Buildapi") # Send a pulse message showing what we did message_sender = MessageHandler() pulse_message = { 'resultset_id': resultset_id, 'graph': builders_graph, 'requester': requester, 'status': status} routing_key = '{}.{}'.format(repo_name, 'runnable') try: message_sender.publish_message(pulse_message, routing_key) except: LOG.warning("Failed to publish message over pulse stream.") if not dry_run: # We need to ack the message to remove it from our queue message.ack()
def main(): parser = ArgumentParser() parser.add_argument("--debug", action="store_true", dest="debug", help="set debug for logging.") parser.add_argument("--dry-run", action="store_true", dest="dry_run", help="Dry run. No real actions are taken.") parser.add_argument("--repo-name", action="store", dest="repo_name", type=str, help="Repository name, e.g. mozilla-inbound.") parser.add_argument("--revision", action="store", dest="revision", type=str, help="12-char representing a push.") parser.add_argument("--trigger-from-task-id", action="store", dest="trigger_from_task_id", type=str, help="Trigger builders based on build task (use with " "--builders).") parser.add_argument("--builders", action="store", dest="builders", type=str, help="Use this if you want to pass a list of builders " "(e.g. \"['builder 1']\".") parser.add_argument("--children-of", action="store", dest="children_of", type=str, help="This allows you to request a list of all the associated " "test jobs to a build job.") parser.add_argument("-g", "--graph", action="store", dest="builders_graph", help='Graph of builders in the form of: ' 'dict(builder: [dep_builders].') options = parser.parse_args() if options.debug: setup_logging(logging.DEBUG) else: setup_logging() assert options.repo_name and options.revision, \ "Make sure you specify --repo-name and --revision" if not options.dry_run and not credentials_available(): sys.exit(1) mgr = TaskClusterBuildbotManager() builders = None if options.builders: builders = ast.literal_eval(options.builders) else: builders = get_downstream_jobs(options.children_of) if options.trigger_from_task_id and builders: trigger_builders_based_on_task_id( repo_name=options.repo_name, revision=options.revision, task_id=options.trigger_from_task_id, builders=builders, dry_run=options.dry_run ) elif builders and len(builders) == 1: mgr.schedule_arbitrary_job( repo_name=options.repo_name, revision=options.revision, uuid=builders[0], dry_run=options.dry_run ) elif options.builders_graph: mgr.schedule_graph( repo_name=options.repo_name, revision=options.revision, builders_graph=ast.literal_eval(options.builders_graph), dry_run=options.dry_run ) else: print "Please read the help menu to know what options are available to you."