def get_all_jobs(self, repo_name, revision, use_cache=True):
"""
Return a list with all jobs for that revision.
If we can't query about this revision in buildapi_client we return an empty list.
"""
if not use_cache:
JOBS_CACHE[(repo_name, revision)] = \
query_jobs_schedule(repo_name, revision, auth=get_credentials())
if (repo_name, revision) not in JOBS_CACHE:
JOBS_CACHE[(repo_name, revision)] = \
query_jobs_schedule(repo_name, revision, auth=get_credentials())
return JOBS_CACHE[(repo_name, revision)]
def get_all_jobs(self, repo_name, revision, use_cache=True):
"""
Return a list with all jobs for that revision.
If we can't query about this revision in buildapi_client we return an empty list.
"""
if not use_cache:
JOBS_CACHE[(repo_name, revision)] = \
query_jobs_schedule(repo_name, revision, auth=get_credentials())
if (repo_name, revision) not in JOBS_CACHE:
JOBS_CACHE[(repo_name, revision)] = \
query_jobs_schedule(repo_name, revision, auth=get_credentials())
return JOBS_CACHE[(repo_name, revision)]
def make_retrigger_request(repo_name, request_id, count=1, priority=0, dry_run=True):
"""
Retrigger a request using buildapi self-serve. Returns a request.
Buildapi documentation:
POST /self-serve/{branch}/request
Rebuild `request_id`, which must be passed in as a POST parameter.
`priority` and `count` are also accepted as optional
parameters. `count` defaults to 1, and represents the number
of times this build will be rebuilt.
"""
url = '{}/{}/request'.format(HOST_ROOT, repo_name)
payload = {'request_id': request_id}
if count != 1 or priority != 0:
payload.update({'count': count,
'priority': priority})
if dry_run:
LOG.info('We would make a POST request to %s with the payload: %s' % (url, str(payload)))
return None
LOG.info("We're going to re-trigger an existing completed job with request_id: %s %i time(s)."
% (request_id, count))
req = requests.post(
url,
headers={'Accept': 'application/json'},
data=payload,
auth=get_credentials()
)
# TODO: add debug message with job_id URL.
return req
def schedule_arbitrary_job(self, repo_name, revision, uuid, *args, **kwargs):
return trigger_arbitrary_job(repo_name=repo_name,
builder=uuid,
revision=revision,
auth=get_credentials(),
*args,
**kwargs)
def valid_credentials():
"""Verify that the user's credentials are valid."""
LOG.debug("Determine if the user's credentials are valid.")
req = requests.get(HOST_ROOT, auth=get_credentials())
if req.status_code == 401:
remove_credentials()
raise AuthenticationError("Your credentials were invalid. Please try again.")
def make_request(repo_name, builder, revision, files=[], dry_run=False,
extra_properites=None):
"""
Request buildapi to trigger a job for us.
We return the request or None if dry_run is True.
"""
url = _builders_api_url(repo_name, builder, revision)
payload = _payload(repo_name, revision, files, extra_properites)
if dry_run:
LOG.info("Dry-run: We were going to request a job for '%s'" % builder)
LOG.info(" with this payload: %s" % str(payload))
return None
# NOTE: A good response returns json with request_id as one of the keys
req = requests.post(
url,
headers={'Accept': 'application/json'},
data=payload,
auth=get_credentials()
)
assert req.status_code != 401, req.reason
content = req.json()
LOG.debug("Status of the request: %s" %
_jobs_api_url(content["request_id"]))
return req
def cancel(self, uuid, *args, **kwargs):
return make_cancel_request(
repo_name=kwargs['repo_name'],
request_id=uuid,
auth=get_credentials(),
*args,
**kwargs)
def make_retrigger_request(repo_name,
request_id,
count=1,
priority=0,
dry_run=True):
"""
Retrigger a request using buildapi self-serve. Returns a request.
Buildapi documentation:
POST /self-serve/{branch}/request
Rebuild `request_id`, which must be passed in as a POST parameter.
`priority` and `count` are also accepted as optional
parameters. `count` defaults to 1, and represents the number
of times this build will be rebuilt.
"""
url = '{}/{}/request'.format(HOST_ROOT, repo_name)
payload = {'request_id': request_id}
if count != 1 or priority != 0:
payload.update({'count': count, 'priority': priority})
if dry_run:
LOG.info('We would make a POST request to %s with the payload: %s' %
(url, str(payload)))
return None
LOG.info(
"We're going to re-trigger an existing completed job with request_id: %s %i time(s)."
% (request_id, count))
req = requests.post(url,
headers={'Accept': 'application/json'},
data=payload,
auth=get_credentials())
# TODO: add debug message with job_id URL.
return req
def valid_revision(repo_name, revision):
"""
There are revisions that won't exist in buildapi.
This happens on pushes that do not have any jobs scheduled for them.
"""
global VALID_CACHE
if (repo_name, revision) in VALID_CACHE:
return VALID_CACHE[(repo_name, revision)]
LOG.debug("Determine if the revision is valid in buildapi.")
url = "%s/%s/rev/%s?format=json" % (HOST_ROOT, repo_name, revision)
req = requests.get(url, auth=get_credentials())
if req.status_code == 401:
LOG.critical("Your credentials were invalid")
exit(1)
content = json.loads(req.content)
ret = True
if isinstance(content, dict):
failure_message = "Revision %s not found on branch %s" % (revision, repo_name)
if content["msg"] == failure_message:
LOG.warning(failure_message)
ret = False
VALID_CACHE[(repo_name, revision)] = ret
return ret
def trigger(builder,
revision,
files=None,
dry_run=False,
extra_properties=None):
"""Helper to trigger a job.
Returns a request.
"""
_add_builder_to_scheduling_manager(revision=revision, buildername=builder)
repo_name = query_repo_name_from_buildername(builder)
if is_downstream(builder) and not files:
raise MozciError(
'We have requested to trigger a test job, however, we have not provided '
'which files to run against.')
return trigger_arbitrary_job(repo_name=repo_name,
builder=builder,
revision=revision,
auth=get_credentials(),
files=files,
dry_run=dry_run,
extra_properties=extra_properties)
def trigger_arbitrary_job(repo_name, builder, revision, files=[], dry_run=False,
extra_properties=None):
"""
Request buildapi to trigger a job for us.
We return the request or None if dry_run is True.
"""
url = _builders_api_url(repo_name, builder, revision)
payload = _payload(repo_name, revision, files, extra_properties)
if dry_run:
LOG.info("Dry-run: We were going to request a job for '%s'" % builder)
LOG.info(" with this payload: %s" % str(payload))
return None
# NOTE: A good response returns json with request_id as one of the keys
req = requests.post(
url,
headers={'Accept': 'application/json'},
data=payload,
auth=get_credentials()
)
if req.status_code == 401:
remove_credentials()
raise AuthenticationError("Your credentials were invalid. Please try again.")
content = req.json()
LOG.debug("Status of the request: %s" %
_jobs_api_url(content["request_id"]))
return req
def schedule_arbitrary_job(self, repo_name, revision, uuid, *args,
**kwargs):
return trigger_arbitrary_job(repo_name=repo_name,
builder=uuid,
revision=revision,
auth=get_credentials(),
*args,
**kwargs)
def _all_urls_reachable(urls):
"""Determine if the URLs are reachable."""
for url in urls:
url_tested = _public_url(url)
LOG.debug("We are going to test if we can reach %s" % url_tested)
req = requests.head(url_tested, auth=get_credentials())
if not req.ok:
LOG.warning("We can't reach %s for this reason %s" %
(url_tested, req.reason))
return False
return True
def _all_urls_reachable(urls):
"""Determine if the URLs are reachable."""
for url in urls:
url_tested = _public_url(url)
LOG.debug("We are going to test if we can reach %s" % url_tested)
req = requests.head(url_tested, auth=get_credentials())
if not req.ok:
LOG.warning("We can't reach %s for this reason %s" %
(url, req.reason))
return False
return True
def valid_credentials():
"""
Verify that the user's credentials are valid.
Raises an AuthenticationError if the credentials are invalid.
"""
LOG.debug("Determine if the user's credentials are valid.")
req = requests.get(HOST_ROOT, auth=get_credentials())
if req.status_code == 401:
remove_credentials()
raise AuthenticationError(
"Your credentials were invalid. Please try again.")
def trigger(builder, revision, files=[], dry_run=False, extra_properties=None):
"""Helper to trigger a job.
Returns a request.
"""
_add_builder_to_scheduling_manager(revision=revision, buildername=builder)
repo_name = query_repo_name_from_buildername(builder)
return trigger_arbitrary_job(repo_name=repo_name,
builder=builder,
revision=revision,
auth=get_credentials(),
files=files,
dry_run=dry_run,
extra_properties=extra_properties)
def trigger(builder, revision, files=[], dry_run=False, extra_properties=None):
"""Helper to trigger a job.
Returns a request.
"""
_add_builder_to_scheduling_manager(revision=revision, buildername=builder)
repo_name = query_repo_name_from_buildername(builder)
return trigger_arbitrary_job(repo_name=repo_name,
builder=builder,
revision=revision,
auth=get_credentials(),
files=files,
dry_run=dry_run,
extra_properties=extra_properties)
def get_jobs(rev):
"""Get all jobs that ran in a revision."""
url = "https://secure.pub.build.mozilla.org/buildapi/self-serve/try/rev/%s?format=json" % rev
LOG.debug("About to fetch %s" % url)
req = requests.get(url, auth=get_credentials())
if req.status_code not in [200]:
raise Exception
data = req.json()
jobs = []
for build in data:
jobs.append((build["buildername"], build.get("status")))
return jobs
def query_jobs_schedule(repo_name, revision):
""" Query Buildapi for jobs """
repo_url = query_repo_url(repo_name)
if not pushlog.valid_revision(repo_url, revision):
raise BuildapiException
url = "%s/%s/rev/%s?format=json" % (HOST_ROOT, repo_name, revision)
LOG.debug("About to fetch %s" % url)
req = requests.get(url, auth=get_credentials())
# If the revision doesn't exist on buildapi, that means there are
# no builapi jobs for this revision
if req.status_code not in [200]:
return []
return req.json()
def make_cancel_request(repo_name, request_id, dry_run=True):
"""
Cancel a request using buildapi self-serve. Returns a request.
Buildapi documentation:
DELETE /self-serve/{branch}/request/{request_id} Cancel the given request
"""
url = '{}/{}/request/{}'.format(HOST_ROOT, repo_name, request_id)
if dry_run:
LOG.info('We would make a DELETE request to %s.' % url)
return None
LOG.info("We're going to cancel the job at %s" % url)
req = requests.delete(url, auth=get_credentials())
# TODO: add debug message with the canceled job_id URL. Find a way
# to do that without doing an additional request.
return req
def valid_revision(repo_name, revision):
"""
There are revisions that won't exist in buildapi.
This happens on pushes that do not have any jobs scheduled for them.
"""
LOG.debug("Determine if the revision is valid in buildapi.")
url = "%s/%s/rev/%s?format=json" % (HOST_ROOT, repo_name, revision)
req = requests.get(url, auth=get_credentials())
content = json.loads(req.content)
if isinstance(content, dict):
failure_message = "Revision %s not found on branch %s" % (revision, repo_name)
if content["msg"] == failure_message:
LOG.warning(failure_message)
return False
else:
return True
def query_jobs_schedule(repo_name, revision):
"""
Return a list with all jobs for that revision.
If we can't query about this revision in buildapi we return an empty list.
raises BuildapiException
"""
if not valid_revision(repo_name, revision):
raise BuildapiException
url = "%s/%s/rev/%s?format=json" % (HOST_ROOT, repo_name, revision)
LOG.debug("About to fetch %s" % url)
req = requests.get(url, auth=get_credentials())
assert req.status_code in [200], req.content
return req.json()
def make_cancel_request(repo_name, request_id, dry_run=True):
"""
Cancel a request using buildapi self-serve. Returns a request.
Buildapi documentation:
DELETE /self-serve/{branch}/request/{request_id} Cancel the given request
"""
url = '{}/{}/request/{}'.format(HOST_ROOT, repo_name, request_id)
if dry_run:
LOG.info('We would make a DELETE request to %s.' % url)
return None
LOG.info("We're going to cancel the job at %s" % url)
req = requests.delete(url, auth=get_credentials())
# TODO: add debug message with the canceled job_id URL. Find a way
# to do that without doing an additional request.
return req
def query_repositories(clobber=False):
"""
Return dictionary with information about the various repositories.
The data about a repository looks like this:
.. code-block:: python
"ash": {
"repo": "https://hg.mozilla.org/projects/ash",
"graph_branches": ["Ash"],
"repo_type": "hg"
}
Raises an AuthenticationError if the user credentials are invalid.
"""
global REPOSITORIES
if clobber:
REPOSITORIES = {}
if os.path.exists(REPOSITORIES_FILE):
os.remove(REPOSITORIES_FILE)
if REPOSITORIES:
return REPOSITORIES
if os.path.exists(REPOSITORIES_FILE):
LOG.debug("Loading %s" % REPOSITORIES_FILE)
fd = open(REPOSITORIES_FILE)
REPOSITORIES = json.load(fd)
else:
url = "%s/branches?format=json" % HOST_ROOT
LOG.debug("About to fetch %s" % url)
req = requests.get(url, auth=get_credentials())
if req.status_code == 401:
remove_credentials()
raise AuthenticationError(
"Your credentials were invalid. Please try again.")
REPOSITORIES = req.json()
with open(REPOSITORIES_FILE, "wb") as fd:
json.dump(REPOSITORIES, fd)
return REPOSITORIES
def query_repositories(clobber=False):
"""
Return dictionary with information about the various repositories.
The data about a repository looks like this:
.. code-block:: python
"ash": {
"repo": "https://hg.mozilla.org/projects/ash",
"graph_branches": ["Ash"],
"repo_type": "hg"
}
Raises an AuthenticationError if the user credentials are invalid.
"""
global REPOSITORIES
if clobber:
REPOSITORIES = {}
if os.path.exists(REPOSITORIES_FILE):
os.remove(REPOSITORIES_FILE)
if REPOSITORIES:
return REPOSITORIES
if os.path.exists(REPOSITORIES_FILE):
LOG.debug("Loading %s" % REPOSITORIES_FILE)
fd = open(REPOSITORIES_FILE)
REPOSITORIES = json.load(fd)
else:
try:
REPOSITORIES = make_query_repositories_request(
auth=get_credentials(), dry_run=False)
except BuildapiAuthError:
remove_credentials()
raise AuthenticationError(
"Your credentials were invalid. Please try again.")
with open(REPOSITORIES_FILE, "wb") as fd:
json.dump(REPOSITORIES, fd)
return REPOSITORIES
def query_repositories(clobber=False):
"""
Return dictionary with information about the various repositories.
The data about a repository looks like this:
.. code-block:: python
"ash": {
"repo": "https://hg.mozilla.org/projects/ash",
"graph_branches": ["Ash"],
"repo_type": "hg"
}
Raises an AuthenticationError if the user credentials are invalid.
"""
global REPOSITORIES
if clobber:
REPOSITORIES = {}
if os.path.exists(REPOSITORIES_FILE):
os.remove(REPOSITORIES_FILE)
if REPOSITORIES:
return REPOSITORIES
if os.path.exists(REPOSITORIES_FILE):
LOG.debug("Loading %s" % REPOSITORIES_FILE)
fd = open(REPOSITORIES_FILE)
REPOSITORIES = json.load(fd)
else:
url = "%s/branches?format=json" % HOST_ROOT
LOG.debug("About to fetch %s" % url)
req = requests.get(url, auth=get_credentials())
if req.status_code == 401:
remove_credentials()
raise AuthenticationError("Your credentials were invalid. Please try again.")
REPOSITORIES = req.json()
with open(REPOSITORIES_FILE, "wb") as fd:
json.dump(REPOSITORIES, fd)
return REPOSITORIES
def query_jobs_schedule(repo_name, revision):
"""
Query Buildapi for jobs.
Raises a BuildapiError if the revision doesn't exist in repo_name.
"""
repo_url = query_repo_url(repo_name)
if VALIDATE and not pushlog.valid_revision(repo_url, revision):
raise BuildapiError
url = "%s/%s/rev/%s?format=json" % (HOST_ROOT, repo_name, revision)
LOG.debug("About to fetch %s" % url)
req = requests.get(url, auth=get_credentials())
# If the revision doesn't exist on buildapi, that means there are
# no builapi jobs for this revision
if req.status_code not in [200]:
return []
return req.json()
def trigger(builder, revision, files=None, dry_run=False, extra_properties=None):
"""Helper to trigger a job.
Returns a request.
"""
_add_builder_to_scheduling_manager(revision=revision, buildername=builder)
repo_name = query_repo_name_from_buildername(builder)
if is_downstream(builder) and not files:
raise MozciError('We have requested to trigger a test job, however, we have not provided '
'which files to run against.')
return trigger_arbitrary_job(repo_name=repo_name,
builder=builder,
revision=revision,
auth=get_credentials(),
files=files,
dry_run=dry_run,
extra_properties=extra_properties)
def trigger_arbitrary_job(repo_name,
builder,
revision,
files=[],
dry_run=False,
extra_properties=None):
"""
Request buildapi to trigger a job for us.
We return the request or None if dry_run is True.
Raises AuthenticationError if credentials are invalid.
"""
url = _builders_api_url(repo_name, builder, revision)
payload = _payload(repo_name, revision, files, extra_properties)
if dry_run:
LOG.info("Dry-run: We were going to request a job for '%s'" % builder)
LOG.info(" with this payload: %s" % str(payload))
return None
# NOTE: A good response returns json with request_id as one of the keys
req = requests.post(url,
headers={'Accept': 'application/json'},
data=payload,
auth=get_credentials())
if req.status_code == 401:
remove_credentials()
raise AuthenticationError(
"Your credentials were invalid. Please try again.")
try:
content = req.json()
LOG.debug("Status of the request: %s" %
_jobs_api_url(content["request_id"]))
return req
except ValueError:
LOG.warning("We did not get info from %s (status code: %s)" %
(url, req.status_code))
return None
def query_repositories(clobber=False):
"""
Return dictionary with information about the various repositories.
The data about a repository looks like this:
.. code-block:: python
"ash": {
"repo": "https://hg.mozilla.org/projects/ash",
"graph_branches": ["Ash"],
"repo_type": "hg"
}
Raises an AuthenticationError if the user credentials are invalid.
"""
global REPOSITORIES
if clobber:
REPOSITORIES = {}
if os.path.exists(REPOSITORIES_FILE):
os.remove(REPOSITORIES_FILE)
if REPOSITORIES:
return REPOSITORIES
if os.path.exists(REPOSITORIES_FILE):
LOG.debug("Loading %s" % REPOSITORIES_FILE)
fd = open(REPOSITORIES_FILE)
REPOSITORIES = json.load(fd)
else:
try:
REPOSITORIES = make_query_repositories_request(auth=get_credentials(), dry_run=False)
except BuildapiAuthError:
remove_credentials()
raise AuthenticationError("Your credentials were invalid. Please try again.")
with open(REPOSITORIES_FILE, "wb") as fd:
json.dump(REPOSITORIES, fd)
return REPOSITORIES
def trigger(builder, revision, files=[], dry_run=False, extra_properties=None):
"""Helper to trigger a job.
Returns a request.
"""
global SCHEDULING_MANAGER
sch_mgr = SCHEDULING_MANAGER
if revision not in sch_mgr:
sch_mgr[revision] = []
sch_mgr[revision].append(builder)
repo_name = query_repo_name_from_buildername(builder)
return trigger_arbitrary_job(repo_name=repo_name,
builder=builder,
revision=revision,
auth=get_credentials(),
files=files,
dry_run=dry_run,
extra_properties=extra_properties)
def trigger(builder, revision, files=[], dry_run=False, extra_properties=None):
"""Helper to trigger a job.
Returns a request.
"""
global SCHEDULING_MANAGER
sch_mgr = SCHEDULING_MANAGER
if revision not in sch_mgr:
sch_mgr[revision] = []
sch_mgr[revision].append(builder)
repo_name = query_repo_name_from_buildername(builder)
return trigger_arbitrary_job(repo_name=repo_name,
builder=builder,
revision=revision,
auth=get_credentials(),
files=files,
dry_run=dry_run,
extra_properties=extra_properties)
def trigger_range(buildername,
revisions,
times=1,
dry_run=False,
files=None,
extra_properties=None,
trigger_build_if_missing=True):
"""Schedule the job named "buildername" ("times" times) in every revision on 'revisions'."""
repo_name = query_repo_name_from_buildername(buildername)
repo_url = repositories.query_repo_url(repo_name)
if revisions != []:
LOG.info("We want to have %s job(s) of %s on revisions %s" %
(times, buildername, str(revisions)))
for rev in revisions:
LOG.info("")
LOG.info("=== %s ===" % rev)
if VALIDATE and not pushlog.valid_revision(repo_url, rev):
LOG.info(
"We can't trigger anything on pushes without a valid revision."
)
continue
LOG.info("We want to have %s job(s) of %s on revision %s" %
(times, buildername, rev))
# 1) How many potentially completed jobs can we get for this buildername?
matching_jobs = QUERY_SOURCE.get_matching_jobs(repo_name, rev,
buildername)
successful_jobs, pending_jobs, running_jobs, _, failed_jobs = \
_status_summary(matching_jobs)
potential_jobs = pending_jobs + running_jobs + successful_jobs + failed_jobs
# TODO: change this debug message when we have a less hardcoded _status_summary
LOG.debug("We found %d pending/running jobs, %d successful jobs and "
"%d failed jobs" %
(pending_jobs + running_jobs, successful_jobs, failed_jobs))
if potential_jobs >= times:
LOG.info(
"We have %d job(s) for '%s' which is enough for the %d job(s) we want."
% (potential_jobs, buildername, times))
else:
# 2) If we have less potential jobs than 'times' instances then
# we need to fill it in.
LOG.info("We have found %d potential job(s) matching '%s' on %s. "
"We need to trigger more." %
(potential_jobs, buildername, rev))
# If a job matching what we want already exists, we can
# use the retrigger API in self-serve to retrigger that
# instead of creating a new arbitrary job
if len(matching_jobs) > 0 and files is None:
request_id = QUERY_SOURCE.get_buildapi_request_id(
repo_name, matching_jobs[0])
make_retrigger_request(repo_name=repo_name,
request_id=request_id,
auth=get_credentials(),
count=(times - potential_jobs),
dry_run=dry_run)
# If no matching job exists, we have to trigger a new arbitrary job
else:
list_of_requests = trigger_job(
revision=rev,
buildername=buildername,
times=(times - potential_jobs),
dry_run=dry_run,
files=files,
extra_properties=extra_properties,
trigger_build_if_missing=trigger_build_if_missing)
if list_of_requests and any(req.status_code != 202
for req in list_of_requests):
LOG.warning("Not all requests succeeded.")
def main():
options = parse_args()
if options.debug:
LOG = setup_logging(logging.DEBUG)
else:
LOG = setup_logging(logging.INFO)
if options.action == 'trigger-all-talos':
trigger_all_talos_jobs(options.repo_name, options.rev, options.times,
dry_run=options.dry_run)
sys.exit(0)
validate_options(options)
if not options.dry_run and not valid_credentials():
sys.exit(-1)
# Setting the QUERY_SOURCE global variable in mozci.py
set_query_source(options.query_source)
if options.buildernames:
options.buildernames = sanitize_buildernames(options.buildernames)
repo_url = query_repo_url_from_buildername(options.buildernames[0])
if not options.repo_name:
repo_name = query_repo_name_from_buildername(options.buildernames[0])
else:
repo_name = options.repo_name
repo_url = query_repo_url(repo_name)
if options.rev == 'tip':
revision = query_repo_tip(repo_url).changesets[0].node
LOG.info("The tip of %s is %s", repo_name, revision)
else:
revision = query_push_by_revision(repo_url, options.rev,
return_revision_list=True)
# Schedule jobs through TaskCluster if --taskcluster option has been set to true
if options.taskcluster:
mgr = TaskClusterBuildbotManager(web_auth=True)
else:
mgr = BuildAPIManager()
trigger_build_if_missing = options.trigger_build_if_missing
if repo_name == 'try':
trigger_build_if_missing = False
# Mode 0: Backfill
if options.backfill:
manual_backfill(revision, options.buildernames[0], dry_run=options.dry_run)
return
# Mode 1: Trigger coalesced jobs
if options.coalesced:
query_api = BuildApi()
request_ids = query_api.find_all_jobs_by_status(repo_name,
revision, COALESCED)
if len(request_ids) == 0:
LOG.info('We did not find any coalesced job')
for request_id in request_ids:
make_retrigger_request(repo_name=repo_name,
request_id=request_id,
auth=get_credentials(),
dry_run=options.dry_run)
return
# Mode #2: Fill-in a revision or trigger_test_jobs_only
if options.fill_revision or options.trigger_tests_only:
mgr.trigger_missing_jobs_for_revision(
repo_name=repo_name,
revision=revision,
dry_run=options.dry_run,
trigger_build_if_missing=not options.trigger_tests_only
)
return
# Mode #3: Trigger jobs based on revision list modifiers
if not (options.includes or options.exclude or options.failed_jobs or
options.trigger_talos_for_build):
job_names = options.buildernames
# Mode 4 - Schedule every builder matching --includes and does not match --exclude.
elif options.includes or options.exclude:
_includes_excludes(options)
# Mode 5: Use --failed-jobs to trigger jobs for particular revision
elif options.failed_jobs:
job_names = TreeherderApi().find_all_jobs_by_status(
repo_name=repo_name,
revision=revision,
status=WARNING)
elif options.trigger_talos_for_build:
trigger_talos_jobs_for_build(
buildername=options.buildernames[0],
revision=revision,
times=2,
dry_run=options.dry_run,
)
exit(0)
for buildername in job_names:
revlist = determine_revlist(
repo_url=repo_url,
buildername=buildername,
rev=revision,
back_revisions=options.back_revisions,
delta=options.delta,
from_rev=options.from_rev,
backfill=options.backfill,
skips=options.skips,
max_revisions=options.max_revisions)
_print_treeherder_link(
revlist=revlist,
repo_name=repo_name,
buildername=buildername,
revision=revision,
log=LOG,
includes=options.includes,
exclude=options.exclude)
try:
mgr.trigger_range(
buildername=buildername,
repo_name=repo_name,
revisions=revlist,
times=options.times,
dry_run=options.dry_run,
files=options.files,
trigger_build_if_missing=trigger_build_if_missing
)
except Exception as e:
LOG.exception(e)
exit(1)
def main():
options = parse_args()
if options.debug:
LOG = setup_logging(logging.DEBUG)
else:
LOG = setup_logging(logging.INFO)
if options.action == 'trigger-all-talos':
trigger_all_talos_jobs(options.repo_name,
options.rev,
options.times,
dry_run=options.dry_run)
sys.exit(0)
validate_options(options)
if not options.dry_run and not valid_credentials():
sys.exit(-1)
# Setting the QUERY_SOURCE global variable in mozci.py
set_query_source(options.query_source)
if options.buildernames:
options.buildernames = sanitize_buildernames(options.buildernames)
repo_url = query_repo_url_from_buildername(options.buildernames[0])
if not options.repo_name:
repo_name = query_repo_name_from_buildername(options.buildernames[0])
else:
repo_name = options.repo_name
repo_url = query_repo_url(repo_name)
if options.rev == 'tip':
revision = query_repo_tip(repo_url).changesets[0].node
LOG.info("The tip of %s is %s", repo_name, revision)
else:
revision = query_push_by_revision(repo_url,
options.rev,
return_revision_list=True)
# Schedule jobs through TaskCluster if --taskcluster option has been set to true
if options.taskcluster:
mgr = TaskClusterBuildbotManager(web_auth=True)
else:
mgr = BuildAPIManager()
trigger_build_if_missing = options.trigger_build_if_missing
if repo_name == 'try':
trigger_build_if_missing = False
# Mode 0: Backfill
if options.backfill:
manual_backfill(revision,
options.buildernames[0],
dry_run=options.dry_run)
return
# Mode 1: Trigger coalesced jobs
if options.coalesced:
query_api = BuildApi()
request_ids = query_api.find_all_jobs_by_status(
repo_name, revision, COALESCED)
if len(request_ids) == 0:
LOG.info('We did not find any coalesced job')
for request_id in request_ids:
make_retrigger_request(repo_name=repo_name,
request_id=request_id,
auth=get_credentials(),
dry_run=options.dry_run)
return
# Mode #2: Fill-in a revision or trigger_test_jobs_only
if options.fill_revision or options.trigger_tests_only:
mgr.trigger_missing_jobs_for_revision(
repo_name=repo_name,
revision=revision,
dry_run=options.dry_run,
trigger_build_if_missing=not options.trigger_tests_only)
return
# Mode #3: Trigger jobs based on revision list modifiers
if not (options.includes or options.exclude or options.failed_jobs
or options.trigger_talos_for_build):
job_names = options.buildernames
# Mode 4 - Schedule every builder matching --includes and does not match --exclude.
elif options.includes or options.exclude:
_includes_excludes(options)
# Mode 5: Use --failed-jobs to trigger jobs for particular revision
elif options.failed_jobs:
job_names = TreeherderApi().find_all_jobs_by_status(
repo_name=repo_name, revision=revision, status=WARNING)
elif options.trigger_talos_for_build:
trigger_talos_jobs_for_build(
buildername=options.buildernames[0],
revision=revision,
times=2,
dry_run=options.dry_run,
)
exit(0)
for buildername in job_names:
revlist = determine_revlist(repo_url=repo_url,
buildername=buildername,
rev=revision,
back_revisions=options.back_revisions,
delta=options.delta,
from_rev=options.from_rev,
backfill=options.backfill,
skips=options.skips,
max_revisions=options.max_revisions)
_print_treeherder_link(revlist=revlist,
repo_name=repo_name,
buildername=buildername,
revision=revision,
log=LOG,
includes=options.includes,
exclude=options.exclude)
try:
mgr.trigger_range(
buildername=buildername,
repo_name=repo_name,
revisions=revlist,
times=options.times,
dry_run=options.dry_run,
files=options.files,
trigger_build_if_missing=trigger_build_if_missing)
except Exception as e:
LOG.exception(e)
exit(1)
def trigger_range(buildername, revisions, times=1, dry_run=False,
files=None, extra_properties=None, trigger_build_if_missing=True):
"""Schedule the job named "buildername" ("times" times) in every revision on 'revisions'."""
repo_name = query_repo_name_from_buildername(buildername)
repo_url = repositories.query_repo_url(repo_name)
if revisions != []:
LOG.info("We want to have %s job(s) of %s on the following revisions: "
% (times, buildername))
for r in revisions:
LOG.info(" - %s" % r)
for rev in revisions:
LOG.info("")
LOG.info("=== %s ===" % rev)
if VALIDATE and not valid_revision(repo_url, rev):
LOG.info("We can't trigger anything on pushes without a valid revision.")
continue
LOG.info("We want to have %s job(s) of %s" % (times, buildername))
# 1) How many potentially completed jobs can we get for this buildername?
matching_jobs = QUERY_SOURCE.get_matching_jobs(repo_name, rev, buildername)
status_summary = StatusSummary(matching_jobs)
# TODO: change this debug message when we have a less hardcoded _status_summary
LOG.debug("We found %d pending/running jobs, %d successful jobs and "
"%d failed jobs" % (status_summary.pending_jobs + status_summary.running_jobs,
status_summary.successful_jobs, status_summary.failed_jobs))
if status_summary.potential_jobs >= times:
LOG.info("We have %d job(s) for '%s' which is enough for the %d job(s) we want." %
(status_summary.potential_jobs, buildername, times))
else:
# 2) If we have less potential jobs than 'times' instances then
# we need to fill it in.
LOG.info("We have found %d potential job(s) matching '%s' on %s. "
"We need to trigger more." % (status_summary.potential_jobs, buildername, rev))
# If a job matching what we want already exists, we can
# use the retrigger API in self-serve to retrigger that
# instead of creating a new arbitrary job
if len(matching_jobs) > 0 and files is None:
request_id = QUERY_SOURCE.get_buildapi_request_id(repo_name, matching_jobs[0])
make_retrigger_request(
repo_name=repo_name,
request_id=request_id,
auth=get_credentials(),
count=(times - status_summary.potential_jobs),
dry_run=dry_run)
# If no matching job exists, we have to trigger a new arbitrary job
else:
list_of_requests = trigger_job(
revision=rev,
buildername=buildername,
times=(times - status_summary.potential_jobs),
dry_run=dry_run,
files=files,
extra_properties=extra_properties,
trigger_build_if_missing=trigger_build_if_missing)
if list_of_requests and any(req.status_code != 202 for req in list_of_requests):
LOG.warning("Not all requests succeeded.")
def trigger_range(buildername,
revisions,
times=1,
dry_run=False,
files=None,
extra_properties=None,
trigger_build_if_missing=True):
"""Schedule the job named "buildername" ("times" times) in every revision on 'revisions'."""
repo_name = query_repo_name_from_buildername(buildername)
repo_url = repositories.query_repo_url(repo_name)
if revisions != []:
LOG.info(
"We want to have %s job(s) of %s on the following revisions: " %
(times, buildername))
for r in revisions:
LOG.info(" - %s" % r)
for rev in revisions:
LOG.info("")
LOG.info("=== %s ===" % rev)
if VALIDATE and not valid_revision(repo_url, rev):
LOG.info(
"We can't trigger anything on pushes without a valid revision."
)
continue
# 1) How many potentially completed jobs can we get for this buildername?
matching_jobs = QUERY_SOURCE.get_matching_jobs(repo_name, rev,
buildername)
status_summary = StatusSummary(matching_jobs)
# TODO: change this debug message when we have a less hardcoded _status_summary
LOG.debug("We found %d pending/running jobs, %d successful jobs and "
"%d failed jobs" %
(status_summary.pending_jobs + status_summary.running_jobs,
status_summary.successful_jobs, status_summary.failed_jobs))
if status_summary.potential_jobs >= times:
LOG.info(
"We have %d job(s) for '%s' which is enough for the %d job(s) we want."
% (status_summary.potential_jobs, buildername, times))
else:
# 2) If we have less potential jobs than 'times' instances then
# we need to fill it in.
LOG.info("We have found %d potential job(s) matching '%s' on %s. "
"We need to trigger more." %
(status_summary.potential_jobs, buildername, rev))
schedule_new_job = True
# If a job matching what we want already exists, we can
# use the retrigger API in self-serve to retrigger that
# instead of creating a new arbitrary job
if len(matching_jobs) > 0 and files is None:
try:
request_id = QUERY_SOURCE.get_buildapi_request_id(
repo_name, matching_jobs[0])
make_retrigger_request(
repo_name=repo_name,
request_id=request_id,
auth=get_credentials(),
count=(times - status_summary.potential_jobs),
dry_run=dry_run)
schedule_new_job = False
except (IndexError, ConnectionError, ReadTimeout,
ValueError) as e:
# Logging until we can determine why we get these errors
# We should have one of these:
# {'requests': [{'request_id': int]}
# {'request_id': int}
LOG.info(matching_jobs[0])
LOG.info(str(e))
LOG.warning("We failed to retrigger the job, however, "
"we will try to schedule a new one.")
# If no matching job exists, we have to trigger a new arbitrary job
if schedule_new_job:
list_of_requests = trigger_job(
revision=rev,
buildername=buildername,
times=(times - status_summary.potential_jobs),
dry_run=dry_run,
files=files,
extra_properties=extra_properties,
trigger_build_if_missing=trigger_build_if_missing)
if list_of_requests and any(req.status_code != 202
for req in list_of_requests):
LOG.warning("Not all requests succeeded.")
def retrigger_build(self, uuid, *args, **kwargs):
return make_retrigger_build_request(build_id=uuid,
auth=get_credentials(),
*args,
**kwargs)
def main():
options = parse_args()
validate_options(options)
repo_url = query_repo_url(options.repo_name)
if not valid_credentials():
sys.exit(-1)
if options.debug:
LOG = setup_logging(logging.DEBUG)
else:
LOG = setup_logging(logging.INFO)
# Setting the QUERY_SOURCE global variable in mozci.py
set_query_source(options.query_source)
if options.buildernames:
options.buildernames = sanitize_buildernames(options.buildernames)
repo_url = query_repo_url_from_buildername(options.buildernames[0])
if not options.repo_name:
options.repo_name = query_repo_name_from_buildername(
options.buildernames[0])
if options.rev == 'tip':
revision = query_repo_tip(repo_url)
LOG.info("The tip of %s is %s", options.repo_name, revision)
else:
revision = query_full_revision_info(repo_url, options.rev)
# Mode 1: Trigger coalesced jobs
if options.coalesced:
query_api = BuildApi()
request_ids = query_api.find_all_jobs_by_status(
options.repo_name, revision, COALESCED)
if len(request_ids) == 0:
LOG.info('We did not find any coalesced job')
for request_id in request_ids:
make_retrigger_request(repo_name=options.repo_name,
request_id=request_id,
auth=get_credentials(),
dry_run=options.dry_run)
return
# Mode #2: Fill-in a revision or trigger_test_jobs_only
if options.fill_revision or options.trigger_tests_only:
BuildAPIManager().trigger_missing_jobs_for_revision(
repo_name=options.repo_name,
revision=revision,
dry_run=options.dry_run,
trigger_build_if_missing=not options.trigger_tests_only)
return
# Mode #3: Trigger jobs based on revision list modifiers
for buildername in options.buildernames:
revlist = determine_revlist(repo_url=repo_url,
buildername=buildername,
rev=revision,
back_revisions=options.back_revisions,
delta=options.delta,
from_rev=options.from_rev,
backfill=options.backfill,
skips=options.skips,
max_revisions=options.max_revisions)
try:
trigger_range(
buildername=buildername,
revisions=revlist,
times=options.times,
dry_run=options.dry_run,
files=options.files,
trigger_build_if_missing=options.trigger_build_if_missing)
except Exception, e:
LOG.exception(e)
exit(1)
if revlist:
LOG.info('https://treeherder.mozilla.org/#/jobs?%s' %
urllib.urlencode({
'repo': options.repo_name,
'fromchange': revlist[-1],
'tochange': revlist[0],
'filter-searchStr': buildername
}))
def main():
options = parse_args()
if options.debug:
LOG = setup_logging(logging.DEBUG)
else:
LOG = setup_logging(logging.INFO)
validate_options(options)
if not valid_credentials():
sys.exit(-1)
# Setting the QUERY_SOURCE global variable in mozci.py
set_query_source(options.query_source)
if options.buildernames:
options.buildernames = sanitize_buildernames(options.buildernames)
repo_url = query_repo_url_from_buildername(options.buildernames[0])
if not options.repo_name:
repo_name = query_repo_name_from_buildername(options.buildernames[0])
else:
repo_name = options.repo_name
repo_url = query_repo_url(repo_name)
if options.rev == 'tip':
revision = query_repo_tip(repo_url).changesets[0].node
LOG.info("The tip of %s is %s", repo_name, revision)
else:
revision = query_push_by_revision(repo_url, options.rev,
return_revision_list=True)
# Schedule jobs through TaskCluster if --taskcluster option has been set to true
if options.taskcluster:
mgr = TaskClusterBuildbotManager()
else:
mgr = BuildAPIManager()
trigger_build_if_missing = options.trigger_build_if_missing
if repo_name == 'try':
trigger_build_if_missing = False
# Mode 1: Trigger coalesced jobs
if options.coalesced:
query_api = BuildApi()
request_ids = query_api.find_all_jobs_by_status(repo_name,
revision, COALESCED)
if len(request_ids) == 0:
LOG.info('We did not find any coalesced job')
for request_id in request_ids:
make_retrigger_request(repo_name=repo_name,
request_id=request_id,
auth=get_credentials(),
dry_run=options.dry_run)
return
# Mode #2: Fill-in a revision or trigger_test_jobs_only
if options.fill_revision or options.trigger_tests_only:
mgr.trigger_missing_jobs_for_revision(
repo_name=repo_name,
revision=revision,
dry_run=options.dry_run,
trigger_build_if_missing=not options.trigger_tests_only
)
return
# Mode #3: Trigger jobs based on revision list modifiers
if not (options.includes or options.exclude or options.failed_jobs):
job_names = options.buildernames
# Mode 4 - Schedule every builder matching --includes and does not match --exclude.
elif options.includes or options.exclude:
filters_in = options.includes.split(',') + [repo_name]
filters_out = []
if options.exclude:
filters_out = options.exclude.split(',')
job_names = filter_buildernames(
buildernames=query_builders(repo_name=repo_name),
include=filters_in,
exclude=filters_out
)
if len(job_names) == 0:
LOG.info("0 jobs match these filters. please try again.")
return
if options.existing_only:
# We query all succesful jobs for a given revision and filter
# them by include/exclude filters.
trigger_build_if_missing = False
successful_jobs = TreeherderApi().find_all_jobs_by_status(
repo_name=repo_name,
revision=revision,
status=SUCCESS)
# We will filter out all the existing job from those successful job we have.
job_names = [buildername for buildername in successful_jobs
if buildername in job_names]
cont = raw_input("The ones which have existing builds out of %i jobs will be triggered,\
do you wish to continue? y/n/d (d=show details) " % len(job_names))
else:
cont = raw_input("%i jobs will be triggered, do you wish to continue? \
y/n/d (d=show details) " % len(job_names))
if cont.lower() == 'd':
LOG.info("The following jobs will be triggered: \n %s" % '\n'.join(job_names))
cont = raw_input("Do you wish to continue? y/n ")
if cont.lower() != 'y':
exit(1)
# Mode 5: Use --failed-jobs to trigger jobs for particular revision
elif options.failed_jobs:
job_names = TreeherderApi().find_all_jobs_by_status(
repo_name=repo_name,
revision=revision,
status=WARNING)
for buildername in job_names:
revlist = determine_revlist(
repo_url=repo_url,
buildername=buildername,
rev=revision,
back_revisions=options.back_revisions,
delta=options.delta,
from_rev=options.from_rev,
backfill=options.backfill,
skips=options.skips,
max_revisions=options.max_revisions)
_print_treeherder_link(
revlist=revlist,
repo_name=repo_name,
buildername=buildername,
revision=revision,
log=LOG,
includes=options.includes,
exclude=options.exclude)
try:
mgr.trigger_range(
buildername=buildername,
repo_name=repo_name,
revisions=revlist,
times=options.times,
dry_run=options.dry_run,
files=options.files,
trigger_build_if_missing=trigger_build_if_missing
)
except Exception, e:
LOG.exception(e)
exit(1)
def retrigger(self, uuid, *args, **kwargs):
return make_retrigger_request(request_id=uuid,
auth=get_credentials(),
*args,
**kwargs)
def cancel(self, uuid, *args, **kwargs):
return make_cancel_request(repo_name=kwargs['repo_name'],
request_id=uuid,
auth=get_credentials(),
*args,
**kwargs)
def main():
options = parse_args()
if options.debug:
LOG = setup_logging(logging.DEBUG)
else:
LOG = setup_logging(logging.INFO)
validate_options(options)
if not options.dry_run and not valid_credentials():
sys.exit(-1)
# Setting the QUERY_SOURCE global variable in mozci.py
set_query_source(options.query_source)
if options.buildernames:
options.buildernames = sanitize_buildernames(options.buildernames)
repo_url = query_repo_url_from_buildername(options.buildernames[0])
if not options.repo_name:
repo_name = query_repo_name_from_buildername(options.buildernames[0])
else:
repo_name = options.repo_name
repo_url = query_repo_url(repo_name)
if options.rev == 'tip':
revision = query_repo_tip(repo_url).changesets[0].node
LOG.info("The tip of %s is %s", repo_name, revision)
else:
revision = query_push_by_revision(repo_url, options.rev,
return_revision_list=True)
# Schedule jobs through TaskCluster if --taskcluster option has been set to true
if options.taskcluster:
mgr = TaskClusterBuildbotManager()
else:
mgr = BuildAPIManager()
trigger_build_if_missing = options.trigger_build_if_missing
if repo_name == 'try':
trigger_build_if_missing = False
# Mode 0: Backfill
if options.backfill:
manual_backfill(revision, options.buildernames[0], dry_run=options.dry_run)
return
# Mode 1: Trigger coalesced jobs
if options.coalesced:
query_api = BuildApi()
request_ids = query_api.find_all_jobs_by_status(repo_name,
revision, COALESCED)
if len(request_ids) == 0:
LOG.info('We did not find any coalesced job')
for request_id in request_ids:
make_retrigger_request(repo_name=repo_name,
request_id=request_id,
auth=get_credentials(),
dry_run=options.dry_run)
return
# Mode #2: Fill-in a revision or trigger_test_jobs_only
if options.fill_revision or options.trigger_tests_only:
mgr.trigger_missing_jobs_for_revision(
repo_name=repo_name,
revision=revision,
dry_run=options.dry_run,
trigger_build_if_missing=not options.trigger_tests_only
)
return
# Mode #3: Trigger jobs based on revision list modifiers
if not (options.includes or options.exclude or options.failed_jobs):
job_names = options.buildernames
# Mode 4 - Schedule every builder matching --includes and does not match --exclude.
elif options.includes or options.exclude:
filters_in = options.includes.split(',') + [repo_name]
filters_out = []
if options.exclude:
filters_out = options.exclude.split(',')
job_names = filter_buildernames(
buildernames=query_builders(repo_name=repo_name),
include=filters_in,
exclude=filters_out
)
if len(job_names) == 0:
LOG.info("0 jobs match these filters. please try again.")
return
if options.existing_only:
# We query all successful jobs for a given revision and filter
# them by include/exclude filters.
trigger_build_if_missing = False
successful_jobs = TreeherderApi().find_all_jobs_by_status(
repo_name=repo_name,
revision=revision,
status=SUCCESS)
# We will filter out all the existing job from those successful job we have.
job_names = [buildername for buildername in successful_jobs
if buildername in job_names]
cont = raw_input("The ones which have existing builds out of %i jobs will be triggered,\
do you wish to continue? y/n/d (d=show details) " % len(job_names))
else:
cont = raw_input("%i jobs will be triggered, do you wish to continue? \
y/n/d (d=show details) " % len(job_names))
if cont.lower() == 'd':
LOG.info("The following jobs will be triggered: \n %s" % '\n'.join(job_names))
cont = raw_input("Do you wish to continue? y/n ")
if cont.lower() != 'y':
exit(1)
# Mode 5: Use --failed-jobs to trigger jobs for particular revision
elif options.failed_jobs:
job_names = TreeherderApi().find_all_jobs_by_status(
repo_name=repo_name,
revision=revision,
status=WARNING)
for buildername in job_names:
revlist = determine_revlist(
repo_url=repo_url,
buildername=buildername,
rev=revision,
back_revisions=options.back_revisions,
delta=options.delta,
from_rev=options.from_rev,
backfill=options.backfill,
skips=options.skips,
max_revisions=options.max_revisions)
_print_treeherder_link(
revlist=revlist,
repo_name=repo_name,
buildername=buildername,
revision=revision,
log=LOG,
includes=options.includes,
exclude=options.exclude)
try:
mgr.trigger_range(
buildername=buildername,
repo_name=repo_name,
revisions=revlist,
times=options.times,
dry_run=options.dry_run,
files=options.files,
trigger_build_if_missing=trigger_build_if_missing
)
except Exception, e:
LOG.exception(e)
exit(1)
