def trigger_job(revision, buildername, times=1, files=None, dry_run=False,
extra_properties=None):
"""Trigger a job through self-serve.
We return a list of all requests made.
"""
repo_name = query_repo_name_from_buildername(buildername)
builder_to_trigger = None
list_of_requests = []
LOG.info("We want to trigger '%s' on revision '%s' a total of %d time(s)." %
(buildername, revision, times))
if not buildapi.valid_revision(repo_name, revision):
return list_of_requests
if not valid_builder(buildername):
LOG.error("The builder %s requested is invalid" % buildername)
# XXX How should we exit cleanly?
exit(-1)
if files:
builder_to_trigger = buildername
_all_urls_reachable(files)
else:
builder_to_trigger, files = _determine_trigger_objective(
revision,
buildername,
)
if builder_to_trigger != buildername and times != 1:
# The user wants to trigger a downstream job,
# however, we need a build job instead.
# We should trigger the downstream job multiple times, however,
# we only trigger the upstream jobs once.
LOG.debug("Since we need to trigger a build job we don't need to "
"trigger it %s times but only once." % times)
LOG.info("In order to trigger %s %i times, please run the script again after %s ends."
% (buildername, times, builder_to_trigger))
times = 1
if builder_to_trigger:
if dry_run:
LOG.info("Dry-run: We were going to request '%s' %s times." %
(builder_to_trigger, times))
# Running with dry_run being True will only output information
trigger(builder_to_trigger, revision, files, dry_run, extra_properties)
else:
for _ in range(times):
req = trigger(builder_to_trigger, revision, files, dry_run, extra_properties)
if req is not None:
list_of_requests.append(req)
else:
LOG.debug("Nothing needs to be triggered")
return list_of_requests
def determine_trigger_objective(revision,
buildername,
trigger_build_if_missing=True):
"""
Determine if we need to trigger any jobs and which job.
Returns:
* The name of the builder we need to trigger
* Files, if needed, to trigger such builder
"""
builder_to_trigger = None
files = None
repo_name = query_repo_name_from_buildername(buildername)
build_buildername = determine_upstream_builder(buildername)
if VALIDATE and not valid_builder(build_buildername):
raise MozciError("Our platforms mapping system has failed.")
if build_buildername == buildername:
# For a build job we know that we don't need files to
# trigger it and it's the build job we want to trigger
return build_buildername, None
# Let's figure out which jobs are associated to such revision
query_api = BuildApi()
# Let's only look at jobs that match such build_buildername
build_jobs = query_api.get_matching_jobs(repo_name, revision,
build_buildername)
# We need to determine if we need to trigger a build job
# or the test job
working_job = None
running_job = None
failed_job = None
LOG.debug("List of matching jobs:")
for job in build_jobs:
try:
status = query_api.get_job_status(job)
except buildjson.BuildjsonException:
LOG.debug(
"We have hit bug 1159279 and have to work around it. We will "
"pretend that we could not reach the files for it.")
continue
# Sometimes running jobs have status unknown in buildapi
if status in (RUNNING, PENDING, UNKNOWN):
LOG.debug(
"We found a running/pending build job. We don't search anymore."
)
running_job = job
# We cannot call _find_files for a running job
continue
# Having a coalesced build is the same as not having a build available
if status == COALESCED:
LOG.debug(
"The build we found was a coalesced one; this is the same as "
"non-existant.")
continue
# Successful or failed jobs may have the files we need
files = _find_files(job)
if files != [] and _all_urls_reachable(files.values()):
working_job = job
break
else:
LOG.debug("We can't determine the files for this build or "
"can't reach them.")
files = None
LOG.info("We found a job that finished but it did not "
"produced files. status: %d" % status)
failed_job = job
# End of for loop
if working_job:
# We found a build job with the necessary files. It could be a
# successful job, a running job that already emitted files or a
# testfailed job
LOG.debug(str(working_job))
LOG.info("We have the necessary files to trigger the downstream job.")
# We have the files needed to trigger the test job
builder_to_trigger = buildername
elif running_job:
LOG.info(
"We found a running/pending build job. We will not trigger another one."
)
LOG.info(
"You have to run the script again after the build job is finished to "
"trigger %s." % buildername)
builder_to_trigger = None
elif failed_job:
LOG.info(
"The build job %s failed on revision %s without generating the "
"necessary files. We will not trigger anything." %
(build_buildername, revision))
builder_to_trigger = None
else:
# We were trying to build a test job, however, we determined
# that we need an upstream builder instead
if not trigger_build_if_missing or not _unique_build_request(
build_buildername, revision):
# This is a safeguard to prevent triggering a build
# job multiple times if it is not intentional
builder_to_trigger = None
if not trigger_build_if_missing:
LOG.info(
"We would have to triggered build '%s' in order to trigger "
"job '%s'. On this mode we will not trigger either." %
(build_buildername, buildername))
else:
LOG.info("We will trigger 1) "
"'%s' instead of 2) '%s'" %
(build_buildername, buildername))
LOG.info("We need to trigger the build job once (1) "
"in order to be able to run the test job (2).")
if repo_name == 'try':
LOG.info(
"You'll need to run the script again after (1) is done to "
"trigger (2).")
else:
LOG.info(
"After (1) is done and if no coalesccing happens the test "
"jobs associated with it will be triggered.")
builder_to_trigger = build_buildername
if files:
return builder_to_trigger, files['packageUrl'], files[
'testPackagesUrl']
else:
return builder_to_trigger, None, None
def trigger_job(revision,
buildername,
times=1,
files=None,
dry_run=False,
extra_properties=None,
trigger_build_if_missing=True):
"""Trigger a job through self-serve.
We return a list of all requests made.
"""
repo_name = query_repo_name_from_buildername(buildername)
builder_to_trigger = None
list_of_requests = []
repo_url = repositories.query_repo_url(repo_name)
if VALIDATE and not pushlog.valid_revision(repo_url, revision):
return list_of_requests
LOG.info(
"===> We want to trigger '%s' on revision '%s' a total of %d time(s)."
% (buildername, revision, times))
LOG.info("") # Extra line to help visual of logs
if VALIDATE and not valid_builder(buildername):
LOG.error("The builder %s requested is invalid" % buildername)
# XXX How should we exit cleanly?
exit(-1)
if files:
builder_to_trigger = buildername
_all_urls_reachable(files)
else:
builder_to_trigger, package_url, test_url = determine_trigger_objective(
revision=revision,
buildername=buildername,
trigger_build_if_missing=trigger_build_if_missing)
if builder_to_trigger != buildername and times != 1:
# The user wants to trigger a downstream job,
# however, we need a build job instead.
# We should trigger the downstream job multiple times, however,
# we only trigger the upstream jobs once.
LOG.debug("Since we need to trigger a build job we don't need to "
"trigger it %s times but only once." % times)
if trigger_build_if_missing:
LOG.info("In order to trigger %s %i times, "
"please run the script again after %s ends." %
(buildername, times, builder_to_trigger))
else:
LOG.info(
"We won't trigger '%s' because there is no working build."
% buildername)
LOG.info("")
times = 1
if builder_to_trigger:
if dry_run:
LOG.info("Dry-run: We were going to request '%s' %s times." %
(builder_to_trigger, times))
# Running with dry_run being True will only output information
trigger(builder=builder_to_trigger,
revision=revision,
files=[package_url, test_url],
dry_run=dry_run,
extra_properties=extra_properties)
else:
for _ in range(times):
req = trigger(builder=builder_to_trigger,
revision=revision,
files=[package_url, test_url],
dry_run=dry_run,
extra_properties=extra_properties)
if req is not None:
list_of_requests.append(req)
else:
LOG.debug("Nothing needs to be triggered")
# Cleanup old buildjson files.
clean_directory()
return list_of_requests
def trigger_job(revision, buildername, times=1, files=None, dry_run=False,
extra_properties=None, trigger_build_if_missing=True):
"""Trigger a job through self-serve.
We return a list of all requests made.
"""
repo_name = query_repo_name_from_buildername(buildername)
builder_to_trigger = None
list_of_requests = []
repo_url = repositories.query_repo_url(repo_name)
if VALIDATE and not valid_revision(repo_url, revision):
return list_of_requests
LOG.info("==> We want to trigger '%s' a total of %d time(s)." % (buildername, times))
LOG.info("") # Extra line to help visual of logs
if VALIDATE and not valid_builder(buildername):
LOG.error("The builder %s requested is invalid" % buildername)
# XXX How should we exit cleanly?
exit(-1)
if files:
builder_to_trigger = buildername
_all_urls_reachable(files)
else:
builder_to_trigger, package_url, test_url = determine_trigger_objective(
revision=revision,
buildername=buildername,
trigger_build_if_missing=trigger_build_if_missing,
will_use_buildapi=True
)
if builder_to_trigger != buildername and times != 1:
# The user wants to trigger a downstream job,
# however, we need a build job instead.
# We should trigger the downstream job multiple times, however,
# we only trigger the upstream jobs once.
LOG.debug("Since we need to trigger a build job we don't need to "
"trigger it %s times but only once." % times)
if trigger_build_if_missing:
LOG.info("In order to trigger %s %i times, "
"please run the script again after %s ends."
% (buildername, times, builder_to_trigger))
else:
LOG.info("We won't trigger '%s' because there is no working build."
% buildername)
LOG.info("")
times = 1
if builder_to_trigger:
if dry_run:
LOG.info("Dry-run: We were going to request '%s' %s times." %
(builder_to_trigger, times))
# Running with dry_run being True will only output information
trigger(
builder=builder_to_trigger,
revision=revision,
files=[package_url, test_url],
dry_run=dry_run,
extra_properties=extra_properties
)
else:
for _ in range(times):
req = trigger(
builder=builder_to_trigger,
revision=revision,
files=[package_url, test_url],
dry_run=dry_run,
extra_properties=extra_properties
)
if req is not None:
list_of_requests.append(req)
else:
LOG.debug("Nothing needs to be triggered")
# Cleanup old buildjson files.
clean_directory()
return list_of_requests
def determine_trigger_objective(revision, buildername, trigger_build_if_missing=True,
will_use_buildapi=False):
"""
Determine if we need to trigger any jobs and which job.
Returns:
* The name of the builder we need to trigger
* Files, if needed, to trigger such builder
"""
builder_to_trigger = None
files = None
repo_name = query_repo_name_from_buildername(buildername)
build_buildername = determine_upstream_builder(buildername)
if VALIDATE and not valid_builder(build_buildername):
raise MozciError("Our platforms mapping system has failed.")
if build_buildername == buildername:
# For a build job we know that we don't need files to
# trigger it and it's the build job we want to trigger
return build_buildername, None, None
# Let's figure out which jobs are associated to such revision
query_api = BuildApi()
# Let's only look at jobs that match such build_buildername
build_jobs = query_api.get_matching_jobs(repo_name, revision, build_buildername)
# We need to determine if we need to trigger a build job
# or the test job
working_job = None
running_job = None
failed_job = None
LOG.debug("List of matching jobs:")
for job in build_jobs:
try:
status = query_api.get_job_status(job)
except buildjson.BuildjsonException:
LOG.debug("We have hit bug 1159279 and have to work around it. We will "
"pretend that we could not reach the files for it.")
continue
# Sometimes running jobs have status unknown in buildapi
if status in (RUNNING, PENDING, UNKNOWN):
LOG.debug("We found a running/pending build job. We don't search anymore.")
running_job = job
# We cannot call _find_files for a running job
continue
# Having a coalesced build is the same as not having a build available
if status == COALESCED:
LOG.debug("The build we found was a coalesced one; this is the same as "
"non-existant.")
continue
# Successful or failed jobs may have the files we need
files = _find_files(job)
if files != [] and _all_urls_reachable(files.values()):
working_job = job
break
else:
LOG.debug("We can't determine the files for this build or "
"can't reach them.")
files = None
LOG.info("We found a job that finished but it did not "
"produced files. status: %d" % status)
failed_job = job
# End of for loop
if working_job:
# We found a build job with the necessary files. It could be a
# successful job, a running job that already emitted files or a
# testfailed job
LOG.debug(str(working_job))
LOG.info("We have the necessary files to trigger the downstream job.")
# We have the files needed to trigger the test job
builder_to_trigger = buildername
elif running_job:
LOG.info("We found a running/pending build job. We will not trigger another one.")
LOG.info("You have to run the script again after the build job is finished to "
"trigger %s." % buildername)
builder_to_trigger = None
elif failed_job:
LOG.info("The build job %s failed on revision %s without generating the "
"necessary files. We will not trigger anything." %
(build_buildername, revision))
builder_to_trigger = None
else:
# We were trying to build a test job, however, we determined
# that we need an upstream builder instead
if not trigger_build_if_missing or not _unique_build_request(build_buildername, revision):
# This is a safeguard to prevent triggering a build
# job multiple times if it is not intentional
builder_to_trigger = None
if not trigger_build_if_missing:
LOG.info("We would have to triggered build '%s' in order to trigger "
"job '%s'. On this mode we will not trigger either." %
(build_buildername, buildername))
else:
if will_use_buildapi:
LOG.info("We will trigger 1) '%s'" % build_buildername)
LOG.info("instead of 2) '%s'" % buildername)
LOG.info("We need to trigger the build job once (1) "
"in order to be able to run the test job (2).")
if repo_name == 'try':
LOG.info("You'll need to run the script again after (1) is done to "
"trigger (2).")
else:
LOG.info("After (1) is done and if no coalesccing happens the test "
"jobs associated with it will be triggered.")
builder_to_trigger = build_buildername
if files:
return builder_to_trigger, files['packageUrl'], files['testsUrl']
else:
return builder_to_trigger, None, None
def _determine_trigger_objective(revision, buildername):
"""
Determine if we need to trigger any jobs and which job.
Returns:
* The name of the builder we need to trigger
* Files, if needed, to trigger such builder
"""
builder_to_trigger = None
files = None
repo_name = query_repo_name_from_buildername(buildername)
build_buildername = determine_upstream_builder(buildername)
assert valid_builder(build_buildername), \
"Our platforms mapping system has failed."
if build_buildername == buildername:
# For a build job we know that we don't need files to
# trigger it and it's the build job we want to trigger
return build_buildername, None
# Let's figure out which jobs are associated to such revision
all_jobs = query_jobs(repo_name, revision)
# Let's only look at jobs that match such build_buildername
build_jobs = _matching_jobs(build_buildername, all_jobs)
# We need to determine if we need to trigger a build job
# or the test job
successful_job = None
running_job = None
LOG.debug("List of matching jobs:")
for job in build_jobs:
try:
status = buildapi.query_job_status(job)
except buildjson.BuildjsonException:
LOG.debug("We have hit bug 1159279 and have to work around it. We will pretend that "
"we could not reach the files for it.")
continue
if status == buildapi.RUNNING:
LOG.debug("We found a running build job. We don't search anymore.")
running_job = job
elif status == buildapi.SUCCESS:
LOG.debug("We found a successful job. We don't search anymore.")
files = _find_files(job)
if files != [] and _all_urls_reachable(files):
successful_job = job
break
else:
LOG.debug("We can't determine the files for this build or "
"can't reach them.")
files = None
else:
LOG.debug("We found a job that finished but its status "
"is not successful. status: %d" % status)
if successful_job:
# A build job has completed successfully and the files can be reached
LOG.info("There is a _build_ job that has completed successfully.")
LOG.debug(str(successful_job))
LOG.info("We have the necessary files to trigger the downstream job.")
# We have the files needed to trigger the test job
builder_to_trigger = buildername
elif running_job:
# NOTE: Note that a build might have not finished yet
# the installer and test.zip might already have been uploaded
# For now, we will ignore this situation but need to take note of it
LOG.info("We are waiting for the associated build job to finish.")
LOG.debug(str(running_job))
builder_to_trigger = None
else:
# We were trying to build a test job, however, we determined
# that we need an upstream builder instead
if not _unique_build_request(build_buildername, revision):
# This is a safeguard to prevent triggering a build
# job multiple times if it is not intentional
builder_to_trigger = None
else:
LOG.info("We need to trigger the build job (1) in order to be able to run the test job (2)"
"which we'll be triggered later")
LOG.info("We will trigger 1) '%s' instead of 2) '%s'" % (build_buildername, buildername))
builder_to_trigger = build_buildername
return builder_to_trigger, files
def _determine_trigger_objective(revision, buildername):
"""
Determine if we need to trigger any jobs and which job.
Returns:
* The name of the builder we need to trigger
* Files, if needed, to trigger such builder
"""
builder_to_trigger = None
files = None
repo_name = query_repo_name_from_buildername(buildername)
build_buildername = determine_upstream_builder(buildername)
assert valid_builder(build_buildername), \
"Our platforms mapping system has failed."
if build_buildername == buildername:
# For a build job we know that we don't need files to
# trigger it and it's the build job we want to trigger
return build_buildername, None
# Let's figure out which jobs are associated to such revision
all_jobs = query_jobs(repo_name, revision)
# Let's only look at jobs that match such build_buildername
build_jobs = _matching_jobs(build_buildername, all_jobs)
# We need to determine if we need to trigger a build job
# or the test job
working_job = None
running_job = None
failed_job = None
LOG.debug("List of matching jobs:")
for job in build_jobs:
# Successful, running and failed jobs may have the files we need
files = _find_files(job)
if files != [] and _all_urls_reachable(files):
working_job = job
break
else:
LOG.debug("We can't determine the files for this build or "
"can't reach them.")
files = None
try:
status = buildapi.query_job_status(job)
except buildjson.BuildjsonException:
LOG.debug("We have hit bug 1159279 and have to work around it. We will pretend that "
"we could not reach the files for it.")
continue
if status == buildapi.RUNNING:
LOG.debug("We found a running build job. We don't search anymore.")
running_job = job
else:
LOG.info("We found a job that finished but its status "
"is not successful. status: %d" % status)
failed_job = job
if working_job:
# We found a build job with the necessary files. It could be a
# successful job, a running job that already emitted files or a
# testfailed job
LOG.debug(str(working_job))
LOG.info("We have the necessary files to trigger the downstream job.")
# We have the files needed to trigger the test job
builder_to_trigger = buildername
elif running_job:
LOG.info("We found a running build job without files. We will not trigger another one. "
"You have to run the script again after the build job is finished to trigger %s." %
buildername)
builder_to_trigger = None
elif failed_job:
LOG.info("The build job %s failed on revision %s without generating the necessary files. "
"We will not trigger anything." % (build_buildername, revision))
builder_to_trigger = None
else:
# We were trying to build a test job, however, we determined
# that we need an upstream builder instead
if not _unique_build_request(build_buildername, revision):
# This is a safeguard to prevent triggering a build
# job multiple times if it is not intentional
builder_to_trigger = None
else:
LOG.info("We will trigger 1) '%s' instead of 2) '%s'" % (build_buildername, buildername))
LOG.info("We need to trigger the build job once (1) in order to be able to run the test job (2).")
if repo_name == 'try':
LOG.info("You'll need to run the script again after (1) is done to trigger (2).")
else:
LOG.info("After (1) is done every test job associated with it will be triggered.")
builder_to_trigger = build_buildername
return builder_to_trigger, files
def test_not_all_urls_are_reachable(get_credentials, urls, result):
get_credentials.return_value = ('', '')
assert _all_urls_reachable(urls=urls) == result
