def test_in_memory_cache(self, get):
"""Calling the function with in-memory cache should return without calling request.get."""
pushlog.VALID_CACHE = {("try", "146071751b1e"): True}
self.assertEquals(
pushlog.valid_revision("try", "146071751b1e"), True)
assert get.call_count == 0
def test_valid_without_any_cache(self, get):
"""Calling the function without in-memory cache."""
# Making sure the original cache is empty
pushlog.VALID_CACHE = {}
self.assertEquals(pushlog.valid_revision("try", "4e030c8cf8c3"), True)
# The in-memory cache should be filed now
self.assertEquals(pushlog.VALID_CACHE, {("try", "4e030c8cf8c3"): True})
def test_valid_without_any_cache(self, get):
"""Calling the function without in-memory cache."""
# Making sure the original cache is empty
pushlog.VALID_CACHE = {}
self.assertEquals(
pushlog.valid_revision("try", "4e030c8cf8c3"), True)
# The in-memory cache should be filed now
self.assertEquals(
pushlog.VALID_CACHE, {("try", "4e030c8cf8c3"): True})
def query_jobs_schedule(repo_name, revision):
""" Query Buildapi for jobs """
repo_url = query_repo_url(repo_name)
if not pushlog.valid_revision(repo_url, revision):
raise BuildapiException
url = "%s/%s/rev/%s?format=json" % (HOST_ROOT, repo_name, revision)
LOG.debug("About to fetch %s" % url)
req = requests.get(url, auth=get_credentials())
# If the revision doesn't exist on buildapi, that means there are
# no builapi jobs for this revision
if req.status_code not in [200]:
return []
return req.json()
def query_jobs_schedule(repo_name, revision):
"""
Query Buildapi for jobs.
Raises a BuildapiError if the revision doesn't exist in repo_name.
"""
repo_url = query_repo_url(repo_name)
if VALIDATE and not pushlog.valid_revision(repo_url, revision):
raise BuildapiError
url = "%s/%s/rev/%s?format=json" % (HOST_ROOT, repo_name, revision)
LOG.debug("About to fetch %s" % url)
req = requests.get(url, auth=get_credentials())
# If the revision doesn't exist on buildapi, that means there are
# no builapi jobs for this revision
if req.status_code not in [200]:
return []
return req.json()
def trigger_range(buildername,
revisions,
times=1,
dry_run=False,
files=None,
extra_properties=None,
trigger_build_if_missing=True):
"""Schedule the job named "buildername" ("times" times) in every revision on 'revisions'."""
repo_name = query_repo_name_from_buildername(buildername)
repo_url = repositories.query_repo_url(repo_name)
if revisions != []:
LOG.info("We want to have %s job(s) of %s on revisions %s" %
(times, buildername, str(revisions)))
for rev in revisions:
LOG.info("")
LOG.info("=== %s ===" % rev)
if VALIDATE and not pushlog.valid_revision(repo_url, rev):
LOG.info(
"We can't trigger anything on pushes without a valid revision."
)
continue
LOG.info("We want to have %s job(s) of %s on revision %s" %
(times, buildername, rev))
# 1) How many potentially completed jobs can we get for this buildername?
matching_jobs = QUERY_SOURCE.get_matching_jobs(repo_name, rev,
buildername)
successful_jobs, pending_jobs, running_jobs, _, failed_jobs = \
_status_summary(matching_jobs)
potential_jobs = pending_jobs + running_jobs + successful_jobs + failed_jobs
# TODO: change this debug message when we have a less hardcoded _status_summary
LOG.debug("We found %d pending/running jobs, %d successful jobs and "
"%d failed jobs" %
(pending_jobs + running_jobs, successful_jobs, failed_jobs))
if potential_jobs >= times:
LOG.info(
"We have %d job(s) for '%s' which is enough for the %d job(s) we want."
% (potential_jobs, buildername, times))
else:
# 2) If we have less potential jobs than 'times' instances then
# we need to fill it in.
LOG.info("We have found %d potential job(s) matching '%s' on %s. "
"We need to trigger more." %
(potential_jobs, buildername, rev))
# If a job matching what we want already exists, we can
# use the retrigger API in self-serve to retrigger that
# instead of creating a new arbitrary job
if len(matching_jobs) > 0 and files is None:
request_id = QUERY_SOURCE.get_buildapi_request_id(
repo_name, matching_jobs[0])
make_retrigger_request(repo_name=repo_name,
request_id=request_id,
auth=get_credentials(),
count=(times - potential_jobs),
dry_run=dry_run)
# If no matching job exists, we have to trigger a new arbitrary job
else:
list_of_requests = trigger_job(
revision=rev,
buildername=buildername,
times=(times - potential_jobs),
dry_run=dry_run,
files=files,
extra_properties=extra_properties,
trigger_build_if_missing=trigger_build_if_missing)
if list_of_requests and any(req.status_code != 202
for req in list_of_requests):
LOG.warning("Not all requests succeeded.")
def trigger_job(revision,
buildername,
times=1,
files=None,
dry_run=False,
extra_properties=None,
trigger_build_if_missing=True):
"""Trigger a job through self-serve.
We return a list of all requests made.
"""
repo_name = query_repo_name_from_buildername(buildername)
builder_to_trigger = None
list_of_requests = []
repo_url = repositories.query_repo_url(repo_name)
if VALIDATE and not pushlog.valid_revision(repo_url, revision):
return list_of_requests
LOG.info(
"===> We want to trigger '%s' on revision '%s' a total of %d time(s)."
% (buildername, revision, times))
LOG.info("") # Extra line to help visual of logs
if VALIDATE and not valid_builder(buildername):
LOG.error("The builder %s requested is invalid" % buildername)
# XXX How should we exit cleanly?
exit(-1)
if files:
builder_to_trigger = buildername
_all_urls_reachable(files)
else:
builder_to_trigger, package_url, test_url = determine_trigger_objective(
revision=revision,
buildername=buildername,
trigger_build_if_missing=trigger_build_if_missing)
if builder_to_trigger != buildername and times != 1:
# The user wants to trigger a downstream job,
# however, we need a build job instead.
# We should trigger the downstream job multiple times, however,
# we only trigger the upstream jobs once.
LOG.debug("Since we need to trigger a build job we don't need to "
"trigger it %s times but only once." % times)
if trigger_build_if_missing:
LOG.info("In order to trigger %s %i times, "
"please run the script again after %s ends." %
(buildername, times, builder_to_trigger))
else:
LOG.info(
"We won't trigger '%s' because there is no working build."
% buildername)
LOG.info("")
times = 1
if builder_to_trigger:
if dry_run:
LOG.info("Dry-run: We were going to request '%s' %s times." %
(builder_to_trigger, times))
# Running with dry_run being True will only output information
trigger(builder=builder_to_trigger,
revision=revision,
files=[package_url, test_url],
dry_run=dry_run,
extra_properties=extra_properties)
else:
for _ in range(times):
req = trigger(builder=builder_to_trigger,
revision=revision,
files=[package_url, test_url],
dry_run=dry_run,
extra_properties=extra_properties)
if req is not None:
list_of_requests.append(req)
else:
LOG.debug("Nothing needs to be triggered")
# Cleanup old buildjson files.
clean_directory()
return list_of_requests
def trigger_range(
buildername, revisions, times=1, dry_run=False, files=None, extra_properties=None, trigger_build_if_missing=True
):
"""Schedule the job named "buildername" ("times" times) in every revision on 'revisions'."""
repo_name = query_repo_name_from_buildername(buildername)
repo_url = repositories.query_repo_url(repo_name)
if revisions != []:
LOG.info("We want to have %s job(s) of %s on revisions %s" % (times, buildername, str(revisions)))
for rev in revisions:
LOG.info("")
LOG.info("=== %s ===" % rev)
if VALIDATE and not pushlog.valid_revision(repo_url, rev):
LOG.info("We can't trigger anything on pushes without a valid revision.")
continue
LOG.info("We want to have %s job(s) of %s on revision %s" % (times, buildername, rev))
# 1) How many potentially completed jobs can we get for this buildername?
matching_jobs = QUERY_SOURCE.get_matching_jobs(repo_name, rev, buildername)
successful_jobs, pending_jobs, running_jobs, _, failed_jobs = _status_summary(matching_jobs)
potential_jobs = pending_jobs + running_jobs + successful_jobs + failed_jobs
# TODO: change this debug message when we have a less hardcoded _status_summary
LOG.debug(
"We found %d pending/running jobs, %d successful jobs and "
"%d failed jobs" % (pending_jobs + running_jobs, successful_jobs, failed_jobs)
)
if potential_jobs >= times:
LOG.info(
"We have %d job(s) for '%s' which is enough for the %d job(s) we want."
% (potential_jobs, buildername, times)
)
else:
# 2) If we have less potential jobs than 'times' instances then
# we need to fill it in.
LOG.info(
"We have found %d potential job(s) matching '%s' on %s. "
"We need to trigger more." % (potential_jobs, buildername, rev)
)
# If a job matching what we want already exists, we can
# use the retrigger API in self-serve to retrigger that
# instead of creating a new arbitrary job
if len(matching_jobs) > 0 and files is None:
request_id = QUERY_SOURCE.get_buildapi_request_id(repo_name, matching_jobs[0])
make_retrigger_request(repo_name, request_id, count=(times - potential_jobs), dry_run=dry_run)
# If no matching job exists, we have to trigger a new arbitrary job
else:
list_of_requests = trigger_job(
revision=rev,
buildername=buildername,
times=(times - potential_jobs),
dry_run=dry_run,
files=files,
extra_properties=extra_properties,
trigger_build_if_missing=trigger_build_if_missing,
)
if list_of_requests and any(req.status_code != 202 for req in list_of_requests):
LOG.warning("Not all requests succeeded.")
def trigger_job(
revision, buildername, times=1, files=None, dry_run=False, extra_properties=None, trigger_build_if_missing=True
):
"""Trigger a job through self-serve.
We return a list of all requests made.
"""
repo_name = query_repo_name_from_buildername(buildername)
builder_to_trigger = None
list_of_requests = []
repo_url = repositories.query_repo_url(repo_name)
if VALIDATE and not pushlog.valid_revision(repo_url, revision):
return list_of_requests
LOG.info("===> We want to trigger '%s' on revision '%s' a total of %d time(s)." % (buildername, revision, times))
LOG.info("") # Extra line to help visual of logs
if VALIDATE and not valid_builder(buildername):
LOG.error("The builder %s requested is invalid" % buildername)
# XXX How should we exit cleanly?
exit(-1)
if files:
builder_to_trigger = buildername
_all_urls_reachable(files)
else:
builder_to_trigger, files = determine_trigger_objective(
revision=revision, buildername=buildername, trigger_build_if_missing=trigger_build_if_missing
)
if builder_to_trigger != buildername and times != 1:
# The user wants to trigger a downstream job,
# however, we need a build job instead.
# We should trigger the downstream job multiple times, however,
# we only trigger the upstream jobs once.
LOG.debug(
"Since we need to trigger a build job we don't need to " "trigger it %s times but only once." % times
)
if trigger_build_if_missing:
LOG.info(
"In order to trigger %s %i times, "
"please run the script again after %s ends." % (buildername, times, builder_to_trigger)
)
else:
LOG.info("We won't trigger '%s' because there is no working build." % buildername)
LOG.info("")
times = 1
if builder_to_trigger:
if dry_run:
LOG.info("Dry-run: We were going to request '%s' %s times." % (builder_to_trigger, times))
# Running with dry_run being True will only output information
trigger(builder_to_trigger, revision, files, dry_run, extra_properties)
else:
for _ in range(times):
req = trigger(builder_to_trigger, revision, files, dry_run, extra_properties)
if req is not None:
list_of_requests.append(req)
else:
LOG.debug("Nothing needs to be triggered")
# Cleanup old buildjson files.
clean_directory()
return list_of_requests
def test_invalid(self, get):
"""Calling the function with a bad revision."""
self.assertEquals(
pushlog.valid_revision("try", "123456123456"), False)