def test_in_memory_cache(self, get):
"""Calling the function with in-memory cache should return without calling request.get."""
pushlog_client.VALID_CACHE = {("try", "146071751b1e"): True}
self.assertEquals(pushlog_client.valid_revision("try", "146071751b1e"),
True)
assert get.call_count == 0
def test_in_memory_cache(self, get):
"""Calling the function with in-memory cache should return without calling request.get."""
pushlog_client.VALID_CACHE = {("try", "146071751b1e"): True}
self.assertEquals(
pushlog_client.valid_revision("try", "146071751b1e"), True)
assert get.call_count == 0
def test_valid_without_any_cache(self, get):
"""Calling the function without in-memory cache."""
# Making sure the original cache is empty
pushlog_client.VALID_CACHE = {}
self.assertEquals(
pushlog_client.valid_revision("try", "4e030c8cf8c3"), True)
# The in-memory cache should be filed now
self.assertEquals(
pushlog_client.VALID_CACHE, {("try", "4e030c8cf8c3"): True})
def test_valid_without_any_cache(self, get):
"""Calling the function without in-memory cache."""
# Making sure the original cache is empty
pushlog_client.VALID_CACHE = {}
self.assertEquals(pushlog_client.valid_revision("try", "4e030c8cf8c3"),
True)
# The in-memory cache should be filed now
self.assertEquals(pushlog_client.VALID_CACHE,
{("try", "4e030c8cf8c3"): True})
def trigger_range(buildername, revisions, times=1, dry_run=False,
files=None, extra_properties=None, trigger_build_if_missing=True):
"""Schedule the job named "buildername" ("times" times) in every revision on 'revisions'."""
repo_name = query_repo_name_from_buildername(buildername)
repo_url = repositories.query_repo_url(repo_name)
if revisions != []:
LOG.info("We want to have %s job(s) of %s on the following revisions: "
% (times, buildername))
for r in revisions:
LOG.info(" - %s" % r)
for rev in revisions:
LOG.info("")
LOG.info("=== %s ===" % rev)
if VALIDATE and not valid_revision(repo_url, rev):
LOG.info("We can't trigger anything on pushes without a valid revision.")
continue
LOG.info("We want to have %s job(s) of %s" % (times, buildername))
# 1) How many potentially completed jobs can we get for this buildername?
matching_jobs = QUERY_SOURCE.get_matching_jobs(repo_name, rev, buildername)
status_summary = StatusSummary(matching_jobs)
# TODO: change this debug message when we have a less hardcoded _status_summary
LOG.debug("We found %d pending/running jobs, %d successful jobs and "
"%d failed jobs" % (status_summary.pending_jobs + status_summary.running_jobs,
status_summary.successful_jobs, status_summary.failed_jobs))
if status_summary.potential_jobs >= times:
LOG.info("We have %d job(s) for '%s' which is enough for the %d job(s) we want." %
(status_summary.potential_jobs, buildername, times))
else:
# 2) If we have less potential jobs than 'times' instances then
# we need to fill it in.
LOG.info("We have found %d potential job(s) matching '%s' on %s. "
"We need to trigger more." % (status_summary.potential_jobs, buildername, rev))
# If a job matching what we want already exists, we can
# use the retrigger API in self-serve to retrigger that
# instead of creating a new arbitrary job
if len(matching_jobs) > 0 and files is None:
request_id = QUERY_SOURCE.get_buildapi_request_id(repo_name, matching_jobs[0])
make_retrigger_request(
repo_name=repo_name,
request_id=request_id,
auth=get_credentials(),
count=(times - status_summary.potential_jobs),
dry_run=dry_run)
# If no matching job exists, we have to trigger a new arbitrary job
else:
list_of_requests = trigger_job(
revision=rev,
buildername=buildername,
times=(times - status_summary.potential_jobs),
dry_run=dry_run,
files=files,
extra_properties=extra_properties,
trigger_build_if_missing=trigger_build_if_missing)
if list_of_requests and any(req.status_code != 202 for req in list_of_requests):
LOG.warning("Not all requests succeeded.")
def trigger_job(revision, buildername, times=1, files=None, dry_run=False,
extra_properties=None, trigger_build_if_missing=True):
"""Trigger a job through self-serve.
We return a list of all requests made.
"""
repo_name = query_repo_name_from_buildername(buildername)
builder_to_trigger = None
list_of_requests = []
repo_url = repositories.query_repo_url(repo_name)
if VALIDATE and not valid_revision(repo_url, revision):
return list_of_requests
LOG.info("==> We want to trigger '%s' a total of %d time(s)." % (buildername, times))
LOG.info("") # Extra line to help visual of logs
if VALIDATE and not valid_builder(buildername):
LOG.error("The builder %s requested is invalid" % buildername)
# XXX How should we exit cleanly?
exit(-1)
if files:
builder_to_trigger = buildername
_all_urls_reachable(files)
else:
builder_to_trigger, package_url, test_url = determine_trigger_objective(
revision=revision,
buildername=buildername,
trigger_build_if_missing=trigger_build_if_missing,
will_use_buildapi=True
)
if builder_to_trigger != buildername and times != 1:
# The user wants to trigger a downstream job,
# however, we need a build job instead.
# We should trigger the downstream job multiple times, however,
# we only trigger the upstream jobs once.
LOG.debug("Since we need to trigger a build job we don't need to "
"trigger it %s times but only once." % times)
if trigger_build_if_missing:
LOG.info("In order to trigger %s %i times, "
"please run the script again after %s ends."
% (buildername, times, builder_to_trigger))
else:
LOG.info("We won't trigger '%s' because there is no working build."
% buildername)
LOG.info("")
times = 1
if builder_to_trigger:
if dry_run:
LOG.info("Dry-run: We were going to request '%s' %s times." %
(builder_to_trigger, times))
# Running with dry_run being True will only output information
trigger(
builder=builder_to_trigger,
revision=revision,
files=[package_url, test_url],
dry_run=dry_run,
extra_properties=extra_properties
)
else:
for _ in range(times):
req = trigger(
builder=builder_to_trigger,
revision=revision,
files=[package_url, test_url],
dry_run=dry_run,
extra_properties=extra_properties
)
if req is not None:
list_of_requests.append(req)
else:
LOG.debug("Nothing needs to be triggered")
# Cleanup old buildjson files.
clean_directory()
return list_of_requests
def test_invalid(self, get):
"""Calling the function with a bad revision."""
self.assertEquals(
pushlog_client.valid_revision("try", "123456123456"), False)
def test_invalid(self, get):
"""Calling the function with a bad revision."""
self.assertEquals(pushlog_client.valid_revision("try", "123456123456"),
False)
def trigger_range(buildername,
revisions,
times=1,
dry_run=False,
files=None,
extra_properties=None,
trigger_build_if_missing=True):
"""Schedule the job named "buildername" ("times" times) in every revision on 'revisions'."""
repo_name = query_repo_name_from_buildername(buildername)
repo_url = repositories.query_repo_url(repo_name)
if revisions != []:
LOG.info(
"We want to have %s job(s) of %s on the following revisions: " %
(times, buildername))
for r in revisions:
LOG.info(" - %s" % r)
for rev in revisions:
LOG.info("")
LOG.info("=== %s ===" % rev)
if VALIDATE and not valid_revision(repo_url, rev):
LOG.info(
"We can't trigger anything on pushes without a valid revision."
)
continue
# 1) How many potentially completed jobs can we get for this buildername?
matching_jobs = QUERY_SOURCE.get_matching_jobs(repo_name, rev,
buildername)
status_summary = StatusSummary(matching_jobs)
# TODO: change this debug message when we have a less hardcoded _status_summary
LOG.debug("We found %d pending/running jobs, %d successful jobs and "
"%d failed jobs" %
(status_summary.pending_jobs + status_summary.running_jobs,
status_summary.successful_jobs, status_summary.failed_jobs))
if status_summary.potential_jobs >= times:
LOG.info(
"We have %d job(s) for '%s' which is enough for the %d job(s) we want."
% (status_summary.potential_jobs, buildername, times))
else:
# 2) If we have less potential jobs than 'times' instances then
# we need to fill it in.
LOG.info("We have found %d potential job(s) matching '%s' on %s. "
"We need to trigger more." %
(status_summary.potential_jobs, buildername, rev))
schedule_new_job = True
# If a job matching what we want already exists, we can
# use the retrigger API in self-serve to retrigger that
# instead of creating a new arbitrary job
if len(matching_jobs) > 0 and files is None:
try:
request_id = QUERY_SOURCE.get_buildapi_request_id(
repo_name, matching_jobs[0])
make_retrigger_request(
repo_name=repo_name,
request_id=request_id,
auth=get_credentials(),
count=(times - status_summary.potential_jobs),
dry_run=dry_run)
schedule_new_job = False
except (IndexError, ConnectionError, ReadTimeout,
ValueError) as e:
# Logging until we can determine why we get these errors
# We should have one of these:
# {'requests': [{'request_id': int]}
# {'request_id': int}
LOG.info(matching_jobs[0])
LOG.info(str(e))
LOG.warning("We failed to retrigger the job, however, "
"we will try to schedule a new one.")
# If no matching job exists, we have to trigger a new arbitrary job
if schedule_new_job:
list_of_requests = trigger_job(
revision=rev,
buildername=buildername,
times=(times - status_summary.potential_jobs),
dry_run=dry_run,
files=files,
extra_properties=extra_properties,
trigger_build_if_missing=trigger_build_if_missing)
if list_of_requests and any(req.status_code != 202
for req in list_of_requests):
LOG.warning("Not all requests succeeded.")
def trigger_job(revision,
buildername,
times=1,
files=None,
dry_run=False,
extra_properties={},
trigger_build_if_missing=True):
"""Trigger a job through self-serve.
We return a list of all requests made.
"""
if not extra_properties:
extra_properties = {}
extra_properties.update(get_builder_extra_properties(buildername))
repo_name = query_repo_name_from_buildername(buildername)
builder_to_trigger = None
list_of_requests = []
repo_url = repositories.query_repo_url(repo_name)
if len(revision) != 40:
LOG.warning(
'We should not be using revisions less than 40 chars ({}).'.format(
revision))
push_info = query_push_by_revision(repo_url, revision)
revision = push_info.changesets[0].node
assert len(revision) == 40, 'This should have been a 40 char revision.'
if VALIDATE and not valid_revision(repo_url, revision):
return list_of_requests
LOG.info("==> We want to trigger '%s' a total of %d time(s)." %
(buildername, times))
LOG.info("") # Extra line to help visual of logs
if VALIDATE and not valid_builder(buildername):
LOG.error("The builder %s requested is invalid" % buildername)
# XXX How should we exit cleanly?
exit(-1)
if files:
builder_to_trigger = buildername
_all_urls_reachable(files)
else:
builder_to_trigger, package_url, test_url = determine_trigger_objective(
revision=revision,
buildername=buildername,
trigger_build_if_missing=trigger_build_if_missing,
will_use_buildapi=True)
if builder_to_trigger != buildername and times != 1:
# The user wants to trigger a downstream job,
# however, we need a build job instead.
# We should trigger the downstream job multiple times, however,
# we only trigger the upstream jobs once.
LOG.debug("Since we need to trigger a build job we don't need to "
"trigger it %s times but only once." % times)
if trigger_build_if_missing:
LOG.info("In order to trigger %s %i times, "
"please run the script again after %s ends." %
(buildername, times, builder_to_trigger))
else:
LOG.info(
"We won't trigger '%s' because there is no working build."
% buildername)
LOG.info("")
times = 1
if builder_to_trigger:
if dry_run:
LOG.info("Dry-run: We were going to request '%s' %s times." %
(builder_to_trigger, times))
# Running with dry_run being True will only output information
trigger(builder=builder_to_trigger,
revision=revision,
files=[package_url, test_url],
dry_run=dry_run,
extra_properties=extra_properties)
else:
for _ in range(times):
req = trigger(builder=builder_to_trigger,
revision=revision,
files=[package_url, test_url],
dry_run=dry_run,
extra_properties=extra_properties)
if req is not None:
list_of_requests.append(req)
else:
LOG.debug("Nothing needs to be triggered")
# Cleanup old buildjson files.
clean_directory()
return list_of_requests