class TestBuildApiGetJobStatus(unittest.TestCase):
"""Test query_job_status with different types of jobs."""
def setUp(self):
self.query_api = BuildApi()
def test_pending_job(self):
"""Test get_job_status with a pending job."""
pending_job = json.loads(BASE_JSON %
('null', 'null', 0, 1433166609))[0]
pending_job.pop("status")
self.assertEquals(self.query_api.get_job_status(pending_job), PENDING)
def test_running_job(self):
"""Test get_job_status with a running job."""
running_job = json.loads(BASE_JSON % ('null', 'null', 0, 'null'))[0]
self.assertEquals(self.query_api.get_job_status(running_job), RUNNING)
def test_unknown_job(self):
"""Test get_job_status with an unknown job."""
unknown_job = json.loads(BASE_JSON %
('null', 'null', 0, 1433166609))[0]
self.assertEquals(self.query_api.get_job_status(unknown_job), UNKNOWN)
@patch('mozci.query_jobs.BuildApi._is_coalesced', return_value=SUCCESS)
def test_successful_job(self, _is_coalesced):
"""Test get_job_status with a successful job. We will mock _is_coalesced for that."""
successful_job = json.loads(BASE_JSON %
(SUCCESS, 1433166610, 1, 1433166609))[0]
self.assertEquals(self.query_api.get_job_status(successful_job),
SUCCESS)
@patch('mozci.query_jobs.BuildApi._is_coalesced', return_value=COALESCED)
def test_coalesced_job(self, _is_coalesced):
"""Test get_job_status with a coalesced job. We will mock _is_coalesced for that."""
coalesced_job = json.loads(BASE_JSON %
(SUCCESS, 1433166610, 1, 1433166609))[0]
self.assertEquals(self.query_api.get_job_status(coalesced_job),
COALESCED)
def test_failed_job(self):
"""Test get_job_status with a failed job."""
failed_job = json.loads(BASE_JSON %
(FAILURE, 1433166610, 1, 1433166609))[0]
self.assertEquals(self.query_api.get_job_status(failed_job), FAILURE)
def test_weird_job(self):
"""get_job_status should raise an Exception when it encounters an unexpected status."""
weird_job = json.loads(BASE_JSON % (20, 1433166610, 1, 1433166609))[0]
with self.assertRaises(Exception):
self.query_api.get_job_status(weird_job)
class TestBuildApiGetJobStatus(unittest.TestCase):
"""Test query_job_status with different types of jobs."""
def setUp(self):
self.query_api = BuildApi()
def test_pending_job(self):
"""Test get_job_status with a pending job."""
pending_job = json.loads(BASE_JSON % ('null', 'null', 0, 1433166609))[0]
pending_job.pop("status")
self.assertEquals(self.query_api.get_job_status(pending_job), PENDING)
def test_running_job(self):
"""Test get_job_status with a running job."""
running_job = json.loads(BASE_JSON % ('null', 'null', 0, 'null'))[0]
self.assertEquals(self.query_api.get_job_status(running_job), RUNNING)
def test_unknown_job(self):
"""Test get_job_status with an unknown job."""
unknown_job = json.loads(BASE_JSON % ('null', 'null', 0, 1433166609))[0]
self.assertEquals(self.query_api.get_job_status(unknown_job), UNKNOWN)
@patch('mozci.query_jobs.BuildApi._is_coalesced', return_value=SUCCESS)
def test_successful_job(self, _is_coalesced):
"""Test get_job_status with a successful job. We will mock _is_coalesced for that."""
successful_job = json.loads(BASE_JSON % (SUCCESS, 1433166610, 1, 1433166609))[0]
self.assertEquals(self.query_api.get_job_status(successful_job), SUCCESS)
@patch('mozci.query_jobs.BuildApi._is_coalesced', return_value=COALESCED)
def test_coalesced_job(self, _is_coalesced):
"""Test get_job_status with a coalesced job. We will mock _is_coalesced for that."""
coalesced_job = json.loads(BASE_JSON % (SUCCESS, 1433166610, 1, 1433166609))[0]
self.assertEquals(self.query_api.get_job_status(coalesced_job), COALESCED)
def test_failed_job(self):
"""Test get_job_status with a failed job."""
failed_job = json.loads(BASE_JSON % (FAILURE, 1433166610, 1, 1433166609))[0]
self.assertEquals(self.query_api.get_job_status(failed_job), FAILURE)
def test_weird_job(self):
"""get_job_status should raise an Exception when it encounters an unexpected status."""
weird_job = json.loads(BASE_JSON % (20, 1433166610, 1, 1433166609))[0]
with self.assertRaises(Exception):
self.query_api.get_job_status(weird_job)
def determine_trigger_objective(revision,
buildername,
trigger_build_if_missing=True):
"""
Determine if we need to trigger any jobs and which job.
Returns:
* The name of the builder we need to trigger
* Files, if needed, to trigger such builder
"""
builder_to_trigger = None
files = None
repo_name = query_repo_name_from_buildername(buildername)
build_buildername = determine_upstream_builder(buildername)
if VALIDATE and not valid_builder(build_buildername):
raise MozciError("Our platforms mapping system has failed.")
if build_buildername == buildername:
# For a build job we know that we don't need files to
# trigger it and it's the build job we want to trigger
return build_buildername, None
# Let's figure out which jobs are associated to such revision
query_api = BuildApi()
# Let's only look at jobs that match such build_buildername
build_jobs = query_api.get_matching_jobs(repo_name, revision,
build_buildername)
# We need to determine if we need to trigger a build job
# or the test job
working_job = None
running_job = None
failed_job = None
LOG.debug("List of matching jobs:")
for job in build_jobs:
try:
status = query_api.get_job_status(job)
except buildjson.BuildjsonException:
LOG.debug(
"We have hit bug 1159279 and have to work around it. We will "
"pretend that we could not reach the files for it.")
continue
# Sometimes running jobs have status unknown in buildapi
if status in (RUNNING, PENDING, UNKNOWN):
LOG.debug(
"We found a running/pending build job. We don't search anymore."
)
running_job = job
# We cannot call _find_files for a running job
continue
# Having a coalesced build is the same as not having a build available
if status == COALESCED:
LOG.debug(
"The build we found was a coalesced one; this is the same as "
"non-existant.")
continue
# Successful or failed jobs may have the files we need
files = _find_files(job)
if files != [] and _all_urls_reachable(files.values()):
working_job = job
break
else:
LOG.debug("We can't determine the files for this build or "
"can't reach them.")
files = None
LOG.info("We found a job that finished but it did not "
"produced files. status: %d" % status)
failed_job = job
# End of for loop
if working_job:
# We found a build job with the necessary files. It could be a
# successful job, a running job that already emitted files or a
# testfailed job
LOG.debug(str(working_job))
LOG.info("We have the necessary files to trigger the downstream job.")
# We have the files needed to trigger the test job
builder_to_trigger = buildername
elif running_job:
LOG.info(
"We found a running/pending build job. We will not trigger another one."
)
LOG.info(
"You have to run the script again after the build job is finished to "
"trigger %s." % buildername)
builder_to_trigger = None
elif failed_job:
LOG.info(
"The build job %s failed on revision %s without generating the "
"necessary files. We will not trigger anything." %
(build_buildername, revision))
builder_to_trigger = None
else:
# We were trying to build a test job, however, we determined
# that we need an upstream builder instead
if not trigger_build_if_missing or not _unique_build_request(
build_buildername, revision):
# This is a safeguard to prevent triggering a build
# job multiple times if it is not intentional
builder_to_trigger = None
if not trigger_build_if_missing:
LOG.info(
"We would have to triggered build '%s' in order to trigger "
"job '%s'. On this mode we will not trigger either." %
(build_buildername, buildername))
else:
LOG.info("We will trigger 1) "
"'%s' instead of 2) '%s'" %
(build_buildername, buildername))
LOG.info("We need to trigger the build job once (1) "
"in order to be able to run the test job (2).")
if repo_name == 'try':
LOG.info(
"You'll need to run the script again after (1) is done to "
"trigger (2).")
else:
LOG.info(
"After (1) is done and if no coalesccing happens the test "
"jobs associated with it will be triggered.")
builder_to_trigger = build_buildername
if files:
return builder_to_trigger, files['packageUrl'], files[
'testPackagesUrl']
else:
return builder_to_trigger, None, None
def determine_trigger_objective(revision, buildername, trigger_build_if_missing=True,
will_use_buildapi=False):
"""
Determine if we need to trigger any jobs and which job.
Returns:
* The name of the builder we need to trigger
* Files, if needed, to trigger such builder
"""
builder_to_trigger = None
files = None
repo_name = query_repo_name_from_buildername(buildername)
build_buildername = determine_upstream_builder(buildername)
if VALIDATE and not valid_builder(build_buildername):
raise MozciError("Our platforms mapping system has failed.")
if build_buildername == buildername:
# For a build job we know that we don't need files to
# trigger it and it's the build job we want to trigger
return build_buildername, None, None
# Let's figure out which jobs are associated to such revision
query_api = BuildApi()
# Let's only look at jobs that match such build_buildername
build_jobs = query_api.get_matching_jobs(repo_name, revision, build_buildername)
# We need to determine if we need to trigger a build job
# or the test job
working_job = None
running_job = None
failed_job = None
LOG.debug("List of matching jobs:")
for job in build_jobs:
try:
status = query_api.get_job_status(job)
except buildjson.BuildjsonException:
LOG.debug("We have hit bug 1159279 and have to work around it. We will "
"pretend that we could not reach the files for it.")
continue
# Sometimes running jobs have status unknown in buildapi
if status in (RUNNING, PENDING, UNKNOWN):
LOG.debug("We found a running/pending build job. We don't search anymore.")
running_job = job
# We cannot call _find_files for a running job
continue
# Having a coalesced build is the same as not having a build available
if status == COALESCED:
LOG.debug("The build we found was a coalesced one; this is the same as "
"non-existant.")
continue
# Successful or failed jobs may have the files we need
files = _find_files(job)
if files != [] and _all_urls_reachable(files.values()):
working_job = job
break
else:
LOG.debug("We can't determine the files for this build or "
"can't reach them.")
files = None
LOG.info("We found a job that finished but it did not "
"produced files. status: %d" % status)
failed_job = job
# End of for loop
if working_job:
# We found a build job with the necessary files. It could be a
# successful job, a running job that already emitted files or a
# testfailed job
LOG.debug(str(working_job))
LOG.info("We have the necessary files to trigger the downstream job.")
# We have the files needed to trigger the test job
builder_to_trigger = buildername
elif running_job:
LOG.info("We found a running/pending build job. We will not trigger another one.")
LOG.info("You have to run the script again after the build job is finished to "
"trigger %s." % buildername)
builder_to_trigger = None
elif failed_job:
LOG.info("The build job %s failed on revision %s without generating the "
"necessary files. We will not trigger anything." %
(build_buildername, revision))
builder_to_trigger = None
else:
# We were trying to build a test job, however, we determined
# that we need an upstream builder instead
if not trigger_build_if_missing or not _unique_build_request(build_buildername, revision):
# This is a safeguard to prevent triggering a build
# job multiple times if it is not intentional
builder_to_trigger = None
if not trigger_build_if_missing:
LOG.info("We would have to triggered build '%s' in order to trigger "
"job '%s'. On this mode we will not trigger either." %
(build_buildername, buildername))
else:
if will_use_buildapi:
LOG.info("We will trigger 1) '%s'" % build_buildername)
LOG.info("instead of 2) '%s'" % buildername)
LOG.info("We need to trigger the build job once (1) "
"in order to be able to run the test job (2).")
if repo_name == 'try':
LOG.info("You'll need to run the script again after (1) is done to "
"trigger (2).")
else:
LOG.info("After (1) is done and if no coalesccing happens the test "
"jobs associated with it will be triggered.")
builder_to_trigger = build_buildername
if files:
return builder_to_trigger, files['packageUrl'], files['testsUrl']
else:
return builder_to_trigger, None, None
