def testWaitForSingleJobHostsWithTimeout(self):
"""Discover a single host for this job then timeout."""
self.mox.StubOutWithMock(time, 'sleep')
self.mox.StubOutWithMock(job_status, 'gather_job_hostnames')
self.mox.StubOutWithMock(job_status, '_abort_jobs_if_timedout')
manager = self.mox.CreateMock(host_lock_manager.HostLockManager)
expected_hostnames = ['host1', 'host0']
expected_hosts = [FakeHost(h) for h in expected_hostnames]
job = FakeJob(7, hostnames=[None, None])
time.sleep(mox.IgnoreArg()).MultipleTimes()
job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(False)
self.expect_hosts_query_and_lock([job], manager, [], False)
# First, only one test in the job has had a host assigned at all.
# Since no hosts are running, expect no locking.
job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(False)
job.hostnames = [None] + expected_hostnames[1:]
self.expect_hosts_query_and_lock([job], manager, [], False)
# Then, that host starts running, but no other tests have hosts.
job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(False)
self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:])
# The second test gets a host assigned, but it's not yet running.
# Since no new running hosts are found, no locking should happen.
job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(False)
job.hostnames = expected_hostnames
self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:],
False)
# A timeout occurs, and only the locked hosts should be returned.
job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(),
mox.IgnoreArg(),
mox.IgnoreArg()).AndReturn(True)
# The last loop update; doesn't impact behavior.
job_status.gather_job_hostnames(mox.IgnoreArg(),
job).AndReturn(expected_hostnames)
self.mox.ReplayAll()
# Because of the timeout only one host is returned.
expect_timeout_hostnames = ['host0']
self.assertEquals(
sorted(expect_timeout_hostnames),
sorted(
job_status.wait_for_and_lock_job_hosts(
self.afe, [job],
manager,
wait_timeout_mins=DEFAULT_WAITTIMEOUT_MINS)))
def testWaitForMultiJobHostsToRunAndGetLocked(self):
"""Ensure we lock all running hosts for all jobs as discovered."""
self.mox.StubOutWithMock(time, 'sleep')
self.mox.StubOutWithMock(job_status, 'gather_job_hostnames')
manager = self.mox.CreateMock(host_lock_manager.HostLockManager)
expected_hostnames = ['host1', 'host0', 'host2']
expected_hosts = [FakeHost(h) for h in expected_hostnames]
job0 = FakeJob(0, hostnames=[])
job1 = FakeJob(1, hostnames=[])
time.sleep(mox.IgnoreArg()).MultipleTimes()
# First, only one test in either job has had a host assigned at all.
# Since no hosts are running, expect no locking.
job0.hostnames = [None, expected_hostnames[2]]
job1.hostnames = [None]
self.expect_hosts_query_and_lock([job0, job1], manager, [], False)
# Then, that host starts running, but no other tests have hosts.
self.expect_hosts_query_and_lock([job0, job1], manager,
expected_hosts[2:])
# The test in the second job gets a host assigned, but it's not yet
# running.
# Since no new running hosts are found, no locking should happen.
job1.hostnames = expected_hostnames[1:2]
self.expect_hosts_query_and_lock([job0, job1], manager,
expected_hosts[2:], False)
# The second job's test's host starts running as well.
self.expect_hosts_query_and_lock([job0, job1], manager,
expected_hosts[1:])
# All three hosts across both jobs are now running.
job0.hostnames = [expected_hostnames[0], expected_hostnames[2]]
self.expect_hosts_query_and_lock([job0, job1], manager, expected_hosts)
# The last loop update; doesn't impact behavior.
job_status.gather_job_hostnames(mox.IgnoreArg(),
job0).AndReturn(job0.hostnames)
job_status.gather_job_hostnames(mox.IgnoreArg(),
job1).AndReturn(job1.hostnames)
self.mox.ReplayAll()
self.assertEquals(
sorted(expected_hostnames),
sorted(
job_status.wait_for_and_lock_job_hosts(self.afe, [job0, job1],
manager)))
def testWaitForSingleJobHostsToRunAndGetLockedSerially(self):
"""Lock running hosts as discovered, serially."""
self.mox.StubOutWithMock(time, 'sleep')
self.mox.StubOutWithMock(job_status, 'gather_job_hostnames')
manager = self.mox.CreateMock(host_lock_manager.HostLockManager)
expected_hostnames = ['host1', 'host0']
expected_hosts = [FakeHost(h) for h in expected_hostnames]
job = FakeJob(7, hostnames=[None, None])
time.sleep(mox.IgnoreArg()).MultipleTimes()
self.expect_hosts_query_and_lock([job], manager, [], False)
# First, only one test in the job has had a host assigned at all.
# Since no hosts are running, expect no locking.
job.hostnames = [None] + expected_hostnames[1:]
self.expect_hosts_query_and_lock([job], manager, [], False)
# Then, that host starts running, but no other tests have hosts.
self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:])
# The second test gets a host assigned, but it's not yet running.
# Since no new running hosts are found, no locking should happen.
job.hostnames = expected_hostnames
self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:],
False)
# The second test's host starts running as well, and the first stops.
self.expect_hosts_query_and_lock([job], manager, expected_hosts[:1])
# The last loop update; doesn't impact behavior.
job_status.gather_job_hostnames(mox.IgnoreArg(),
job).AndReturn(expected_hostnames)
self.mox.ReplayAll()
self.assertEquals(
sorted(expected_hostnames),
sorted(
job_status.wait_for_and_lock_job_hosts(self.afe, [job],
manager)))