def testWaitForSingleJobHostsWithTimeout(self): """Discover a single host for this job then timeout.""" self.mox.StubOutWithMock(time, 'sleep') self.mox.StubOutWithMock(job_status, 'gather_job_hostnames') self.mox.StubOutWithMock(job_status, '_abort_jobs_if_timedout') manager = self.mox.CreateMock(host_lock_manager.HostLockManager) expected_hostnames = ['host1', 'host0'] expected_hosts = [FakeHost(h) for h in expected_hostnames] job = FakeJob(7, hostnames=[None, None]) time.sleep(mox.IgnoreArg()).MultipleTimes() job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(False) self.expect_hosts_query_and_lock([job], manager, [], False) # First, only one test in the job has had a host assigned at all. # Since no hosts are running, expect no locking. job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(False) job.hostnames = [None] + expected_hostnames[1:] self.expect_hosts_query_and_lock([job], manager, [], False) # Then, that host starts running, but no other tests have hosts. job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(False) self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:]) # The second test gets a host assigned, but it's not yet running. # Since no new running hosts are found, no locking should happen. job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(False) job.hostnames = expected_hostnames self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:], False) # A timeout occurs, and only the locked hosts should be returned. job_status._abort_jobs_if_timedout(mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg(), mox.IgnoreArg()).AndReturn(True) # The last loop update; doesn't impact behavior. job_status.gather_job_hostnames(mox.IgnoreArg(), job).AndReturn(expected_hostnames) self.mox.ReplayAll() # Because of the timeout only one host is returned. expect_timeout_hostnames = ['host0'] self.assertEquals( sorted(expect_timeout_hostnames), sorted( job_status.wait_for_and_lock_job_hosts( self.afe, [job], manager, wait_timeout_mins=DEFAULT_WAITTIMEOUT_MINS)))
def testWaitForMultiJobHostsToRunAndGetLocked(self): """Ensure we lock all running hosts for all jobs as discovered.""" self.mox.StubOutWithMock(time, 'sleep') self.mox.StubOutWithMock(job_status, 'gather_job_hostnames') manager = self.mox.CreateMock(host_lock_manager.HostLockManager) expected_hostnames = ['host1', 'host0', 'host2'] expected_hosts = [FakeHost(h) for h in expected_hostnames] job0 = FakeJob(0, hostnames=[]) job1 = FakeJob(1, hostnames=[]) time.sleep(mox.IgnoreArg()).MultipleTimes() # First, only one test in either job has had a host assigned at all. # Since no hosts are running, expect no locking. job0.hostnames = [None, expected_hostnames[2]] job1.hostnames = [None] self.expect_hosts_query_and_lock([job0, job1], manager, [], False) # Then, that host starts running, but no other tests have hosts. self.expect_hosts_query_and_lock([job0, job1], manager, expected_hosts[2:]) # The test in the second job gets a host assigned, but it's not yet # running. # Since no new running hosts are found, no locking should happen. job1.hostnames = expected_hostnames[1:2] self.expect_hosts_query_and_lock([job0, job1], manager, expected_hosts[2:], False) # The second job's test's host starts running as well. self.expect_hosts_query_and_lock([job0, job1], manager, expected_hosts[1:]) # All three hosts across both jobs are now running. job0.hostnames = [expected_hostnames[0], expected_hostnames[2]] self.expect_hosts_query_and_lock([job0, job1], manager, expected_hosts) # The last loop update; doesn't impact behavior. job_status.gather_job_hostnames(mox.IgnoreArg(), job0).AndReturn(job0.hostnames) job_status.gather_job_hostnames(mox.IgnoreArg(), job1).AndReturn(job1.hostnames) self.mox.ReplayAll() self.assertEquals( sorted(expected_hostnames), sorted( job_status.wait_for_and_lock_job_hosts(self.afe, [job0, job1], manager)))
def testWaitForSingleJobHostsToRunAndGetLockedSerially(self): """Lock running hosts as discovered, serially.""" self.mox.StubOutWithMock(time, 'sleep') self.mox.StubOutWithMock(job_status, 'gather_job_hostnames') manager = self.mox.CreateMock(host_lock_manager.HostLockManager) expected_hostnames = ['host1', 'host0'] expected_hosts = [FakeHost(h) for h in expected_hostnames] job = FakeJob(7, hostnames=[None, None]) time.sleep(mox.IgnoreArg()).MultipleTimes() self.expect_hosts_query_and_lock([job], manager, [], False) # First, only one test in the job has had a host assigned at all. # Since no hosts are running, expect no locking. job.hostnames = [None] + expected_hostnames[1:] self.expect_hosts_query_and_lock([job], manager, [], False) # Then, that host starts running, but no other tests have hosts. self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:]) # The second test gets a host assigned, but it's not yet running. # Since no new running hosts are found, no locking should happen. job.hostnames = expected_hostnames self.expect_hosts_query_and_lock([job], manager, expected_hosts[1:], False) # The second test's host starts running as well, and the first stops. self.expect_hosts_query_and_lock([job], manager, expected_hosts[:1]) # The last loop update; doesn't impact behavior. job_status.gather_job_hostnames(mox.IgnoreArg(), job).AndReturn(expected_hostnames) self.mox.ReplayAll() self.assertEquals( sorted(expected_hostnames), sorted( job_status.wait_for_and_lock_job_hosts(self.afe, [job], manager)))