def test_build_status_returns_finished_after_all_subjobs_complete_and_slaves_finished( self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=3) build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config()) build.allocate_slave( mock_slave) # all three subjobs are now "in progress" # Mock out call to create build artifacts after subjobs complete build._create_build_artifact = MagicMock() for subjob in subjobs: build.mark_subjob_complete(subjob.subjob_id()) # Note: this was never a unit test! We have to wait for a thread to complete post build # actions here. TODO: Fix this poll.wait_for(lambda: build._postbuild_tasks_are_finished, 5) # Verify build artifacts was called after subjobs completed build._create_build_artifact.assert_called_once_with() build.finish() status = build._status() self.assertTrue(build._subjobs_are_finished) self.assertTrue(build._postbuild_tasks_are_finished) self.assertTrue(build._teardowns_finished) self.assertEqual(status, BuildStatus.FINISHED)
def test_build_status_returns_finished_after_all_subjobs_complete_and_slaves_finished(self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=3) build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config(self._FAKE_MAX_EXECUTORS)) build.allocate_slave(mock_slave) # all three subjobs are now "in progress" # Mock out call to create build artifacts after subjobs complete build._create_build_artifact = MagicMock() for subjob in subjobs: build.mark_subjob_complete(subjob.subjob_id()) # Note: this was never a unit test! We have to wait for a thread to complete post build # actions here. TODO: Fix this poll.wait_for(lambda: build._postbuild_tasks_are_finished, 5) # Verify build artifacts was called after subjobs completed build._create_build_artifact.assert_called_once_with() build.finish() status = build._status() self.assertTrue(build._subjobs_are_finished) self.assertTrue(build._postbuild_tasks_are_finished) self.assertTrue(build._teardowns_finished) self.assertEqual(status, BuildStatus.FINISHED)
def test_build_status_returns_finished_after_all_subjobs_complete_and_slaves_finished( self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=3) postbuild_tasks_complete_event = Event() build = Build(BuildRequest({})) build._project_type = mock_project_type build._create_build_artifact = MagicMock() self._on_async_postbuild_tasks_completed( build, postbuild_tasks_complete_event.set) build.prepare(subjobs, self._create_job_config()) build.allocate_slave( mock_slave) # all three subjobs are now "in progress" for subjob in subjobs: build.complete_subjob(subjob.subjob_id()) # Wait for the async thread to complete executing postbuild tasks. self.assertTrue( postbuild_tasks_complete_event.wait(timeout=2), 'Postbuild tasks should complete within a few' 'seconds.') # Verify build artifacts was called after subjobs completed build._create_build_artifact.assert_called_once_with() self.assertTrue(build._subjobs_are_finished) self.assertEqual(build._status(), BuildStatus.FINISHED)
def test_allocate_slave_increments_by_per_slave_when_max_not_inf_and_less_than_num(self): build = Build(BuildRequest({})) build._max_executors_per_slave = 5 slave = Mock() slave.num_executors = 10 build.allocate_slave(slave) self.assertEqual(build._num_executors_allocated, 5, "Should be incremented by num executors")
def test_allocate_slave_increments_by_num_executors_when_max_is_inf(self): build = Build(BuildRequest({})) slave = Mock() slave.num_executors = 10 build.allocate_slave(slave) self.assertEqual(build._num_executors_allocated, 10, "Should be incremented by num executors")
def test_need_more_slaves_returns_true_if_max_processes_is_not_reached(self): subjobs = self._create_subjobs(count=8) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=5) build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config(max_executors=8)) build.allocate_slave(mock_slave) self.assertTrue(build.needs_more_slaves(), "if max_processes is not reached, we should need more slaves")
def test_allocate_slave_increments_by_per_slave_when_max_not_inf_and_less_than_num( self): build = Build(BuildRequest({})) build._max_executors_per_slave = 5 slave = Mock() slave.num_executors = 10 build.allocate_slave(slave) self.assertEqual(build._num_executors_allocated, 5, "Should be incremented by num executors")
def test_need_more_slaves_returns_false_if_max_processes_is_reached(self): subjobs = self._create_subjobs(count=5) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=1) build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config(max_executors=1)) build.allocate_slave(mock_slave) self.assertFalse(build.needs_more_slaves(), "if max processes is reached, we shouldn't need more slaves")
def test_build_status_returns_building_after_some_subjobs_are_executing(self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=2) build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config(self._FAKE_MAX_EXECUTORS)) build.allocate_slave(mock_slave) # two out of three subjobs are now "in progress" status = build._status() self.assertEqual(status, BuildStatus.BUILDING)
def test_allocate_slave_calls_slave_setup(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave() build = Build(Mock(spec_set=BuildRequest)) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) build.allocate_slave(mock_slave) mock_slave.setup.assert_called_once_with(build)
def test_build_status_returns_building_after_setup_has_started(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave() build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) build.allocate_slave(mock_slave) self.assertEqual(build._status(), BuildStatus.BUILDING, 'Build status should be BUILDING after setup has started on slaves.')
def test_allocate_slave_doesnt_use_more_than_max_executors(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'mock command' mock_slave = self._create_mock_slave() build = Build(BuildRequest({'setup': fake_setup_command})) build.prepare(subjobs, mock_project_type, self._create_job_config(1)) build.allocate_slave(mock_slave) self.assertEqual(build._num_allocated_executors, build._max_executors)
def test_need_more_slaves_returns_false_if_max_processes_is_reached(self): subjobs = self._create_subjobs(count=5) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=1) build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config(max_executors=1)) build.allocate_slave(mock_slave) self.assertFalse( build.needs_more_slaves(), "if max processes is reached, we shouldn't need more slaves")
def test_build_status_returns_building_after_setup_is_complete_and_subjobs_are_executing(self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=2) build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config()) build.allocate_slave(mock_slave) build.begin_subjob_executions_on_slave(mock_slave) # two out of three subjobs are now in progress self.assertEqual(build._status(), BuildStatus.BUILDING, 'Build status should be BUILDING after subjobs have started executing on slaves.')
def test_need_more_slaves_returns_true_if_max_processes_is_not_reached( self): subjobs = self._create_subjobs(count=8) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=5) build = Build(BuildRequest({})) build._project_type = mock_project_type build.prepare(subjobs, self._create_job_config(max_executors=8)) build.allocate_slave(mock_slave) self.assertTrue( build.needs_more_slaves(), "if max_processes is not reached, we should need more slaves")
def test_build_status_returns_building_after_setup_has_started(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave() build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config()) build.allocate_slave(mock_slave) self.assertEqual( build._status(), BuildStatus.BUILDING, 'Build status should be BUILDING after setup has started on slaves.' )
def test_allocate_slave_calls_slave_setup(self): # arrange subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'docker pull my:leg' mock_slave = self._create_mock_slave() # act build = Build(BuildRequest({'setup': fake_setup_command})) build.prepare(subjobs, mock_project_type, self._create_job_config(self._FAKE_MAX_EXECUTORS)) build.allocate_slave(mock_slave) # assert mock_slave.setup.assert_called_once_with(build.build_id(), project_type_params={'setup': fake_setup_command})
def test_build_doesnt_use_more_than_max_executors_per_slave(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'mock command' mock_slaves = [ self._create_mock_slave(num_executors=5) for _ in range(3) ] max_executors_per_slave = 2 expected_total_num_executors_used = 6 # We expect the build to use 2 executors on each of the 3 slaves. build = Build(BuildRequest({'setup': fake_setup_command})) build.execute_next_subjob_on_slave = MagicMock() build.prepare( subjobs, mock_project_type, self._create_job_config( max_executors_per_slave=max_executors_per_slave)) [build.allocate_slave(mock_slave) for mock_slave in mock_slaves] expected_current_num_executors_used = 0 for i in range(len(mock_slaves)): build.begin_subjob_executions_on_slave(mock_slaves[i]) expected_current_num_executors_used += max_executors_per_slave self.assertEqual( build.execute_next_subjob_on_slave.call_count, expected_current_num_executors_used, 'After allocating {} slaves, build with max_executors_per_slave set to {} should only be using {} ' 'executors.'.format(i + 1, max_executors_per_slave, expected_current_num_executors_used)) self.assertEqual( build.execute_next_subjob_on_slave.call_count, expected_total_num_executors_used, 'Build should start executing as many subjobs per slave as its max_executors_per_slave setting.' )
def test_build_doesnt_use_more_than_max_executors_per_slave(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'mock command' mock_slaves = [self._create_mock_slave(num_executors=5) for _ in range(3)] max_executors_per_slave = 2 expected_total_num_executors_used = 6 # We expect the build to use 2 executors on each of the 3 slaves. build = Build(BuildRequest({'setup': fake_setup_command})) build._project_type = mock_project_type build.execute_next_subjob_or_teardown_slave = MagicMock() build.prepare(subjobs, self._create_job_config(max_executors_per_slave=max_executors_per_slave)) [build.allocate_slave(mock_slave) for mock_slave in mock_slaves] expected_current_num_executors_used = 0 for i in range(len(mock_slaves)): build.begin_subjob_executions_on_slave(mock_slaves[i]) expected_current_num_executors_used += max_executors_per_slave self.assertEqual( build.execute_next_subjob_or_teardown_slave.call_count, expected_current_num_executors_used, 'After allocating {} slaves, build with max_executors_per_slave set to {} should only be using {} ' 'executors.'.format(i + 1, max_executors_per_slave, expected_current_num_executors_used)) self.assertEqual( build.execute_next_subjob_or_teardown_slave.call_count, expected_total_num_executors_used, 'Build should start executing as many subjobs per slave as its max_executors_per_slave setting.')
def test_build_doesnt_use_more_than_max_executors(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'mock command' mock_slaves = [ self._create_mock_slave(num_executors=5) for _ in range(3) ] expected_num_executors = 12 # We expect the build to use 12 out of 15 available executors. build = Build(BuildRequest({'setup': fake_setup_command})) build.execute_next_subjob_on_slave = MagicMock() build.prepare( subjobs, mock_project_type, self._create_job_config(max_executors=expected_num_executors)) [build.allocate_slave(mock_slave) for mock_slave in mock_slaves] [ build.begin_subjob_executions_on_slave(mock_slave) for mock_slave in mock_slaves ] self.assertEqual( build.execute_next_subjob_on_slave.call_count, expected_num_executors, 'Build should start executing as many subjobs as its max_executors setting.' )
def test_build_status_returns_building_after_setup_is_complete_and_subjobs_are_executing( self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=2) build = Build(BuildRequest({})) build.prepare(subjobs, mock_project_type, self._create_job_config()) build.allocate_slave(mock_slave) build.begin_subjob_executions_on_slave( mock_slave) # two out of three subjobs are now in progress self.assertEqual( build._status(), BuildStatus.BUILDING, 'Build status should be BUILDING after subjobs have started executing on slaves.' )
def test_allocate_slave_calls_slave_setup(self): # arrange subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'docker pull my:leg' mock_slave = self._create_mock_slave() # act build = Build(BuildRequest({'setup': fake_setup_command})) build.prepare(subjobs, mock_project_type, self._create_job_config()) build.allocate_slave(mock_slave) # assert mock_slave.setup.assert_called_once_with( build.build_id(), project_type_params={'setup': fake_setup_command})
def test_build_status_returns_finished_after_all_subjobs_complete_and_slaves_finished(self): subjobs = self._create_subjobs(count=3) mock_project_type = self._create_mock_project_type() mock_slave = self._create_mock_slave(num_executors=3) postbuild_tasks_complete_event = Event() build = Build(BuildRequest({})) build._project_type = mock_project_type build._create_build_artifact = MagicMock() self._on_async_postbuild_tasks_completed(build, postbuild_tasks_complete_event.set) build.prepare(subjobs, self._create_job_config()) build.allocate_slave(mock_slave) # all three subjobs are now "in progress" for subjob in subjobs: build.complete_subjob(subjob.subjob_id()) # Wait for the async thread to complete executing postbuild tasks. self.assertTrue(postbuild_tasks_complete_event.wait(timeout=2), 'Postbuild tasks should complete within a few' 'seconds.') # Verify build artifacts was called after subjobs completed build._create_build_artifact.assert_called_once_with() self.assertTrue(build._subjobs_are_finished) self.assertEqual(build._status(), BuildStatus.FINISHED)
def test_build_doesnt_use_more_than_max_executors(self): subjobs = self._create_subjobs() mock_project_type = self._create_mock_project_type() fake_setup_command = 'mock command' mock_slaves = [self._create_mock_slave(num_executors=5) for _ in range(3)] expected_num_executors = 12 # We expect the build to use 12 out of 15 available executors. build = Build(BuildRequest({'setup': fake_setup_command})) build._project_type = mock_project_type build.execute_next_subjob_or_teardown_slave = MagicMock() build.prepare(subjobs, self._create_job_config(max_executors=expected_num_executors)) [build.allocate_slave(mock_slave) for mock_slave in mock_slaves] [build.begin_subjob_executions_on_slave(mock_slave) for mock_slave in mock_slaves] self.assertEqual(build.execute_next_subjob_or_teardown_slave.call_count, expected_num_executors, 'Build should start executing as many subjobs as its max_executors setting.')